Skip to content

Commit 0ea1a2d

Browse files
xiazhveraxiazhvera
authored
Merge pull request #313 from aws/pipeline
1. Added the scripts for python SDK deployment. The deployment pipeline has the following stages a. Verify the package version. Check if we need push a new release continuous-delivery/test_version_exists.yml continuous-delivery/test_version_exists b. Deliver to Test pypi and test continuous-delivery/publish_to_test_pypi.yml continuous-delivery/test_test_pypi.yml c. Publish to prod pypi continuous-delivery/publish_to_prod_pypi.yml continuous-delivery/test_prod_pypi.yml 2. Added "--print_discover_resp_only" flag for greengrass sample. The flag is used for deployment test only, once the flag is set, the program always return true as long as the SDK API is imported correctly 3. setup_test.py The setup file is for test deployment only. Since the package name is already used in another account, for the test pypi, we use a different package name. By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.
2 parents b5ba6fe + 6a7da4e commit 0ea1a2d

9 files changed

+235
-1
lines changed

‎continuous-delivery/pip-install-with-retry.py

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
import time
2+
import sys
3+
import subprocess
4+
5+
DOCS = """Given cmdline args, executes: python3 -m pip install [args...]
6+
Keeps retrying until the new version becomes available in pypi (or we time out)"""
7+
if len(sys.argv) < 2:
8+
sys.exit(DOCS)
9+
10+
RETRY_INTERVAL_SECS = 10
11+
GIVE_UP_AFTER_SECS = 60 * 15
12+
13+
pip_install_args = [sys.executable, '-m', 'pip', 'install'] + sys.argv[1:]
14+
15+
start_time = time.time()
16+
while True:
17+
print(subprocess.list2cmdline(pip_install_args))
18+
result = subprocess.run(pip_install_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
19+
20+
stdout = result.stdout.decode().strip()
21+
if stdout:
22+
print(stdout)
23+
24+
if result.returncode == 0:
25+
# success
26+
sys.exit(0)
27+
28+
if "could not find a version" in stdout.lower():
29+
elapsed_secs = time.time() - start_time
30+
if elapsed_secs < GIVE_UP_AFTER_SECS:
31+
# try again
32+
print("Retrying in", RETRY_INTERVAL_SECS, "secs...")
33+
time.sleep(RETRY_INTERVAL_SECS)
34+
continue
35+
else:
36+
print("Giving up on retries after", int(elapsed_secs), "total secs.")
37+
38+
# fail
39+
sys.exit(result.returncode)

‎continuous-delivery/publish_to_prod_pypi.yml

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
version: 0.2
2+
# this image assumes Ubuntu 14.04 base image
3+
phases:
4+
install:
5+
commands:
6+
- sudo apt-get update -y
7+
- sudo apt-get install python3 python3-pip -y
8+
- export PATH=$PATH:$HOME/.local/bin
9+
- python3 -m pip install --user --upgrade pip
10+
- python3 -m pip install --user --upgrade twine setuptools wheel awscli PyOpenSSL six
11+
pre_build:
12+
commands:
13+
- cd aws-iot-device-sdk-python
14+
- pypirc=$(aws secretsmanager get-secret-value --secret-id "prod/aws-crt-python/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc
15+
- export PKG_VERSION=$(git describe --tags | cut -f2 -dv)
16+
- echo "Updating package version to ${PKG_VERSION}"
17+
- sed --in-place -E "s/__version__ = \".+\"/__version__ = \"${PKG_VERSION}\"/" AWSIoTPythonSDK/__init__.py
18+
build:
19+
commands:
20+
- echo Build started on `date`
21+
- python3 setup.py sdist bdist_wheel --universal
22+
- python3 -m twine upload -r pypi dist/*
23+
post_build:
24+
commands:
25+
- echo Build completed on `date`

‎continuous-delivery/publish_to_test_pypi.yml

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
version: 0.2
2+
# this image assumes Ubuntu 14.04 base image
3+
phases:
4+
install:
5+
commands:
6+
- sudo apt-get update -y
7+
- sudo apt-get install python3 python3-pip -y
8+
- export PATH=$PATH:$HOME/.local/bin
9+
- python3 -m pip install --user --upgrade pip
10+
- python3 -m pip install --user --upgrade twine setuptools wheel awscli PyOpenSSL six
11+
pre_build:
12+
commands:
13+
- pypirc=$(aws secretsmanager get-secret-value --secret-id "alpha/aws-crt-python/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc
14+
- cd aws-iot-device-sdk-python
15+
- export PKG_VERSION=$(git describe --tags | cut -f2 -dv)
16+
- echo "Updating package version to ${PKG_VERSION}"
17+
- sed --in-place -E "s/__version__ = \".+\"/__version__ = \"${PKG_VERSION}\"/" AWSIoTPythonSDK/__init__.py
18+
build:
19+
commands:
20+
- echo Build started on `date`
21+
- python3 setup_test.py sdist bdist_wheel --universal
22+
- python3 -m twine upload -r testpypi dist/* --verbose
23+
post_build:
24+
commands:
25+
- echo Build completed on `date`

‎continuous-delivery/test_prod_pypi.yml

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
version: 0.2
2+
# this image assumes Ubuntu 14.04 base image
3+
phases:
4+
install:
5+
commands:
6+
- sudo apt-get update -y
7+
- sudo apt-get install python3 python3-pip -y
8+
- python3 -m pip install --upgrade pip
9+
- python3 -m pip install --upgrade setuptools
10+
11+
pre_build:
12+
commands:
13+
- curl https://www.amazontrust.com/repository/AmazonRootCA1.pem --output /tmp/AmazonRootCA1.pem
14+
- cert=$(aws secretsmanager get-secret-value --secret-id "unit-test/certificate" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$cert" > /tmp/certificate.pem
15+
- key=$(aws secretsmanager get-secret-value --secret-id "unit-test/privatekey" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$key" > /tmp/privatekey.pem
16+
- ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
17+
build:
18+
commands:
19+
- echo Build started on `date`
20+
- cd aws-iot-device-sdk-python
21+
- CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv)
22+
- python3 continuous-delivery/pip-install-with-retry.py --no-cache-dir --user AWSIoTPythonSDK==$CURRENT_TAG_VERSION
23+
- python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem --print_discover_resp_only
24+
25+
post_build:
26+
commands:
27+
- echo Build completed on `date`
28+

‎continuous-delivery/test_test_pypi.yml

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
version: 0.2
2+
# this image assumes Ubuntu 14.04 base image
3+
phases:
4+
install:
5+
commands:
6+
- sudo apt-get update -y
7+
- sudo apt-get install python3 python3-pip -y
8+
- python3 -m pip install --upgrade pip
9+
- python3 -m pip install --upgrade setuptools
10+
11+
pre_build:
12+
commands:
13+
- curl https://www.amazontrust.com/repository/AmazonRootCA1.pem --output /tmp/AmazonRootCA1.pem
14+
- cert=$(aws secretsmanager get-secret-value --secret-id "unit-test/certificate" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$cert" > /tmp/certificate.pem
15+
- key=$(aws secretsmanager get-secret-value --secret-id "unit-test/privatekey" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$key" > /tmp/privatekey.pem
16+
- ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
17+
build:
18+
commands:
19+
- echo Build started on `date`
20+
- cd aws-iot-device-sdk-python
21+
- CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv)
22+
# this is here because typing isn't in testpypi, so pull it from prod instead
23+
- python3 -m pip install typing
24+
- python3 continuous-delivery/pip-install-with-retry.py -i https://testpypi.python.org/simple --user AWSIoTPythonSDK-V1==$CURRENT_TAG_VERSION
25+
- python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem --print_discover_resp_only
26+
27+
post_build:
28+
commands:
29+
- echo Build completed on `date`
30+

‎continuous-delivery/test_version_exists

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
set -x
4+
# force a failure if there's no tag
5+
git describe --tags
6+
# now get the tag
7+
CURRENT_TAG=$(git describe --tags | cut -f2 -dv)
8+
# convert v0.2.12-2-g50254a9 to 0.2.12
9+
CURRENT_TAG_VERSION=$(git describe --tags | cut -f1 -d'-' | cut -f2 -dv)
10+
# if there's a hash on the tag, then this is not a release tagged commit
11+
if [ "$CURRENT_TAG" != "$CURRENT_TAG_VERSION" ]; then
12+
echo "Current tag version is not a release tag, cut a new release if you want to publish."
13+
exit 1
14+
fi
15+
16+
if python3 -m pip install --no-cache-dir -vvv AWSIoTPythonSDK==$CURRENT_TAG_VERSION; then
17+
echo "$CURRENT_TAG_VERSION is already in pypi, cut a new tag if you want to upload another version."
18+
exit 1
19+
fi
20+
21+
echo "$CURRENT_TAG_VERSION currently does not exist in pypi, allowing pipeline to continue."
22+
exit 0

‎continuous-delivery/test_version_exists.yml

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
version: 0.2
2+
#this build spec assumes the ubuntu 14.04 trusty image
3+
#this build run simply verifies we haven't published something at this tag yet.
4+
#if we have we fail the build and stop the pipeline, if we haven't we allow the pipeline to run.
5+
phases:
6+
install:
7+
commands:
8+
- sudo apt-get update -y
9+
- sudo apt-get install python3 python3-pip -y
10+
- pip3 install --upgrade setuptools
11+
pre_build:
12+
commands:
13+
- echo Build start on `date`
14+
build:
15+
commands:
16+
- cd aws-iot-device-sdk-python
17+
- bash ./continuous-delivery/test_version_exists
18+
post_build:
19+
commands:
20+
- echo Build completed on `date`
21+

‎samples/greengrass/basicDiscovery.py

+7-1
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ def customOnMessage(message):
4747
help="Operation modes: %s"%str(AllowedActions))
4848
parser.add_argument("-M", "--message", action="store", dest="message", default="Hello World!",
4949
help="Message to publish")
50+
#--print_discover_resp_only used for delopyment testing. The test run will return 0 as long as the SDK installed correctly.
51+
parser.add_argument("-p", "--print_discover_resp_only", action="store_true", dest="print_only", default=False)
5052

5153
args = parser.parse_args()
5254
host = args.host
@@ -56,6 +58,7 @@ def customOnMessage(message):
5658
clientId = args.thingName
5759
thingName = args.thingName
5860
topic = args.topic
61+
print_only = args.print_only
5962

6063
if args.mode not in AllowedActions:
6164
parser.error("Unknown --mode option %s. Must be one of %s" % (args.mode, str(AllowedActions)))
@@ -94,7 +97,7 @@ def customOnMessage(message):
9497
discoveryInfoProvider.configureCredentials(rootCAPath, certificatePath, privateKeyPath)
9598
discoveryInfoProvider.configureTimeout(10) # 10 sec
9699

97-
retryCount = MAX_DISCOVERY_RETRIES
100+
retryCount = MAX_DISCOVERY_RETRIES if not print_only else 1
98101
discovered = False
99102
groupCA = None
100103
coreInfo = None
@@ -136,6 +139,9 @@ def customOnMessage(message):
136139
backOffCore.backOff()
137140

138141
if not discovered:
142+
# With print_discover_resp_only flag, we only woud like to check if the API get called correctly.
143+
if print_only:
144+
sys.exit(0)
139145
print("Discovery failed after %d retries. Exiting...\n" % (MAX_DISCOVERY_RETRIES))
140146
sys.exit(-1)
141147

‎setup_test.py

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# For test deployment with package AWSIoTPythonSDK. The package name has already taken. Therefore we used an
2+
# alternative name for test pypi.
3+
# prod_pypi : AWSIoTPythonSDK
4+
# test_pypi : AWSIoTPythonSDK-V1
5+
import sys
6+
sys.path.insert(0, 'AWSIoTPythonSDK')
7+
import AWSIoTPythonSDK
8+
currentVersion = AWSIoTPythonSDK.__version__
9+
10+
from distutils.core import setup
11+
setup(
12+
name = 'AWSIoTPythonSDK-V1',
13+
packages=['AWSIoTPythonSDK', 'AWSIoTPythonSDK.core',
14+
'AWSIoTPythonSDK.core.util', 'AWSIoTPythonSDK.core.shadow', 'AWSIoTPythonSDK.core.protocol',
15+
'AWSIoTPythonSDK.core.jobs',
16+
'AWSIoTPythonSDK.core.protocol.paho', 'AWSIoTPythonSDK.core.protocol.internal',
17+
'AWSIoTPythonSDK.core.protocol.connection', 'AWSIoTPythonSDK.core.greengrass',
18+
'AWSIoTPythonSDK.core.greengrass.discovery', 'AWSIoTPythonSDK.exception'],
19+
version = currentVersion,
20+
description = 'SDK for connecting to AWS IoT using Python.',
21+
author = 'Amazon Web Service',
22+
author_email = '',
23+
url = 'https://github.com/aws/aws-iot-device-sdk-python.git',
24+
download_url = 'https://s3.amazonaws.com/aws-iot-device-sdk-python/aws-iot-device-sdk-python-latest.zip',
25+
keywords = ['aws', 'iot', 'mqtt'],
26+
classifiers = [
27+
"Development Status :: 5 - Production/Stable",
28+
"Intended Audience :: Developers",
29+
"Natural Language :: English",
30+
"License :: OSI Approved :: Apache Software License",
31+
"Programming Language :: Python",
32+
"Programming Language :: Python :: 2.7",
33+
"Programming Language :: Python :: 3",
34+
"Programming Language :: Python :: 3.3",
35+
"Programming Language :: Python :: 3.4",
36+
"Programming Language :: Python :: 3.5"
37+
]
38+
)

0 commit comments

Comments
 (0)