Merge pull request #313 from aws/pipeline

1. Added the scripts for python SDK deployment. The deployment pipeline has the following stages a. Verify the package version. Check if we need push a new release continuous-delivery/test_version_exists.yml continuous-delivery/test_version_exists b. Deliver to Test pypi and test continuous-delivery/publish_to_test_pypi.yml continuous-delivery/test_test_pypi.yml c. Publish to prod pypi continuous-delivery/publish_to_prod_pypi.yml continuous-delivery/test_prod_pypi.yml 2. Added "--print_discover_resp_only" flag for greengrass sample. The flag is used for deployment test only, once the flag is set, the program always return true as long as the SDK API is imported correctly 3. setup_test.py The setup file is for test deployment only. Since the package name is already used in another account, for the test pypi, we use a different package name. By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.
aws · Apr 13, 2022 · 0ea1a2d · 0ea1a2d
2 parents b5ba6fe + 6a7da4e
commit 0ea1a2d
Show file tree

Hide file tree

Showing 9 changed files with 235 additions and 1 deletion.
diff --git a/continuous-delivery/pip-install-with-retry.py b/continuous-delivery/pip-install-with-retry.py
@@ -0,0 +1,39 @@
+import time
+import sys
+import subprocess
+
+DOCS = """Given cmdline args, executes: python3 -m pip install [args...]
+Keeps retrying until the new version becomes available in pypi (or we time out)"""
+if len(sys.argv) < 2:
+    sys.exit(DOCS)
+
+RETRY_INTERVAL_SECS = 10
+GIVE_UP_AFTER_SECS = 60 * 15
+
+pip_install_args = [sys.executable, '-m', 'pip', 'install'] + sys.argv[1:]
+
+start_time = time.time()
+while True:
+    print(subprocess.list2cmdline(pip_install_args))
+    result = subprocess.run(pip_install_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+    stdout = result.stdout.decode().strip()
+    if stdout:
+        print(stdout)
+
+    if result.returncode == 0:
+        # success
+        sys.exit(0)
+
+    if "could not find a version" in stdout.lower():
+        elapsed_secs = time.time() - start_time
+        if elapsed_secs < GIVE_UP_AFTER_SECS:
+            # try again
+            print("Retrying in", RETRY_INTERVAL_SECS, "secs...")
+            time.sleep(RETRY_INTERVAL_SECS)
+            continue
+        else:
+            print("Giving up on retries after", int(elapsed_secs), "total secs.")
+
+    # fail
+    sys.exit(result.returncode)
diff --git a/continuous-delivery/publish_to_prod_pypi.yml b/continuous-delivery/publish_to_prod_pypi.yml
@@ -0,0 +1,25 @@
+version: 0.2
+# this image assumes Ubuntu 14.04 base image
+phases:
+  install:
+    commands:
+      - sudo apt-get update -y
+      - sudo apt-get install python3 python3-pip -y
+      - export PATH=$PATH:$HOME/.local/bin
+      - python3 -m pip install --user --upgrade pip
+      - python3 -m pip install --user --upgrade twine setuptools wheel awscli PyOpenSSL six
+  pre_build:
+    commands:
+      - cd aws-iot-device-sdk-python
+      - pypirc=$(aws secretsmanager get-secret-value --secret-id "prod/aws-crt-python/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc
+      - export PKG_VERSION=$(git describe --tags | cut -f2 -dv)
+      - echo "Updating package version to ${PKG_VERSION}"
+      - sed --in-place -E "s/__version__ = \".+\"/__version__ = \"${PKG_VERSION}\"/" AWSIoTPythonSDK/__init__.py
+  build:
+    commands:
+      - echo Build started on `date`
+      - python3 setup.py sdist bdist_wheel --universal
+      - python3 -m twine upload -r pypi dist/*
+  post_build:
+    commands:
+      - echo Build completed on `date`
diff --git a/continuous-delivery/publish_to_test_pypi.yml b/continuous-delivery/publish_to_test_pypi.yml
@@ -0,0 +1,25 @@
+version: 0.2
+# this image assumes Ubuntu 14.04 base image
+phases:
+  install:
+    commands:
+      - sudo apt-get update -y
+      - sudo apt-get install python3 python3-pip -y
+      - export PATH=$PATH:$HOME/.local/bin
+      - python3 -m pip install --user --upgrade pip
+      - python3 -m pip install --user --upgrade twine setuptools wheel awscli PyOpenSSL six
+  pre_build:
+    commands:
+      - pypirc=$(aws secretsmanager get-secret-value --secret-id "alpha/aws-crt-python/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc
+      - cd aws-iot-device-sdk-python
+      - export PKG_VERSION=$(git describe --tags | cut -f2 -dv)
+      - echo "Updating package version to ${PKG_VERSION}"
+      - sed --in-place -E "s/__version__ = \".+\"/__version__ = \"${PKG_VERSION}\"/" AWSIoTPythonSDK/__init__.py
+  build:
+    commands:
+      - echo Build started on `date`
+      - python3 setup_test.py sdist bdist_wheel --universal
+      - python3 -m twine upload -r testpypi dist/* --verbose
+  post_build:
+    commands:
+      - echo Build completed on `date`
diff --git a/continuous-delivery/test_prod_pypi.yml b/continuous-delivery/test_prod_pypi.yml
@@ -0,0 +1,28 @@
+version: 0.2
+# this image assumes Ubuntu 14.04 base image
+phases:
+  install:
+    commands:
+      - sudo apt-get update -y
+      - sudo apt-get install python3 python3-pip -y
+      - python3 -m pip install --upgrade pip
+      - python3 -m pip install --upgrade setuptools
+
+  pre_build:
+    commands:
+      - curl https://www.amazontrust.com/repository/AmazonRootCA1.pem --output /tmp/AmazonRootCA1.pem
+      - cert=$(aws secretsmanager get-secret-value --secret-id "unit-test/certificate" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$cert" > /tmp/certificate.pem
+      - key=$(aws secretsmanager get-secret-value --secret-id "unit-test/privatekey" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$key" > /tmp/privatekey.pem
+      - ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
+  build:
+    commands:
+      - echo Build started on `date`
+      - cd aws-iot-device-sdk-python
+      - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv)
+      - python3 continuous-delivery/pip-install-with-retry.py --no-cache-dir --user AWSIoTPythonSDK==$CURRENT_TAG_VERSION
+      - python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem --print_discover_resp_only
+
+  post_build:
+    commands:
+      - echo Build completed on `date`
+
diff --git a/continuous-delivery/test_test_pypi.yml b/continuous-delivery/test_test_pypi.yml
@@ -0,0 +1,30 @@
+version: 0.2
+# this image assumes Ubuntu 14.04 base image
+phases:
+  install:
+    commands:
+      - sudo apt-get update -y
+      - sudo apt-get install python3 python3-pip -y
+      - python3 -m pip install --upgrade pip
+      - python3 -m pip install --upgrade setuptools
+
+  pre_build:
+    commands:
+      - curl https://www.amazontrust.com/repository/AmazonRootCA1.pem --output /tmp/AmazonRootCA1.pem
+      - cert=$(aws secretsmanager get-secret-value --secret-id "unit-test/certificate" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$cert" > /tmp/certificate.pem
+      - key=$(aws secretsmanager get-secret-value --secret-id "unit-test/privatekey" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$key" > /tmp/privatekey.pem
+      - ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
+  build:
+    commands:
+      - echo Build started on `date`
+      - cd aws-iot-device-sdk-python
+      - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv)
+      # this is here because typing isn't in testpypi, so pull it from prod instead
+      - python3 -m pip install typing
+      - python3 continuous-delivery/pip-install-with-retry.py -i https://testpypi.python.org/simple --user AWSIoTPythonSDK-V1==$CURRENT_TAG_VERSION
+      - python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem --print_discover_resp_only
+
+  post_build:
+    commands:
+      - echo Build completed on `date`
+
diff --git a/continuous-delivery/test_version_exists b/continuous-delivery/test_version_exists
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -e
+set -x
+# force a failure if there's no tag
+git describe --tags
+# now get the tag
+CURRENT_TAG=$(git describe --tags | cut -f2 -dv)
+# convert v0.2.12-2-g50254a9 to 0.2.12
+CURRENT_TAG_VERSION=$(git describe --tags | cut -f1 -d'-' | cut -f2 -dv)
+# if there's a hash on the tag, then this is not a release tagged commit
+if [ "$CURRENT_TAG" != "$CURRENT_TAG_VERSION" ]; then
+    echo "Current tag version is not a release tag, cut a new release if you want to publish."
+    exit 1
+fi
+
+if python3 -m pip install --no-cache-dir -vvv AWSIoTPythonSDK==$CURRENT_TAG_VERSION; then
+    echo "$CURRENT_TAG_VERSION is already in pypi, cut a new tag if you want to upload another version."
+    exit 1
+fi
+
+echo "$CURRENT_TAG_VERSION currently does not exist in pypi, allowing pipeline to continue."
+exit 0
diff --git a/continuous-delivery/test_version_exists.yml b/continuous-delivery/test_version_exists.yml
@@ -0,0 +1,21 @@
+version: 0.2
+#this build spec assumes the ubuntu 14.04 trusty image
+#this build run simply verifies we haven't published something at this tag yet.
+#if we have we fail the build and stop the pipeline, if we haven't we allow the pipeline to run.
+phases:
+  install:
+    commands:
+      - sudo apt-get update -y
+      - sudo apt-get install python3 python3-pip -y
+      - pip3 install --upgrade setuptools
+  pre_build:
+    commands:
+      - echo Build start on `date`
+  build:
+    commands:
+      - cd aws-iot-device-sdk-python
+      - bash ./continuous-delivery/test_version_exists
+  post_build:
+    commands:
+      - echo Build completed on `date`
+
diff --git a/samples/greengrass/basicDiscovery.py b/samples/greengrass/basicDiscovery.py
@@ -47,6 +47,8 @@ def customOnMessage(message):
                     help="Operation modes: %s"%str(AllowedActions))
 parser.add_argument("-M", "--message", action="store", dest="message", default="Hello World!",
                     help="Message to publish")
+#--print_discover_resp_only used for delopyment testing. The test run will return 0 as long as the SDK installed correctly.
+parser.add_argument("-p", "--print_discover_resp_only", action="store_true", dest="print_only", default=False)
 
 args = parser.parse_args()
 host = args.host
@@ -56,6 +58,7 @@ def customOnMessage(message):
 clientId = args.thingName
 thingName = args.thingName
 topic = args.topic
+print_only = args.print_only
 
 if args.mode not in AllowedActions:
     parser.error("Unknown --mode option %s. Must be one of %s" % (args.mode, str(AllowedActions)))
@@ -94,7 +97,7 @@ def customOnMessage(message):
 discoveryInfoProvider.configureCredentials(rootCAPath, certificatePath, privateKeyPath)
 discoveryInfoProvider.configureTimeout(10)  # 10 sec
 
-retryCount = MAX_DISCOVERY_RETRIES
+retryCount = MAX_DISCOVERY_RETRIES if not print_only else 1
 discovered = False
 groupCA = None
 coreInfo = None
@@ -136,6 +139,9 @@ def customOnMessage(message):
         backOffCore.backOff()
 
 if not discovered:
+    # With print_discover_resp_only flag, we only woud like to check if the API get called correctly. 
+    if print_only:
+        sys.exit(0)
     print("Discovery failed after %d retries. Exiting...\n" % (MAX_DISCOVERY_RETRIES))
     sys.exit(-1)
 

diff --git a/setup_test.py b/setup_test.py
@@ -0,0 +1,38 @@
+# For test deployment with package AWSIoTPythonSDK. The package name has already taken. Therefore we used an
+# alternative name for test pypi.
+# prod_pypi : AWSIoTPythonSDK
+# test_pypi : AWSIoTPythonSDK-V1
+import sys
+sys.path.insert(0, 'AWSIoTPythonSDK')
+import AWSIoTPythonSDK
+currentVersion = AWSIoTPythonSDK.__version__
+
+from distutils.core import setup
+setup(
+    name = 'AWSIoTPythonSDK-V1',
+    packages=['AWSIoTPythonSDK', 'AWSIoTPythonSDK.core',
+              'AWSIoTPythonSDK.core.util', 'AWSIoTPythonSDK.core.shadow', 'AWSIoTPythonSDK.core.protocol',
+              'AWSIoTPythonSDK.core.jobs',
+              'AWSIoTPythonSDK.core.protocol.paho', 'AWSIoTPythonSDK.core.protocol.internal',
+              'AWSIoTPythonSDK.core.protocol.connection', 'AWSIoTPythonSDK.core.greengrass',
+              'AWSIoTPythonSDK.core.greengrass.discovery', 'AWSIoTPythonSDK.exception'],
+    version = currentVersion,
+    description = 'SDK for connecting to AWS IoT using Python.',
+    author = 'Amazon Web Service',
+    author_email = '',
+    url = 'https://github.com/aws/aws-iot-device-sdk-python.git',
+    download_url = 'https://s3.amazonaws.com/aws-iot-device-sdk-python/aws-iot-device-sdk-python-latest.zip',
+    keywords = ['aws', 'iot', 'mqtt'],
+    classifiers = [
+        "Development Status :: 5 - Production/Stable",
+        "Intended Audience :: Developers",
+        "Natural Language :: English",
+        "License :: OSI Approved :: Apache Software License",
+        "Programming Language :: Python",
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.3",
+        "Programming Language :: Python :: 3.4",
+        "Programming Language :: Python :: 3.5"
+    ]
+)