Add support to Ubuntu 20.04

.github/workflows/ci.yaml

@@ -69,10 +69,16 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name: 'Python 3.6 + MongoDB 4.0'
+          - name: 'Ubuntu 18.04 + Python 3.6 + MongoDB 4.0'
+            vm_image: "ubuntu/bionic64"
             python_version: "3.6"
             mongodb_version: "4.0"
-          - name: 'Python 3.8 + MongoDB 4.4'
+          - name: 'Ubuntu 18.04 + Python 3.8 + MongoDB 4.4'
+            vm_image: "ubuntu/bionic64"
+            python_version: "3.8"
+            mongodb_version: "4.4"
+          - name: 'Ubuntu 20.04 + Python 3.8 + MongoDB 4.4'
+            vm_image: "ubuntu/focal64"
             python_version: "3.8"
             mongodb_version: "4.4"
 
@@ -83,7 +89,7 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.vagrant.d/boxes
-          key: ${{ runner.os }}-vagrant-ubuntu-bionic64
+          key: ${{ runner.os }}-vagrant-ubuntu-${{ matrix.vm_image }}
           restore-keys: |
             ${{ runner.os }}-vagrant-
 
@@ -95,9 +101,10 @@ jobs:
         #env:
         #  ANSIBLE_DEBUG: "true"
         run: |
-          sed -i".bak" -E -e "s#\.\./st2#/tmp/st2#g" Vagrantfile
+          sed -i".bak" -E -e "s#ubuntu/bionic64#${{ matrix.vm_image }}#g" Vagrantfile
           sed -i".bak" -E -e "s#3.6#${{ matrix.python_version }}#g" Vagrantfile
           sed -i".bak" -E -e "s#4.0#${{ matrix.mongodb_version }}#g" Vagrantfile
+          sed -i".bak" -E -e "s#\.\./st2#/tmp/st2#g" Vagrantfile
           cat Vagrantfile
 
           vagrant up

README.md

@@ -31,16 +31,22 @@ automatically reflected inside the virtual machine immediately after you make th
 
 Note: Make sure to run step 5 (`make requirements`) inside `~/st2` to have the `PYTHONPATH` point to the modules inside `~/st2`.
 
-### Specifying a Python and MongoDB version
+### Specifying Distro, Python and MongoDB version
 
-By default, the image is provisioned using Python 3.6 and Mongo DB 4.0. If you want to us Python
-3.8 and MongoDB 4.4 you can do that editing corresponding variables in ``Vagrantfile`` before
-running ``vagrant up`` / ``vagrant provision``.
+By default, the image is provisioned using Python 3.6 and Mongo DB 4.0 on Ubuntu 18.04.
+
+If you want to use a different distro, Python or MOngoDB version, you can do that by editing
+corresponding variables in ``Vagrantfile`` before running ``vagrant up`` / ``vagrant provision``.
 
 For example:
 
 ```ruby
 ...
+# For Ubuntu 18.04 (default)
+VM_BOX = "ubuntu/bionic64"
+# For Ubuntu 20.04
+VM_BOX = "ubuntu/focal64"
+
 PYTHON_VERSION = "3.8"
 MONGODB_VERSION = "4.4"
 ...

Vagrantfile

@@ -10,10 +10,18 @@ PYTHON_VERSION = "3.6"
 # Valid values are 4.0 and 4.4
 MONGODB_VERSION = "4.0"
 
-ERLANG_VERSION = "1:23.3.1-1rmq1ppa1~ubuntu18.04.1"
+# ubuntu/bionic-64 -> bionic, ubuntu/focal64 -> focal
+DISTRO_TYPE = VM_BOX.gsub("ubuntu/", "").gsub("64", "")
+
+if DISTRO_TYPE == "bionic"
+  ERLANG_VERSION = "1:23.3.2-1rmq1ppa1~ubuntu18.04.1"
+elsif DISTRO_TYPE == "focal"
+  ERLANG_VERSION = "1:23.3.2-1rmq1ppa1~ubuntu20.04.1"
+end
+
 RABBITMQ_VERSION = "3.8.14-1"
 
-VM_NAME = "st2-dev-py-" + PYTHON_VERSION.sub(".", "") + "-mongo-" + MONGODB_VERSION.sub(".", "")
+VM_NAME = "st2-dev-" + DISTRO_TYPE + "-py-" + PYTHON_VERSION.sub(".", "") + "-mongo-" + MONGODB_VERSION.sub(".", "")
 
 ANSIBLE_DEBUG = ENV.has_key?('ANSIBLE_DEBUG') ? "vvv" : ""
 
@@ -37,6 +45,7 @@ Vagrant.configure("2") do |config|
     ansible.playbook = "/vagrant/ansible/main.yml"
     ansible.verbose = ANSIBLE_DEBUG
     ansible.extra_vars = {
+      distro_type: DISTRO_TYPE,
       python_version: PYTHON_VERSION,
       mongodb_version: MONGODB_VERSION,
       rabbitmq_version: RABBITMQ_VERSION,

ansible/roles/st2_dev/files/generate_stanley_key.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+if [ -f "/home/stanley/.ssh/authorized_keys" ]; then
+  echo "Key already exists, skipping thos step"
+  exit 0
+fi
+
+echo "Generating stanley SSH key"
+ssh-keygen -b 2048 -t rsa -f "/home/vagrant/.ssh/stanley_rsa" -q -N "" -m pem
+mkdir -p /home/stanley/.ssh
+chmod 660 /home/stanley/.ssh
+chmod +x /home/stanley/.ssh
+cp /home/vagrant/.ssh/stanley_rsa.pub /home/stanley/.ssh/authorized_keys
+chmod 660 /home/stanley/.ssh/authorized_keys
+chown -R stanley: /home/stanley/.ssh/
+chown -R vagrant: /home/vagrant/.ssh/

ansible/roles/st2_dev/files/st2_bash_profile.sh

@@ -4,6 +4,7 @@ PRINT_INFO=${PRINT_INFO:-"1"}
 ST2_SHARE_DIR="/home/vagrant/st2"
 
 PYTHON_BINARY_FILE_PATH="/home/vagrant/.st2_python_binary"
+DISTRO_NAME=$(lsb_release -c | awk '{print $2}')
 
 if [ ! -f "${PYTHON_BINARY_FILE_PATH}" ]; then
     >&2 echo "${PYTHON_BINARY_FILE_PATH} file doesn't exist. Make sure you ran vagrant provision"
@@ -17,9 +18,9 @@ export PYTHON_VERSION=${PYTHON_BINARY_NAME}
 
 PYTHON_VERSION_SHORT_STRING=$(${PYTHON_BINARY} --version | sed "s/Python //g" | sed "s/\.//g" | tr -d "\n")
 
-export VIRTUALENV_DIR=virtualenv-py${PYTHON_VERSION_SHORT_STRING}
-export VIRTUALENV_ST2CLIENT_DIR=virtualenv-st2client-py${PYTHON_VERSION_SHORT_STRING}
-export VIRTUALENV_COMPONENTS_DIR=virtualenv-components-py${PYTHON_VERSION_SHORT_STRING}
+export VIRTUALENV_DIR=virtualenv-${DISTRO_NAME}-py${PYTHON_VERSION_SHORT_STRING}
+export VIRTUALENV_ST2CLIENT_DIR=virtualenv-st2client-${DISTRO_NAME}-py${PYTHON_VERSION_SHORT_STRING}
+export VIRTUALENV_COMPONENTS_DIR=virtualenv-components-${DISTRO_NAME}-py${PYTHON_VERSION_SHORT_STRING}
 
 if [ "${PRINT_INFO}" = "1" ]; then
     echo ""

ansible/roles/st2_dev/tasks/mongo.yml

@@ -10,7 +10,7 @@
     mode: 0644
     dest: /etc/apt/sources.list.d/mongodb.list
     content: >
-        deb https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/{{ mongodb_version }} multiverse
+        deb https://repo.mongodb.org/apt/ubuntu {{ distro_type }}/mongodb-org/{{ mongodb_version }} multiverse
 - name: Install mongo
   become: yes
   apt:

ansible/roles/st2_dev/tasks/python.yml

@@ -17,12 +17,20 @@
       - libsasl2-dev
       - libldap2-dev
       - libssl-dev
-- name: Install pip
+- name: Download pip installer
   become: true
-  ansible.builtin.shell: curl https://bootstrap.pypa.io/get-pip.py | sudo python{{ python_version }}
-- name: Install virtualenv
+  get_url:
+    url: https://bootstrap.pypa.io/get-pip.py
+    dest: /tmp/
+    mode: 0755
+# NOTE: In some version of ansible ansible.builtin.{shell,command} is broken
+# https://github.com/ansible/ansible/issues/71817
+- name: Install pip  # noqa: no-changed-when
   become: true
-  ansible.builtin.shell: python{{ python_version }} -m pip install virtualenv
+  command: python{{ python_version }} /tmp/get-pip.py
+- name: Install virtualenv  # noqa: no-changed-when
+  become: true
+  command: python{{ python_version }} -m pip install virtualenv
 # NOTE: We can't update default python3 binary on the system using update
 # alternatives since it breaks apt, and a bunch of other system tools which
 # depends on a specific Python version

ansible/roles/st2_dev/tasks/rabbitmq.yml

@@ -15,14 +15,14 @@
     mode: 0644
     dest: /etc/apt/sources.list.d/rabbitmq-erlang.list
     content: >
-        deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main
+        deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu {{ distro_type }} main
 - name: Add RabbitMQ server apt repo list
   become: yes
   copy:
     mode: 0644
     dest: /etc/apt/sources.list.d/rabbitmq-server.list
     content: >
-        deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ bionic main
+        deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ {{ distro_type }} main
 # This step is needed so correct latest versions of erlang packages get installed
 - name: Install Erlang
   become: yes

ansible/roles/st2_dev/tasks/system.yml

@@ -1,9 +1,10 @@
 ---
-- name: Install st2 system dependencies
+- name: Install st2 system and build dependencies
   become: true
   apt:
     state: present
     name:
+      - build-essential
       - libffi-dev
       - libssl-dev
       - libxslt1-dev
@@ -24,18 +25,6 @@
     regexp: '^stanley'
     line: 'stanley ALL=(ALL) NOPASSWD: ALL'
     validate: 'visudo -cf %s'
-- name: Generate and add SSH key for stanley user for remote actions
+- name: Generate and add SSH key for stanley user for remote actions  # noqa: no-changed-when
   become: true
-  ansible.builtin.shell: |
-    if [ -f "/home/stanley/.ssh/authorized_keys" ]; then
-      exit 0
-    fi
-
-    ssh-keygen -b 2048 -t rsa -f "/home/vagrant/.ssh/stanley_rsa" -q -N "" -m pem
-    mkdir -p /home/stanley/.ssh
-    chmod 660 /home/stanley/.ssh
-    chmod +x /home/stanley/.ssh
-    cp /home/vagrant/.ssh/stanley_rsa.pub /home/stanley/.ssh/authorized_keys
-    chmod 660 /home/stanley/.ssh/authorized_keys
-    chown -R stanley: /home/stanley/.ssh/
-    chown -R vagrant: /home/vagrant/.ssh/
+  command: bash /vagrant/ansible/roles/st2_dev/files/generate_stanley_key.sh

scripts/ci/verify-provision.sh

@@ -6,6 +6,8 @@ set -e
 PYTHON_VERSION=$1
 MONGODB_VERSION=$2
 
+lsb_release -a
+
 ls -la /home/vagrant/st2
 ls -la /home/vagrant
 ls -la /home/vagrant/.st2_bash_profile.sh