added ansible installer for multicloud (#1359)

* added ansible installer for multicloud

* updated os in ci.yml to ubuntu 20.04

* deleted files that are not required for multicloud installation

* Removed dashboard, updated install_dependencies.sh

* updated the copyright header from OPENSDS to SODA

* updated hardcoded host_ip in common.yml to env variable

* Created README.md

* upgraded docker-compose to docker compose

* added go version check in install_dependencies.sh, minor changes in README

* removed etcd related file

.github/workflows/ci.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       matrix:
         go-version: [1.17.9]
-        os: [ubuntu-18.04]
+        os: [ubuntu-20.04]
 
     runs-on: ${{ matrix.os }}
     steps:
@@ -53,7 +53,7 @@ jobs:
       run: |
         go mod vendor
         make docker
-        docker-compose up -d
+        docker compose up -d
 
     - name: Run CI scripts for Testing
       run: ./install/CI/coverage && ./install/CI/test
@@ -63,5 +63,5 @@ jobs:
 
     - name: Clean up the build
       run: |
-        docker-compose down
+        docker compose down
         make clean

docker-compose.yml

@@ -36,7 +36,7 @@ services:
       - MICRO_SERVER_ADDRESS=:8089
       - MICRO_REGISTRY=mdns
       - OS_AUTH_AUTHSTRATEGY=keystone
-      - OS_AUTH_URL=http://10.10.3.154/identity
+      - OS_AUTH_URL=http://$HOST_IP/identity
       - OS_USER_DOMIN_ID=Default
       - OS_USERNAME=opensds
       - OS_PASSWORD=opensds@123
@@ -58,7 +58,7 @@ services:
       - MICRO_SERVER_ADDRESS=:8090
       - MICRO_REGISTRY=mdns
       - OS_AUTH_AUTHSTRATEGY=keystone
-      - OS_AUTH_URL=http://10.10.3.154/identity
+      - OS_AUTH_URL=http://$HOST_IP/identity
       - OS_USER_DOMIN_ID=Default
       - OS_USERNAME=opensds
       - OS_PASSWORD=opensds@123
@@ -92,7 +92,7 @@ services:
     image: sodafoundation/multi-cloud-aksk
     environment:
       - MICRO_REGISTRY=mdns
-      - IAM_HOST={host_ip}
+      - IAM_HOST=$HOST_IP
     volumes:
       - /etc/localtime:/etc/localtime
       - /var/log/multi-cloud:/var/log/multi-cloud
@@ -117,7 +117,7 @@ services:
       - MICRO_REGISTRY=mdns
       - DB_HOST=datastore:27017
       - OS_AUTH_AUTHSTRATEGY=keystone
-      - OS_AUTH_URL=http://10.10.3.154/identity
+      - OS_AUTH_URL=http://$HOST_IP/identity
       - OS_USER_DOMIN_ID=Default
       - OS_USERNAME=opensds
       - OS_PASSWORD=opensds@123

go.mod

@@ -25,7 +25,7 @@ require (
 	github.com/hashicorp/vault/api v1.0.5-0.20190814205542-3b036e58e950
 	github.com/huaweicloud/golangsdk v0.0.0-20201013113019-d00b298c3cdd
 	github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.0.16-beta
-	github.com/journeymidnight/radoshttpd v0.0.0-20190617133011-609666b51136
+	github.com/journeymidnight/radoshttpd v0.0.7
 	github.com/journeymidnight/yig v2.0.0+incompatible
 	github.com/micro/go-log v0.1.0
 	github.com/micro/go-micro/v2 v2.9.1

install/CI/coverage

@@ -18,7 +18,6 @@ set -o errexit
 # Keep track of the script directory
 TOP_DIR=$(cd $(dirname "$0") && pwd)
 # OpenSDS Root directory
-OPENSDS_DIR=$(cd $TOP_DIR/../.. && pwd)
 
 split_line(){
     echo "================================================================================================"

installer/README.md

@@ -0,0 +1,31 @@
+# Multi-cloud Installation Guide
+
+## Ansible Installer
+* Supported OS: **Ubuntu 18.04, 20.04**
+* Prerequisite: **Python 3.6+** and **Go 1.17+**  should be installed
+
+**Note**: Ensure root user access while performing the installation.
+
+### Install Steps
+```bash
+apt-get update && apt-get install -y git curl wget libltdl7 libseccomp2 libffi-dev
+git clone https://github.com/sodafoundation/multi-cloud.git
+cd multi-cloud/installer
+chmod +x install_dependencies.sh && . install_dependencies.sh
+```
+
+### Set Host IP
+```bash
+cd ansible
+export HOST_IP=<your_host_ip>
+```
+
+### Run the Ansible Playbook to install Multi-cloud
+```bash
+ansible-playbook site.yml -i local.hosts
+```
+
+### Uninstall and Purge
+```bash
+ansible-playbook clean.yml -i local.hosts
+```

installer/ansible/clean.yml

@@ -0,0 +1,45 @@
+# Copyright 2022 The SODA Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+# Defines some clean processes when banishing the cluster.
+
+- name: destroy an opensds cluster
+  hosts: 
+    - controllers
+  remote_user: root
+  vars_files:
+    - group_vars/common.yml
+    - group_vars/auth.yml
+    - group_vars/gelato.yml
+  gather_facts: false
+  become: True
+  tasks:
+    - import_role:
+        name: cleaner
+
+# Uninstall the Multicloud services on Kubernetes Cluster.        
+- name: uninstall multi-cloud HA services on Kubernetes Cluster.
+  hosts: k8smaster
+  remote_user:  "{{ k8s_user }}"
+  vars_files:
+    - group_vars/common.yml
+    - group_vars/gelato-ha.yml
+  gather_facts: false
+  become: True
+  tasks:
+    - import_role:
+        name: gelato-ha-uninstaller
+      when: gelato_ha == true
+  tags: gelato_ha

installer/ansible/group_vars/auth.yml

@@ -0,0 +1,35 @@
+# Copyright 2022 The SODA Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+# Dummy variable to avoid error because ansible does not recognize the
+# file as a good configuration file when no variable in it.
+dummy:
+
+
+###########
+# GENERAL #
+###########
+
+# OpenSDS authentication strategy, support 'noauth' and 'keystone'.
+opensds_auth_strategy: keystone
+
+# Install keystone with docker
+install_keystone_with_docker: true
+
+# The URL should be replaced with the keystone actual URL
+keystone_os_auth_url: "http://{{ host_ip }}/identity"
+
+# delete all keystone packages
+keystone_uninstall_purge: false

installer/ansible/group_vars/common.yml

@@ -0,0 +1,51 @@
+# Copyright 2022 The SODA Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+# Dummy variable to avoid error because ansible does not recognize the
+# file as a good configuration file when no variable in it.
+dummy:
+
+
+###########
+# GENERAL #
+###########
+
+# This field indicates local machine host ip
+host_ip: $HOST_IP
+
+#Deploy multicloud in HA mode - default is false 
+gelato_ha: false
+
+# This field indicates which way user prefers to install, currently support
+# 'repository', 'release' and 'container'
+install_from: release
+
+# These fields below will specify the tag based on install_from type
+repo_branch: master
+release_version: v1.7.0
+
+# This field indicates which os family the system will be running, currently
+# support 'Debian' and 'RedHat'
+ansible_os_family: Debian
+
+# delete all source packages
+source_purge: true
+
+# delete database
+database_purge: true
+
+###############
+# Respective projects release versions
+gelato_release_version: v1.7.0

installer/ansible/group_vars/gelato-ha.yml

@@ -0,0 +1,66 @@
+# Copyright 2022 The SODA Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+# Dummy variable to avoid error because ansible does not recognize the
+# file as a good configuration file when no variable in it.
+dummy:
+
+
+###########
+# GENERAL #
+###########
+
+# User permitted to access the Kubernetes Cluster to deploy the multicloud services.
+# 'ubuntu' is a sample user, it should be replaced with the actual user permitted 
+# to access the Kubernetes Cluster, when deployed in production.
+k8s_user: ubuntu  
+
+# These IPs are dummy IPs, which get updated appropriately during deployment.
+# These are populated by installer in a Multi-Cloud HA installation.
+
+gelato_ha_api_ip: 0.0.0.0
+gelato_ha_api_port: 30099
+
+gelato_ha_s3_ip: 0.0.0.0
+gelato_ha_s3_port: 30090
+
+# These fields are NOT suggested to be modifie
+gelato_work_dir: /opt/opensds-gelato-linux-amd64
+
+# The installation type of gelato-ha is container, both REPOSITORY and RELEASE will
+# use gelato image to deploy.
+gelato_image_tag: "{{ gelato_release_version }}"
+
+##############
+# REPOSITORY #
+##############
+
+# If user specifies intalling from repository, then he can choose the specific
+# repository branch
+gelato_repo_branch: "{{ repo_branch }}"
+
+# These fields are NOT suggested to be modified
+gelato_remote_url: https://github.com/sodafoundation/multi-cloud.git
+
+###########
+# RELEASE #
+###########
+
+# If user specifies intalling from release,then he can choose the specific version
+gelato_release: "{{ gelato_release_version }}"	
+
+# These fields are NOT suggested to be modified
+gelato_download_url: https://github.com/sodafoundation/multi-cloud/archive/{{ gelato_release }}.tar.gz
+gelato_tarball_dir: /tmp/gelato-{{ gelato_release }}

installer/ansible/group_vars/gelato.yml

@@ -0,0 +1,52 @@
+# Copyright 2022 The SODA Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+# Dummy variable to avoid error because ansible does not recognize the
+# file as a good configuration file when no variable in it.
+dummy:
+
+
+###########
+# GENERAL #
+###########
+
+# These fields are NOT suggested to be modifie
+gelato_work_dir: /opt/opensds-gelato-linux-amd64
+
+# The installation type of gelato is container, both REPOSITORY and RELEASE will
+# use gelato image to deploy.
+gelato_image_tag: "{{ gelato_release_version }}"
+
+##############
+# REPOSITORY #
+##############
+
+# If user specifies intalling from repository, then he can choose the specific
+# repository branch
+gelato_repo_branch: "{{ repo_branch }}"
+
+# These fields are NOT suggested to be modified
+gelato_remote_url: https://github.com/sodafoundation/multi-cloud.git
+
+###########
+# RELEASE #
+###########
+
+# If user specifies intalling from release,then he can choose the specific version
+gelato_release: "{{ gelato_release_version }}"	
+
+# These fields are NOT suggested to be modified
+gelato_download_url: https://github.com/sodafoundation/multi-cloud/archive/{{ gelato_release }}.tar.gz
+gelato_tarball_dir: /tmp/gelato-{{ gelato_release }}

installer/ansible/local.hosts

@@ -0,0 +1,20 @@
+# Copyright 2022 The SODA Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[controllers]
+localhost ansible_connection=local
+
+# This group has the Kubernetes Master node details. The MultiCloud Pods are deployed on this Kubernetes cluster.
+[k8smaster]
+# master ansible_port=22 ansible_user=root ansible_host=192.168.20.xx