enable Vault (#466)

docs/job/vault.md

@@ -0,0 +1,37 @@
+# Vault(KV Engines only)
+InfraBox can fetch values as environment from vault service, so if your variable rotation regularly, you can configure it with vault in your projects. You just need to update the variable in Vault rather than update in your Infrabox project when the variable rotation. Login to the InfraBox Dashboard, select your project and go to the Settings tab. There you can create a vault with a name, a url, a namespace, a version, a token and a ca certificate.
+
+## Parameters explanation
+
+name: a DIY name (e.g. myvault)  
+
+url: the url of vault service (e.g. https://vault-service.com:1234)  
+
+namespace: the Vault's namespace, only enterprise edition enable namespace.  
+
+version：Vault provide version 1 or 2 for KV engine. just set it with 1 or 2.
+
+token: a token to access Vault.
+
+ca: provide ca certificate if using https. 
+
+
+
+## Using secrets as environment variable
+If you have created a [vault](#vault) you can make it available as a environment variable.
+
+```json
+{
+    "version": 1,
+    "jobs": [{
+        ...
+        "environment": {
+            "SOME_VALUE": {
+                "$vault": " the name of the vault",
+                "$vault_secret_path": "the secret path in vault's kv engine",
+                "$vault_secret_key": "the key of the vault secret"
+            },
+        }
+    }]
+}
+```

infrabox/generator/e2e.json

@@ -1,33 +1,6 @@
 {
     "version": 1,
     "jobs": [{
-        "type": "docker",
-        "name": "e2e-k8s-1-14",
-        "docker_file": "infrabox/test/e2e/Dockerfile",
-        "build_context": "../..",
-        "build_only": false,
-        "resources": {
-            "limits": { "cpu": 1, "memory": 2048 }
-        },
-        "cache": {
-            "image": true
-        },
-        "timeout": 2700,
-        "services": [{
-            "apiVersion": "gcp.service.infrabox.net/v1alpha1",
-            "kind": "GKECluster",
-            "metadata": {
-                "name": "e2e-cluster"
-            },
-            "spec": {
-                "machineType": "n1-standard-4",
-                "numNodes": 2,
-                "preemptible": false,
-                "zone": "us-east1-b",
-                "clusterVersion": "1.14"
-            }
-        }]
-    }, {
         "type": "docker",
         "name": "e2e-k8s-1-15",
         "docker_file": "infrabox/test/e2e/Dockerfile",

src/api/handlers/job_api.py

@@ -5,6 +5,7 @@
 import copy
 import urllib
 import random
+import tempfile
 from datetime import datetime
 from io import BytesIO
 
@@ -806,18 +807,55 @@ def post(self):
                     value = job['environment'][ename]
 
                     if isinstance(value, dict):
-                        env_var_ref_name = value['$secret']
-                        result = g.db.execute_many("""
-                            SELECT value FROM secret WHERE name = %s and project_id = %s
-                        """, [env_var_ref_name, project_id])
-
-                        if not result:
-                            abort(400, "Secret '%s' not found" % env_var_ref_name)
-
-                        if not job['env_var_refs']:
-                            job['env_var_refs'] = {}
-
-                        job['env_var_refs'][ename] = env_var_ref_name
+                        if '$secret' in value:
+                            env_var_ref_name = value['$secret']
+                            result = g.db.execute_many("""
+                                SELECT value FROM secret WHERE name = %s and project_id = %s
+                            """, [env_var_ref_name, project_id])
+
+                            if not result:
+                                abort(400, "Secret '%s' not found" % env_var_ref_name)
+
+                            if not job['env_var_refs']:
+                                job['env_var_refs'] = {}
+
+                            job['env_var_refs'][ename] = env_var_ref_name
+
+                        if '$vault' in value and '$vault_secret_path' in value and "$vault_secret_key" in value:
+                            name = value['$vault']
+                            secret_path = value['$vault_secret_path']
+                            secret_key = value['$vault_secret_key']
+                            result = g.db.execute_one("""
+                                SELECT url,version,token,ca,namespace FROM vault WHERE name = %s and project_id = %s
+                            """, [name, project_id])
+
+                            if not result:
+                                abort(400, "Cannot get Vault '%s' in project '%s' " % (name, project_id))
+
+                            url, version, token, ca, namespace = result[0], result[1], result[2], result[3], result[4]
+                            if not namespace:
+                                namespace = ''
+                            if version == 'v1':
+                                url += '/v1/' + namespace + '/' + secret_path
+                            elif version == 'v2':
+                                paths = secret_path.split('/')
+                                url += '/v1/' + namespace + '/' + paths[0] + '/data/' + '/'.join(paths[1:])
+                            if not ca:
+                                res = requests.get(url=url, headers={'X-Vault-Token': token}, verify=False)
+                            else:
+                                with tempfile.NamedTemporaryFile(delete=False) as f:
+                                    f.write(ca)
+                                    f.flush()  # ensure all data written
+                                    res = requests.get(url=url, headers={'X-Vault-Token': token}, verify=f.name)
+                            if res.status_code == 200:
+                                json_res = json.loads(res.content)
+                                if json_res['data'].get('data') and isinstance(json_res['data'].get('data'), dict):
+                                    value = json_res['data'].get('data').get(secret_key)
+                                else:
+                                    value = json_res['data'].get(secret_key)
+                                job['env_vars'][ename] = value
+                            else:
+                                abort(400, "Getting value from vault error: url is '%s', token is '%s' " % (url, result))
                     else:
                         job['env_vars'][ename] = value
 

src/api/handlers/projects/__init__.py

@@ -7,3 +7,4 @@
 import api.handlers.projects.tokens
 import api.handlers.projects.cronjobs
 import api.handlers.projects.sshkeys
+import api.handlers.projects.vault

src/api/handlers/projects/vault.py

@@ -0,0 +1,56 @@
+from flask import request, g, abort
+from flask_restx import Resource, fields
+
+from pyinfraboxutils.ibflask import OK
+from pyinfraboxutils.ibrestplus import api, response_model
+
+ns = api.namespace('Vault',
+                   path='/api/v1/projects/<project_id>/vault',
+                   description='Vault service related operations')
+
+project_vault_model = api.model('VaultService', {
+    'name': fields.String(required=True),
+    'url': fields.String(required=True),
+    'namespace': fields.String(required=False),
+    'version': fields.String(required=True),
+    'token': fields.String(required=True),
+    'ca': fields.String(required=False),
+    'id': fields.String(required=False)
+})
+
+@ns.route('/')
+@api.doc(responses={403: 'Not Authorized'})
+class Tokens(Resource):
+
+    @api.marshal_with(project_vault_model)
+    def get(self, project_id):
+        '''one
+        Returns project's vault service
+        '''
+        v = g.db.execute_many_dict('''
+            SELECT id, name, url, namespace, version, token, ca
+            FROM vault
+            WHERE project_id = %s
+        ''', [project_id])
+        return v
+
+    @api.expect(project_vault_model)
+    def post(self, project_id):
+        b = request.get_json()
+        g.db.execute('''
+                    INSERT INTO vault (project_id, name, url, namespace, version, token, ca) VALUES(%s, %s, %s, %s, %s, %s, %s)
+                ''', [project_id, b['name'], b['url'], b['namespace'], b['version'], b['token'], b['ca']])
+        g.db.commit()
+        return OK('Successfully added vault.')
+
+
+@ns.route('/<vault_id>')
+@api.doc(responses={403: 'Not Authorized'})
+class Secret(Resource):
+    @api.response(200, 'Success', response_model)
+    def delete(self, project_id, vault_id):
+        g.db.execute('''
+                    DELETE FROM vault WHERE project_id = %s and id = %s
+                ''', [project_id, vault_id])
+        g.db.commit()
+        return OK('Successfully deleted vault.')

src/dashboard-client/src/components/project/Settings.vue

@@ -12,6 +12,9 @@
         <md-layout md-column md-gutter md-flex-xsmall="100" md-flex-small="50" md-flex-medium="50" md-flex-large="50" md-flex-xlarge="50">
             <ib-project-sshkeys :project="project"></ib-project-sshkeys>
         </md-layout>
+        <md-layout md-column md-gutter md-flex-xsmall="100" md-flex-small="50" md-flex-medium="50" md-flex-large="50" md-flex-xlarge="50">
+            <ib-Vault :project="project"></ib-Vault>
+         </md-layout>
         <md-layout v-if="project.userHasOwnerRights()" md-column md-gutter md-flex-xsmall="100" md-flex-small="50" md-flex-medium="50" md-flex-large="50" md-flex-xlarge="50">
             <ib-project-visibility :project="project"></ib-project-visibility>
         </md-layout>
@@ -32,6 +35,7 @@ import ProjectBadges from './Badges'
 import ProjectCronJobs from './Cron'
 import ProjectSSHKeys from './SSHKeys'
 import ProjectVisibility from './Visibility'
+import ProjectVault from './Vault'
 
 export default {
     props: ['project'],
@@ -42,6 +46,7 @@ export default {
         'ib-project-badges': ProjectBadges,
         'ib-project-cronjobs': ProjectCronJobs,
         'ib-project-visibility': ProjectVisibility,
+        'ib-Vault': ProjectVault,
         'ib-project-sshkeys': ProjectSSHKeys
     }
 }

src/dashboard-client/src/components/project/Vault.vue

@@ -0,0 +1,108 @@
+<template>
+    <div class="m-sm full-height">
+        <md-card md-theme="white" class="full-height clean-card">
+            <md-card-header>
+                <md-card-header-text class="setting-list">
+                    <md-icon>security</md-icon>
+                    <span>Vault</span>
+                </md-card-header-text>
+            </md-card-header>
+            <md-card-area>
+                    <md-list class="m-t-md m-b-md">
+                        <md-list-item>
+                            <md-input-container class="m-l-sm">
+                                <label>Name</label>
+                                <md-textarea v-model="name" required></md-textarea>
+                            </md-input-container>
+                            <md-input-container class="m-l-sm">
+                              <label>Url</label>
+                              <md-textarea v-model="url" required></md-textarea>
+                            </md-input-container>
+                            <md-input-container class="m-l-sm">
+                                <label>Namespace</label>
+                                <md-textarea v-model="namespace"></md-textarea>
+                            </md-input-container>
+                            <md-input-container>
+                                <label>Version</label>
+                                <md-select name="version" id="version" v-model="version" required>
+                                    <md-option value="v1" class="bg-white">1</md-option>
+                                    <md-option value="v2" class="bg-white">2</md-option>
+                                </md-select>
+                            </md-input-container>
+                            <md-input-container class="m-l-sm">
+                              <label>Token</label>
+                              <md-textarea v-model="token" required></md-textarea>
+                            </md-input-container>
+                            <md-input-container class="m-l-sm">
+                                <label>CA</label>
+                                <md-textarea v-model="ca"></md-textarea>
+                            </md-input-container>
+                            <md-button class="md-icon-button md-list-action" @click="addVault()">
+                                <md-icon md-theme="running" class="md-primary">add_circle</md-icon>
+                                <md-tooltip>Add new Vault record</md-tooltip>
+                            </md-button>
+                        </md-list-item>
+                        <md-list-item v-for="v in project.vault" :key="v.id">
+                            <div class="md-input-container m-r-xl md-theme-white">
+                                {{ v.name }}
+                            </div>
+                            <md-button type="submit" class="md-icon-button md-list-action" @click="deleteVault(v.id)">
+                                <md-icon class="md-primary">delete</md-icon>
+                                <md-tooltip>Delete secret permanently</md-tooltip>
+                            </md-button>
+                        </md-list-item>
+                    </md-list>
+            </md-card-area>
+        </md-card>
+    </div>
+</template>
+
+<script>
+import NewAPIService from '../../services/NewAPIService'
+import NotificationService from '../../services/NotificationService'
+import Notification from '../../models/Notification'
+
+export default {
+    props: ['project'],
+    data: () => ({
+        name: '',
+        url: '',
+        namespace: '',
+        version: '',
+        token: '',
+        ca: ''
+    }),
+    created () {
+        this.project._loadVault()
+    },
+    methods: {
+        deleteVault (id) {
+            NewAPIService.delete(`projects/${this.project.id}/vault/${id}`)
+                .then((response) => {
+                    NotificationService.$emit('NOTIFICATION', new Notification(response))
+                    this.project._reloadVault()
+                })
+                .catch((err) => {
+                    NotificationService.$emit('NOTIFICATION', new Notification(err))
+                })
+        },
+        addVault () {
+            const d = { name: this.name, url: this.url, namespace: this.namespace, version: this.version, token: this.token, ca: this.ca }
+            NewAPIService.post(`projects/${this.project.id}/vault`, d)
+                .then((response) => {
+                    NotificationService.$emit('NOTIFICATION', new Notification(response))
+                    this.name = ''
+                    this.url = ''
+                    this.namespace = ''
+                    this.version = ''
+                    this.token = ''
+                    this.ca = ''
+                    this.project._reloadVault()
+                })
+                .catch((err) => {
+                    NotificationService.$emit('NOTIFICATION', new Notification(err))
+                })
+        }
+    }
+}
+</script>

src/dashboard-client/src/models/Project.js

@@ -14,6 +14,7 @@ export default class Project {
         this.state = 'finished'
         this.type = type
         this.secrets = null
+        this.vault = null
         this.cronjobs = null
         this.collaborators = null
         this.roles = null
@@ -331,4 +332,20 @@ export default class Project {
             this.state = 'finished'
         }
     }
+
+    _loadVault () {
+        if (this.vault) {
+            return
+        }
+        this._reloadVault()
+    }
+    _reloadVault () {
+        return NewAPIService.get(`projects/${this.id}/vault`)
+            .then((vault) => {
+                store.commit('setVault', {project: this, vault: vault})
+            })
+            .catch((err) => {
+                NotificationService.$emit('NOTIFICATION', new Notification(err))
+            })
+    }
 }

src/dashboard-client/src/store.js

@@ -212,6 +212,12 @@ function setSecrets (state, data) {
     project.secrets = secrets
 }
 
+function setVault (state, data) {
+    const project = data.project
+    const vault = data.vault
+    project.vault = vault
+}
+
 function setCollaborators (state, data) {
     const project = data.project
     const collaborators = data.collaborators
@@ -358,7 +364,8 @@ const mutations = {
     setAdminProjects,
     setAdminClusters,
     updateAdminCluster,
-    setArchive
+    setArchive,
+    setVault
 }
 
 const getters = {}