[#313] Add auth for EDI (#705)

legion-platform · Dec 29, 2018 · 84e28d6 · 84e28d6
1 parent 4c6a3c4
commit 84e28d6
Show file tree

Hide file tree

Showing 35 changed files with 646 additions and 294 deletions.
diff --git a/deploy/ansible/roles/legion_core_chart/templates/legion-core-values.yaml.j2 b/deploy/ansible/roles/legion_core_chart/templates/legion-core-values.yaml.j2
@@ -5,6 +5,14 @@ localDomain: "cluster.local"
 addLocalDomain: true
 s3BucketName: "{{ legion_data_s3_bucket }}"
 
+{% if dex.enabled == true %}
+auth:
+  enabled: true
+  annotations:
+    nginx.ingress.kubernetes.io/auth-signin: https://auth.{{ cluster_name }}/oauth2/start?rd=https://$host$request_uri$is_args$args
+    nginx.ingress.kubernetes.io/auth-url: http://oauth2-proxy.kube-system.svc.cluster.local:4180/oauth2/auth
+{% endif %}
+
 package:
   version: "{{ legion_version }}"
   repository: "{{ pypi_repo }}"
@@ -21,7 +29,7 @@ jenkins:
   enabled: true
   persistence:
     storageClass: {% if persistent_jenkins_volume %}"jenkins-volume"{% else %}""{% endif %}
-    
+
     size: {{ jenkins_volume_size | default('20') }}Gi
 
   dags_volume_pvc: "{{ airflow_dags_pvc }}"

diff --git a/deploy/ansible/roles/legion_enclave/templates/legion-values.yaml.j2 b/deploy/ansible/roles/legion_enclave/templates/legion-values.yaml.j2
@@ -5,6 +5,14 @@ namespace: default
 
 log_level: {{ log_level }}
 
+{% if dex.enabled == true %}
+auth:
+  enabled: true
+  annotations:
+    nginx.ingress.kubernetes.io/auth-signin: https://auth.{{ cluster_name }}/oauth2/start?rd=https://$host$request_uri$is_args$args
+    nginx.ingress.kubernetes.io/auth-url: http://oauth2-proxy.kube-system.svc.cluster.local:4180/oauth2/auth
+{% endif %}
+
 package:
   version: "{{ legion_version }}"
   repository: "{{ pypi_repo }}"
@@ -33,12 +41,6 @@ grafana:
   ingress:
     tls:
       enabled: {% if use_https == "yes" %}true{% else %}false{% endif %}
-{% if dex.enabled == true %}
-
-    annotations:
-      nginx.ingress.kubernetes.io/auth-signin: https://auth.{{ cluster_name }}/oauth2/start?rd=https://$host$request_uri$is_args$args
-      nginx.ingress.kubernetes.io/auth-url: http://oauth2-proxy.kube-system.svc.cluster.local:4180/oauth2/auth
-{% endif %}
 
 edge:
   enabled: true

diff --git a/deploy/helms/legion-core/templates/_helpers.tpl b/deploy/helms/legion-core/templates/_helpers.tpl
@@ -29,4 +29,15 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- define "fullname" -}}
 {{- $name := default .Chart.Name .Values.nameOverride -}}
 {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
+{{- end -}}
+
+{{/*
+Create dex Ingress auth annotations
+*/}}
+{{- define "dex-ingress-annotations" }}
+    {{- if .Values.auth.enabled -}}
+    {{- range $key, $value := .Values.auth.annotations }}
+    {{ $key }}: {{ $value | quote }}
+    {{- end }}
+    {{- end }}
+{{- end }}
diff --git a/deploy/helms/legion-core/templates/jenkins/jenkins-server.yaml b/deploy/helms/legion-core/templates/jenkins/jenkins-server.yaml
@@ -74,12 +74,6 @@ spec:
         - name: LEGION_BASE_IMAGE_REPOSITORY
           value: "{{ .Values.package.baseImage.repository }}"
         # EDI credentials
-        - name: EDI_USER
-          value: ""
-        - name: EDI_PASSOWRD
-          value: ""
-        - name: EDI_TOKEN
-          value: ""
         # External resource (saving and loading model files)
         - name: EXTERNAL_RESOURCE_PROTOCOL
           value: "http"

diff --git a/deploy/helms/legion-core/templates/metrics/grafana.yaml b/deploy/helms/legion-core/templates/metrics/grafana.yaml
@@ -82,6 +82,7 @@ metadata:
   {{- range $key, $value := .Values.grafana.ingress.annotations }}
     {{ $key }}: {{ $value | quote }}
   {{- end }}
+  {{- template "dex-ingress-annotations" . }}
   labels:
     heritage: {{ .Release.Service | quote }}
     release: {{ .Release.Name | quote }}

diff --git a/deploy/helms/legion-core/templates/nexus/nexus-ingress.yaml b/deploy/helms/legion-core/templates/nexus/nexus-ingress.yaml
@@ -10,9 +10,10 @@ metadata:
     release: {{ .Release.Name | quote }}
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
   annotations:
-    {{- range $key, $value := .Values.nexus.ingress.annotations }}
-      {{ $key }}: {{ $value | quote }}
-    {{- end }}
+  {{- range $key, $value := .Values.nexus.ingress.annotations }}
+    {{ $key }}: {{ $value | quote }}
+  {{- end }}
+  {{- template "dex-ingress-annotations" . }}
 spec:
   rules:
     - host: "{{ .Values.nexus.ingress.domain.partial }}.{{ .Values.rootDomain }}"

diff --git a/deploy/helms/legion-core/values.yaml b/deploy/helms/legion-core/values.yaml
@@ -4,6 +4,10 @@ enclaveDeploymentPrefix: "legion-"
 domainDelimiter: "."
 s3BucketName: ""
 
+auth:
+  enabled: false
+  annotations: {}
+
 secrets:
   nexus:
     admin: "jonny"
@@ -16,7 +20,7 @@ jenkins:
     pullPolicy: "Always"
 
   rbac: true
-  
+
   ingress:
     enabled: true
     annotations: {}

diff --git a/deploy/helms/legion/templates/_helpers.tpl b/deploy/helms/legion/templates/_helpers.tpl
@@ -0,0 +1,12 @@
+{{/* vim: set filetype=mustache: */}}
+
+{{/*
+Create dex Ingress auth annotations
+*/}}
+{{- define "dex-ingress-annotations" }}
+    {{- if .Values.auth.enabled -}}
+    {{- range $key, $value := .Values.auth.annotations }}
+    {{ $key }}: {{ $value | quote }}
+    {{- end }}
+    {{- end }}
+{{- end }}
diff --git a/deploy/helms/legion/templates/edi/edi-server.yaml b/deploy/helms/legion/templates/edi/edi-server.yaml
@@ -115,6 +115,7 @@ metadata:
   {{- range $key, $value := .Values.edi.ingress.annotations }}
     {{ $key }}: {{ $value | quote }}
   {{- end }}
+  {{- template "dex-ingress-annotations" . }}
   labels:
     heritage: {{ .Release.Service | quote }}
     release: {{ .Release.Name | quote }}

diff --git a/deploy/helms/legion/templates/metrics/grafana.yaml b/deploy/helms/legion/templates/metrics/grafana.yaml
@@ -84,6 +84,7 @@ metadata:
   {{- range $key, $value := .Values.grafana.ingress.annotations }}
     {{ $key }}: {{ $value | quote }}
   {{- end }}
+  {{- template "dex-ingress-annotations" . }}
   labels:
     heritage: {{ .Release.Service | quote }}
     release: {{ .Release.Name | quote }}

diff --git a/deploy/helms/legion/values.yaml b/deploy/helms/legion/values.yaml
@@ -8,15 +8,19 @@ clusterName: ""
 
 log_level: info
 
+auth:
+  enabled: false
+  annotations: {}
+
 feedback:
   replicas: 1
   ingress:
-      annotations: {}
-      domain:
-        partial: feedback
-      tls:
-        enabled: true
-        autoSecretNameDeduction: true
+    annotations: {}
+    domain:
+      partial: feedback
+    tls:
+      enabled: true
+      autoSecretNameDeduction: true
 
   image:
     repository: "legion/k8s-fluentd"

diff --git a/k8s/jenkins/legion-jenkins-plugin/src/main/resources/legion.groovy b/k8s/jenkins/legion-jenkins-plugin/src/main/resources/legion.groovy
@@ -77,7 +77,6 @@ def pod(Map podParams=null, Closure body) {
     envToPass = [
             "LEGION_PACKAGE_VERSION", "LEGION_PACKAGE_REPOSITORY", "LEGION_BASE_IMAGE_TAG",
             "LEGION_BASE_IMAGE_REPOSITORY",
-            "EDI_USER", "EDI_PASSOWRD", "EDI_TOKEN",
             "EXTERNAL_RESOURCE_PROTOCOL", "EXTERNAL_RESOURCE_HOST", "EXTERNAL_RESOURCE_USER", "EXTERNAL_RESOURCE_PASSWORD",
             "MODEL_IMAGES_REGISTRY", "MODEL_IMAGES_REGISTRY_HOST", "DOCKER_REGISTRY_USER", "DOCKER_REGISTRY_PASSWORD",
             "GRAPHITE_HOST", "STATSD_HOST", "STATSD_PORT",

diff --git a/legion/bin/legionctl b/legion/bin/legionctl
@@ -20,8 +20,9 @@ import argparse
 import logging
 
 import legion.docker_bootup
+from legion.edi.security import login, add_edi_arguments
 from legion.serving.pyserve import serve_model
-from legion.external.edi import add_edi_arguments, add_arguments_for_wait_operation
+from legion.external.edi import add_arguments_for_wait_operation
 from legion.edi.deploy import \
     build_model, \
     deploy_kubernetes, undeploy_kubernetes, scale_kubernetes, inspect_kubernetes, \
@@ -54,6 +55,11 @@ if __name__ == '__main__':
                         action='store_true')
     subparsers = parser.add_subparsers()
 
+    # --------- LOGIN SECTION -----------
+    login_parser = subparsers.add_parser('login', description='Save edi credentials to the config')
+    add_edi_arguments(login_parser, required=True)
+    login_parser.set_defaults(func=login)
+
     # --------- LOCAL DOCKER SECTION -----------
     build_parser = subparsers.add_parser('build', description='build model into new docker image (should be run '
                                                               'in the docker container)')

diff --git a/legion/docs/source/conf.py b/legion/docs/source/conf.py
@@ -19,6 +19,9 @@
 #
 import os
 import sys
+
+from legion.version import __version__ as legion_version
+
 sys.path.insert(0, os.path.abspath('../..'))
 
 from recommonmark.parser import CommonMarkParser
@@ -63,9 +66,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.0'
+version = legion_version[:legion_version.rfind('.')]
 # The full version, including alpha/beta/rc tags.
-release = '1.0'
+release = legion_version
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

diff --git a/legion/docs/source/index.rst b/legion/docs/source/index.rst
@@ -9,6 +9,7 @@ Welcome to legion's documentation!
    usage
    system_architecture
    model_rest_api
+   legionctl
 
 
 .. toctree::

diff --git a/legion/docs/source/legionctl.md b/legion/docs/source/legionctl.md
@@ -0,0 +1,35 @@
+# Legionctl
+
+`legionctl` is a command line tool that allows you to manage an edi service.
+
+## Basic
+
+To read legionctl help you should use the following command:
+
+```bash
+legionctl --help
+```
+
+for a specific command, for example, inspect:
+
+```bash
+legionctl inspect --help
+```
+
+## Security
+
+You should open edi url in a browser to get a token.
+
+Then you can use it in a legionctl command, for example:
+
+```bash
+legionctl inspect --edi <edi-url> --token <your-token>
+```
+
+Another option save the token in config file. Further you can use legionctl commands without `edi` and
+`token` parameters. For example:
+
+```bash
+legionctl login --edi <edi-url> --token <your-token>
+legionctl inspect
+```
diff --git a/legion/docs/source/testing.md b/legion/docs/source/testing.md
@@ -72,12 +72,12 @@ dex:
       - email: tests-user@legion.com
         password: ~
 ```
-* Export the following environment variables:
+* Go to the `tests\robot` directory. Export the following environment variables:
 ```bash
 export PROFILE="legion-test.epm.kharlamov.biz"
 export LEGION_VERSION="0.10.0-20181213084714.1329.6ba06cc"
 export PATH_TO_COOKIES="credentials.secret"
-export CLUSTER_NAME="legion-test.epm.kharlamov.biz"
+export CLUSTER_NAME="${PROFILE}"
 export CREDENTIAL_SECRETS=".secrets.yaml"
 export ROBOT_OPTIONS="-v PATH_TO_PROFILES_DIR:../../deploy/profiles"
 ```

diff --git a/legion/legion/config.py b/legion/legion/config.py
@@ -50,11 +50,11 @@
 GRAFANA_USER = 'GRAFANA_USER', 'admin'
 GRAFANA_PASSWORD = 'GRAFANA_PASSWORD', 'admin'
 
-EDI_URL = 'EDI_URL', 'http://localhost:5001/'
-EDI_USER = 'EDI_USER', ''
-EDI_PASSWORD = 'EDI_PASSWORD', ''
+EDI_URL = 'EDI_URL', ''
 EDI_TOKEN = 'EDI_TOKEN', ''
 
+LEGION_CONFIG = 'LEGION_CONFIG', ''
+
 LOCAL_DEFAULT_RESOURCE_PREFIX = 'LOCAL_DEFAULT_RESOURCE_PREFIX', ''
 EXTERNAL_RESOURCE_USE_BY_DEFAULT = 'EXTERNAL_RESOURCE_USE_BY_DEFAULT', 'true'
 EXTERNAL_RESOURCE_PROTOCOL = 'EXTERNAL_RESOURCE_PROTOCOL', 'https'