From fefbf908d3667ee7146450ee629c3386710bfac1 Mon Sep 17 00:00:00 2001 From: Shankar M S Date: Thu, 5 Dec 2019 10:55:49 +0100 Subject: [PATCH 1/9] temp tf_state validation check removed --- cli/kaos_cli/commands/build.py | 8 ++++---- cli/kaos_cli/facades/backend_facade.py | 2 +- cli/kaos_cli/utils/validators.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cli/kaos_cli/commands/build.py b/cli/kaos_cli/commands/build.py index 57dfe1f..3e3a117 100644 --- a/cli/kaos_cli/commands/build.py +++ b/cli/kaos_cli/commands/build.py @@ -49,12 +49,12 @@ def deploy(backend: BackendFacade, cloud: str, env: str, force: bool, verbose: b env_state = EnvironmentState.initialize(cloud, env) - if env_state.if_tfstate_exists and not force: + if env_state.if_build_dir_exists and not force: click.echo('{} - {} backend is already built.'.format(click.style("Aborting", bold=True, fg='red'), click.style("kaos", bold=True))) sys.exit(1) - elif env_state.if_tfstate_exists and force: + elif env_state.if_build_dir_exists and force: click.echo('{} - Performing {} build of the backend'.format( click.style("Warning", bold=True, fg='yellow'), click.style("force", bold=True))) @@ -272,7 +272,7 @@ def destroy(backend: BackendFacade, cloud, env, verbose, yes): env_state.set_build_env() # Ensure that appropriate warnings are displayed - env_state.validate_if_tfstate_exits() + env_state.validate_if_build_dir_exits() if not yes: # confirm creation of backend @@ -295,7 +295,7 @@ def destroy(backend: BackendFacade, cloud, env, verbose, yes): env_state = backend.destroy(env_state, verbose=verbose) - if not env_state.if_tfstate_exists: + if not env_state.if_build_dir_exists: if env_state.env: click.echo( "{} - Successfully destroyed {} [{}] environment".format(click.style("Info", bold=True, fg='green'), diff --git a/cli/kaos_cli/facades/backend_facade.py b/cli/kaos_cli/facades/backend_facade.py index c6dd5a3..92971e1 100644 --- a/cli/kaos_cli/facades/backend_facade.py +++ b/cli/kaos_cli/facades/backend_facade.py @@ -144,7 +144,7 @@ def build(self, provider, env, local_backend=False, verbose=False): # Refresh environment states after terraform service operations env_state = EnvironmentState.initialize(provider, env) - if env_state.if_tfstate_exists: + if env_state.if_build_dir_exists: url, kubeconfig = self._parse_config(env_state.build_dir) current_context = provider if provider in [DOCKER, MINIKUBE] else f"{provider}_{env}" diff --git a/cli/kaos_cli/utils/validators.py b/cli/kaos_cli/utils/validators.py index 2b7065b..e3283e6 100644 --- a/cli/kaos_cli/utils/validators.py +++ b/cli/kaos_cli/utils/validators.py @@ -38,11 +38,11 @@ def initialize(cls, cloud, env) -> "EnvironmentState": env_state.tf_state_path = tf_state_path return env_state - def validate_if_tfstate_exits(self) -> "EnvironmentState": + def validate_if_build_dir_exits(self) -> "EnvironmentState": """ Ensure existence of kaos backend dir (tf_path) and throw appropriate warnings if it does not exist """ - if not self.if_tfstate_exists: + if not self.if_build_dir_exists: if self.env: click.echo("{} - {} [{}] backend in {} has not been deployed!".format( click.style("Warning", bold=True, fg='yellow'), From c7a271c1ee5f4abbdc780b88be8e55f62001a437 Mon Sep 17 00:00:00 2001 From: Shankar M S Date: Tue, 10 Dec 2019 16:50:58 +0100 Subject: [PATCH 2/9] [WIP k-xxx] tf state file validations --- Pipfile.lock | 67 +++++++-------- cli/kaos_cli/commands/build.py | 9 +- cli/kaos_cli/facades/backend_facade.py | 82 ++++++++++--------- cli/kaos_cli/utils/validators.py | 24 +++++- infrastructure/gcp/modules/kubernetes/main.tf | 6 +- 5 files changed, 101 insertions(+), 87 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 519481c..8223728 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -18,10 +18,10 @@ "default": { "certifi": { "hashes": [ - "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", - "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" + "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", + "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" ], - "version": "==2019.9.11" + "version": "==2019.11.28" }, "chardet": { "hashes": [ @@ -186,13 +186,6 @@ } }, "develop": { - "atomicwrites": { - "hashes": [ - "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", - "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" - ], - "version": "==1.3.0" - }, "attrs": { "hashes": [ "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", @@ -222,10 +215,10 @@ }, "certifi": { "hashes": [ - "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", - "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" + "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", + "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" ], - "version": "==2019.9.11" + "version": "==2019.11.28" }, "cgroupspy": { "hashes": [ @@ -365,11 +358,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26", - "sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af" + "sha256:3a8b2dfd0a2c6a3636e7c016a7e54ae04b997d30e69d5eacdca7a6c2221a1402", + "sha256:41e688146d000891f32b1669e8573c57e39e5060e7f5f647aa617cd9a9568278" ], "markers": "python_version < '3.8'", - "version": "==0.23" + "version": "==1.2.0" }, "itsdangerous": { "hashes": [ @@ -449,10 +442,10 @@ }, "more-itertools": { "hashes": [ - "sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832", - "sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4" + "sha256:53ff73f186307d9c8ef17a9600309154a6ae27f25579e80af4db8f047ba14bc2", + "sha256:a0ea684c39bc4315ba7aae406596ef191fd84f873d2d2751f84d64e81a7a2d45" ], - "version": "==7.2.0" + "version": "==8.0.0" }, "mypy-extensions": { "hashes": [ @@ -470,10 +463,10 @@ }, "pluggy": { "hashes": [ - "sha256:0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6", - "sha256:fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34" + "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", + "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" ], - "version": "==0.13.0" + "version": "==0.13.1" }, "protobuf": { "hashes": [ @@ -500,19 +493,19 @@ }, "psutil": { "hashes": [ - "sha256:021d361439586a0fd8e64f8392eb7da27135db980f249329f1a347b9de99c695", - "sha256:145e0f3ab9138165f9e156c307100905fd5d9b7227504b8a9d3417351052dc3d", - "sha256:348ad4179938c965a27d29cbda4a81a1b2c778ecd330a221aadc7bd33681afbd", - "sha256:3feea46fbd634a93437b718518d15b5dd49599dfb59a30c739e201cc79bb759d", - "sha256:474e10a92eeb4100c276d4cc67687adeb9d280bbca01031a3e41fb35dfc1d131", - "sha256:47aeb4280e80f27878caae4b572b29f0ec7967554b701ba33cd3720b17ba1b07", - "sha256:73a7e002781bc42fd014dfebb3fc0e45f8d92a4fb9da18baea6fb279fbc1d966", - "sha256:d051532ac944f1be0179e0506f6889833cf96e466262523e57a871de65a15147", - "sha256:dfb8c5c78579c226841908b539c2374da54da648ee5a837a731aa6a105a54c00", - "sha256:e3f5f9278867e95970854e92d0f5fe53af742a7fc4f2eba986943345bcaed05d", - "sha256:e9649bb8fc5cea1f7723af53e4212056a6f984ee31784c10632607f472dec5ee" + "sha256:094f899ac3ef72422b7e00411b4ed174e3c5a2e04c267db6643937ddba67a05b", + "sha256:10b7f75cc8bd676cfc6fa40cd7d5c25b3f45a0e06d43becd7c2d2871cbb5e806", + "sha256:1b1575240ca9a90b437e5a40db662acd87bbf181f6aa02f0204978737b913c6b", + "sha256:21231ef1c1a89728e29b98a885b8e0a8e00d09018f6da5cdc1f43f988471a995", + "sha256:28f771129bfee9fc6b63d83a15d857663bbdcae3828e1cb926e91320a9b5b5cd", + "sha256:70387772f84fa5c3bb6a106915a2445e20ac8f9821c5914d7cbde148f4d7ff73", + "sha256:b560f5cd86cf8df7bcd258a851ca1ad98f0d5b8b98748e877a0aec4e9032b465", + "sha256:b74b43fecce384a57094a83d2778cdfc2e2d9a6afaadd1ebecb2e75e0d34e10d", + "sha256:e85f727ffb21539849e6012f47b12f6dd4c44965e56591d8dec6e8bc9ab96f4a", + "sha256:fd2e09bb593ad9bdd7429e779699d2d47c1268cbde4dda95fcd1bd17544a0217", + "sha256:ffad8eb2ac614518bbe3c0b8eb9dffdb3a8d2e3a7d5da51c5b974fb723a5c5aa" ], - "version": "==5.6.5" + "version": "==5.6.7" }, "py": { "hashes": [ @@ -536,10 +529,10 @@ }, "pytest": { "hashes": [ - "sha256:8e256fe71eb74e14a4d20a5987bb5e1488f0511ee800680aaedc62b9358714e8", - "sha256:ff0090819f669aaa0284d0f4aad1a6d9d67a6efdc6dd4eb4ac56b704f890a0d6" + "sha256:63344a2e3bce2e4d522fd62b4fdebb647c019f1f9e4ca075debbd13219db4418", + "sha256:f67403f33b2b1d25a6756184077394167fe5e2f9d8bdaab30707d19ccec35427" ], - "version": "==5.2.4" + "version": "==5.3.1" }, "pytest-mock": { "hashes": [ diff --git a/cli/kaos_cli/commands/build.py b/cli/kaos_cli/commands/build.py index 3e3a117..ba2f1a4 100644 --- a/cli/kaos_cli/commands/build.py +++ b/cli/kaos_cli/commands/build.py @@ -61,6 +61,8 @@ def deploy(backend: BackendFacade, cloud: str, env: str, force: bool, verbose: b # set env variable appropriately env_state.set_build_env() + cloud = env_state.cloud + env = env_state.env if not yes: # confirm creation of backend @@ -106,7 +108,6 @@ def deploy(backend: BackendFacade, cloud: str, env: str, force: bool, verbose: b sys.exit(1) try: - is_built_successfully, env_state = backend.build(env_state.cloud, env_state.env, local_backend=local_backend, @@ -129,7 +130,7 @@ def deploy(backend: BackendFacade, cloud: str, env: str, force: bool, verbose: b click.echo("{} - Successfully built {} [{}] environment".format( click.style("Info", bold=True, fg='green'), click.style('kaos', bold=True), - click.style(env_state.env, bold=True, fg='blue'))) + click.style(env, bold=True, fg='blue'))) else: click.echo("{} - Successfully built {} environment".format( click.style("Info", bold=True, fg='green'), @@ -139,8 +140,8 @@ def deploy(backend: BackendFacade, cloud: str, env: str, force: bool, verbose: b click.echo("{} - Deployment Unsuccessful while creating {} [{} {}] environment".format( click.style("Error", bold=True, fg='red'), click.style('kaos', bold=True), - click.style(env_state.cloud, bold=True, fg='red'), - click.style(env_state.env, bold=True, fg='red'))), + click.style(cloud, bold=True, fg='red'), + click.style(env, bold=True, fg='red'))), sys.exit(1) except Exception as e: diff --git a/cli/kaos_cli/facades/backend_facade.py b/cli/kaos_cli/facades/backend_facade.py index 92971e1..68d6a36 100644 --- a/cli/kaos_cli/facades/backend_facade.py +++ b/cli/kaos_cli/facades/backend_facade.py @@ -12,14 +12,10 @@ from kaos_cli.services.terraform_service import TerraformService from kaos_cli.utils.environment import check_environment from kaos_cli.utils.helpers import build_dir -from kaos_cli.utils.validators import EnvironmentState +from kaos_cli.utils.validators import EnvironmentState, is_cloud_provider from kaos_cli.exceptions.handle_exceptions import handle_specific_exception, handle_exception -def is_cloud_provider(cloud): - return cloud not in (DOCKER, MINIKUBE) - - class BackendFacade: """ This class should handle all backend related configuration and settings. @@ -128,58 +124,64 @@ def set_context_by_index(self, index): def build(self, provider, env, local_backend=False, verbose=False): env_state = EnvironmentState.initialize(provider, env) - if not env_state.if_build_dir_exists: + print("build_directory", env_state.build_dir) + if not os.path.exists(env_state.build_dir): build_dir(env_state.build_dir) extra_vars = self._get_vars(provider, env_state.build_dir) self.tf_service.cd_dir(env_state.build_dir) self.tf_service.set_verbose(verbose) - directory = self._tf_init(provider, env, local_backend, destroying=False) - self.tf_service.plan(directory, extra_vars) - self.tf_service.apply(directory, extra_vars) + self._tf_init(env_state, provider, env, local_backend, destroying=False) + self.tf_service.plan(env_state.build_dir, extra_vars) + self.tf_service.apply(env_state.build_dir, extra_vars) self.tf_service.execute() # check if the deployed successfully # Refresh environment states after terraform service operations env_state = EnvironmentState.initialize(provider, env) - if env_state.if_build_dir_exists: - url, kubeconfig = self._parse_config(env_state.build_dir) + # if env_state.if_build_dir_exists: + url, kubeconfig = self._parse_config(env_state.build_dir) - current_context = provider if provider in [DOCKER, MINIKUBE] else f"{provider}_{env}" + current_context = provider if provider in [DOCKER, MINIKUBE] else f"{provider}_{env}" - self.state_service.set(DEFAULT, user=USER) + self.state_service.set(DEFAULT, user=USER) - self._set_context_list(current_context) - self._set_active_context(current_context) - self.state_service.set(current_context) + self._set_context_list(current_context) + self._set_active_context(current_context) + self.state_service.set(current_context) - try: - self.state_service.set_section(current_context, BACKEND, url=url, token=uuid.uuid4()) - self.state_service.set_section(current_context, INFRASTRUCTURE, kubeconfig=kubeconfig) - except Exception as e: - handle_specific_exception(e) - handle_exception(e) - - self.state_service.write() - return True, env_state + try: + print("Inside exception") + self.state_service.set_section(current_context, BACKEND, url=url, token=uuid.uuid4()) + self.state_service.set_section(current_context, INFRASTRUCTURE, kubeconfig=kubeconfig) + print("Inside exception 2") + except Exception as e: + handle_specific_exception(e) + handle_exception(e) + + print("outside exception") + self.state_service.write() + return True, env_state - return False, env_state + # return False, env_state def destroy(self, env_state, verbose=False): extra_vars = self._get_vars(env_state.cloud, env_state.build_dir) self.tf_service.cd_dir(env_state.build_dir) self.tf_service.set_verbose(verbose) - directory = self._tf_init(env_state.cloud, env_state.env, local_backend=False, destroying=True) + + self._tf_init(env_state, env_state.cloud, env_state.env, local_backend=False, destroying=True) + current_context = env_state.cloud if env_state.cloud in [DOCKER, MINIKUBE] \ else env_state.cloud + '_' + env_state.env self._delete_resources(current_context) self._unset_context_list(current_context) self._remove_section(current_context) self._deactivate_context() - self.tf_service.destroy(directory, extra_vars) + self.tf_service.destroy(env_state.build_dir, extra_vars) self.tf_service.execute() self._remove_build_files(env_state.build_dir) self.state_service.write() @@ -202,25 +204,25 @@ def _delete_resources(self, context): if self.state_service.has_section(context, BACKEND): requests.delete(f"{self.url}/internal/resources") - def _tf_init(self, provider, env, local_backend, destroying=False): - directory = PROVIDER_DICT.get(provider) + def _tf_init(self, env_state, provider, env, local_backend, destroying=False): + # directory = PROVIDER_DICT.get(cloud) check_environment(provider) if is_cloud_provider(provider): - provider_directory = f"{directory}/{env}" - directory = f"{directory}/__working_{env}" - if not destroying or not os.path.isdir(directory): - copy_tree(provider_directory, directory) + # provider_directory = f"{directory}/{env}" + # directory = f"{directory}/__working_{env}" + if not destroying or not os.path.isdir(env_state.build_dir): + print("here 1") + copy_tree(env_state.provider_directory, env_state.build_dir) if local_backend: - shutil.copy(LOCAL_CONFIG_DICT.get(provider), directory) + shutil.copy(LOCAL_CONFIG_DICT.get(provider), env_state.build_dir) # simply always create the workspace - self.tf_service.init(directory) - self.tf_service.new_workspace(directory, env) - self.tf_service.select_workspace(directory, env) + self.tf_service.init(env_state.build_dir) + self.tf_service.new_workspace(env_state.build_dir, env) + self.tf_service.select_workspace(env_state.build_dir, env) else: - self.tf_service.init(directory) - return directory + self.tf_service.init(env_state.build_dir) def _set_context_list(self, current_context): try: diff --git a/cli/kaos_cli/utils/validators.py b/cli/kaos_cli/utils/validators.py index e3283e6..c77a115 100644 --- a/cli/kaos_cli/utils/validators.py +++ b/cli/kaos_cli/utils/validators.py @@ -8,7 +8,7 @@ import textdistance from kaos_cli.exceptions.exceptions import MissingArgumentError -from ..constants import KAOS_STATE_DIR, DOCKER, MINIKUBE, KAOS_TF_PATH, TF_STATE +from ..constants import KAOS_STATE_DIR, DOCKER, MINIKUBE, KAOS_TF_PATH, TF_STATE, PROVIDER_DICT class EnvironmentState: @@ -20,6 +20,7 @@ def __init__(self): self.env = None self.build_dir = None self.tf_state_path = None + self.provider_directory = None self.if_build_dir_exists = None self.if_tfstate_exists = None @@ -30,12 +31,25 @@ def initialize(cls, cloud, env) -> "EnvironmentState": env_state.env = env env_dir = f"{env_state.cloud}/{env_state.env}" if env_state.cloud not in [DOCKER, MINIKUBE] \ else f"{env_state.cloud}" - build_dir = os.path.join(KAOS_TF_PATH, env_dir) - tf_state_path = os.path.join(KAOS_TF_PATH, env_dir, f"{TF_STATE}") + + infra_root = PROVIDER_DICT.get(cloud) + + if is_cloud_provider(cloud): + build_dir = f"{infra_root}/__working_{env}" + print("build_dir", build_dir) + tf_state_path = f"{build_dir}/.terraform/terraform.tfstate" + provider_directory = f"{infra_root}/{env}" + env_state.provider_directory = provider_directory + + else: + build_dir = os.path.join(KAOS_TF_PATH, env_dir) + tf_state_path = os.path.join(KAOS_TF_PATH, env_dir, f"{TF_STATE}") + env_state.if_build_dir_exists = os.path.exists(build_dir) env_state.if_tfstate_exists = os.path.exists(tf_state_path) env_state.build_dir = build_dir env_state.tf_state_path = tf_state_path + return env_state def validate_if_build_dir_exits(self) -> "EnvironmentState": @@ -194,3 +208,7 @@ def validate_unused_port(port: int, host: str = '0.0.0.0') -> bool: except socket.error: return False + +def is_cloud_provider(cloud): + return cloud not in (DOCKER, MINIKUBE) + diff --git a/infrastructure/gcp/modules/kubernetes/main.tf b/infrastructure/gcp/modules/kubernetes/main.tf index e1df9c2..8bb1f80 100644 --- a/infrastructure/gcp/modules/kubernetes/main.tf +++ b/infrastructure/gcp/modules/kubernetes/main.tf @@ -16,9 +16,9 @@ resource "google_container_cluster" "new_container_cluster" { disabled = var.http_load_balancing } - kubernetes_dashboard { - disabled = var.kubernetes_dashboard - } +// kubernetes_dashboard { +// disabled = var.kubernetes_dashboard +// } // network_policy_config { // disabled = "${var.network_policy_config}" // } From f4207d64e056bc1e26c8997edacad85c82578792 Mon Sep 17 00:00:00 2001 From: Shankar M S Date: Wed, 11 Dec 2019 15:57:30 +0100 Subject: [PATCH 3/9] tf_state file fixed for local, fixed bug for multiple deployments of the same kind --- cli/kaos_cli/commands/build.py | 9 ++-- cli/kaos_cli/facades/backend_facade.py | 68 +++++++++++++------------- cli/kaos_cli/services/state_service.py | 2 + cli/kaos_cli/utils/validators.py | 20 +++++--- 4 files changed, 54 insertions(+), 45 deletions(-) diff --git a/cli/kaos_cli/commands/build.py b/cli/kaos_cli/commands/build.py index ba2f1a4..b6d2e26 100644 --- a/cli/kaos_cli/commands/build.py +++ b/cli/kaos_cli/commands/build.py @@ -49,15 +49,16 @@ def deploy(backend: BackendFacade, cloud: str, env: str, force: bool, verbose: b env_state = EnvironmentState.initialize(cloud, env) - if env_state.if_build_dir_exists and not force: + if env_state.if_tfstate_exists and not force: click.echo('{} - {} backend is already built.'.format(click.style("Aborting", bold=True, fg='red'), click.style("kaos", bold=True))) sys.exit(1) - elif env_state.if_build_dir_exists and force: + elif env_state.if_tfstate_exists and force: click.echo('{} - Performing {} build of the backend'.format( click.style("Warning", bold=True, fg='yellow'), click.style("force", bold=True))) + env_state.remove_terraform_files() # set env variable appropriately env_state.set_build_env() @@ -273,7 +274,7 @@ def destroy(backend: BackendFacade, cloud, env, verbose, yes): env_state.set_build_env() # Ensure that appropriate warnings are displayed - env_state.validate_if_build_dir_exits() + env_state.validate_if_tf_state_exits() if not yes: # confirm creation of backend @@ -296,7 +297,7 @@ def destroy(backend: BackendFacade, cloud, env, verbose, yes): env_state = backend.destroy(env_state, verbose=verbose) - if not env_state.if_build_dir_exists: + if not env_state.if_tfstate_exists: if env_state.env: click.echo( "{} - Successfully destroyed {} [{}] environment".format(click.style("Info", bold=True, fg='green'), diff --git a/cli/kaos_cli/facades/backend_facade.py b/cli/kaos_cli/facades/backend_facade.py index 68d6a36..c59f224 100644 --- a/cli/kaos_cli/facades/backend_facade.py +++ b/cli/kaos_cli/facades/backend_facade.py @@ -5,7 +5,7 @@ from distutils.dir_util import copy_tree import requests -from kaos_cli.constants import DOCKER, MINIKUBE, PROVIDER_DICT, AWS, BACKEND, INFRASTRUCTURE, GCP, LOCAL_CONFIG_DICT, \ +from kaos_cli.constants import DOCKER, MINIKUBE, AWS, BACKEND, INFRASTRUCTURE, GCP, LOCAL_CONFIG_DICT, \ CONTEXTS, ACTIVE, BACKEND_CACHE, DEFAULT, USER, REMOTE from kaos_cli.exceptions.exceptions import HostnameError from kaos_cli.services.state_service import StateService @@ -124,7 +124,6 @@ def set_context_by_index(self, index): def build(self, provider, env, local_backend=False, verbose=False): env_state = EnvironmentState.initialize(provider, env) - print("build_directory", env_state.build_dir) if not os.path.exists(env_state.build_dir): build_dir(env_state.build_dir) @@ -141,31 +140,29 @@ def build(self, provider, env, local_backend=False, verbose=False): # Refresh environment states after terraform service operations env_state = EnvironmentState.initialize(provider, env) - # if env_state.if_build_dir_exists: - url, kubeconfig = self._parse_config(env_state.build_dir) + if env_state.if_tfstate_exists: + url, kubeconfig = self._parse_config(env_state.build_dir) - current_context = provider if provider in [DOCKER, MINIKUBE] else f"{provider}_{env}" + current_context = provider if provider in [DOCKER, MINIKUBE] else f"{provider}_{env}" - self.state_service.set(DEFAULT, user=USER) + self.state_service.set(DEFAULT, user=USER) - self._set_context_list(current_context) - self._set_active_context(current_context) - self.state_service.set(current_context) + self._set_context_list(current_context) + self._set_active_context(current_context) + self.state_service.set(current_context) - try: - print("Inside exception") - self.state_service.set_section(current_context, BACKEND, url=url, token=uuid.uuid4()) - self.state_service.set_section(current_context, INFRASTRUCTURE, kubeconfig=kubeconfig) - print("Inside exception 2") - except Exception as e: - handle_specific_exception(e) - handle_exception(e) - - print("outside exception") - self.state_service.write() - return True, env_state + try: + self.state_service.set_section(current_context, BACKEND, url=url, token=uuid.uuid4()) + self.state_service.set_section(current_context, INFRASTRUCTURE, kubeconfig=kubeconfig) + except Exception as e: + handle_specific_exception(e) + handle_exception(e) - # return False, env_state + self.state_service.write() + + return True, env_state + + return False, env_state def destroy(self, env_state, verbose=False): extra_vars = self._get_vars(env_state.cloud, env_state.build_dir) @@ -177,6 +174,7 @@ def destroy(self, env_state, verbose=False): current_context = env_state.cloud if env_state.cloud in [DOCKER, MINIKUBE] \ else env_state.cloud + '_' + env_state.env + self._delete_resources(current_context) self._unset_context_list(current_context) self._remove_section(current_context) @@ -205,13 +203,9 @@ def _delete_resources(self, context): requests.delete(f"{self.url}/internal/resources") def _tf_init(self, env_state, provider, env, local_backend, destroying=False): - # directory = PROVIDER_DICT.get(cloud) check_environment(provider) if is_cloud_provider(provider): - # provider_directory = f"{directory}/{env}" - # directory = f"{directory}/__working_{env}" if not destroying or not os.path.isdir(env_state.build_dir): - print("here 1") copy_tree(env_state.provider_directory, env_state.build_dir) if local_backend: shutil.copy(LOCAL_CONFIG_DICT.get(provider), env_state.build_dir) @@ -222,6 +216,7 @@ def _tf_init(self, env_state, provider, env, local_backend, destroying=False): self.tf_service.select_workspace(env_state.build_dir, env) else: + copy_tree(env_state.provider_directory, env_state.build_dir) self.tf_service.init(env_state.build_dir) def _set_context_list(self, current_context): @@ -233,17 +228,20 @@ def _set_context_list(self, current_context): updated_contexts = [] if isinstance(contexts, list): - contexts.append(current_context) + if current_context not in contexts: + contexts.append(current_context) updated_contexts = contexts elif isinstance(contexts, str) or not contexts: - # There is only one context or no context in available contexts - if contexts: - # exactly one available context - updated_contexts.append(contexts) - updated_contexts.append(current_context) - else: - # no available context - updated_contexts.append(current_context) + # check if current context is the same as existing context + if current_context != contexts: + # There is only one context or no context in available contexts + if contexts: + # exactly one available context + updated_contexts.append(contexts) + updated_contexts.append(current_context) + else: + # no available context + updated_contexts.append(current_context) self.state_service.set(CONTEXTS, environments=updated_contexts) diff --git a/cli/kaos_cli/services/state_service.py b/cli/kaos_cli/services/state_service.py index 9afe325..3db960c 100644 --- a/cli/kaos_cli/services/state_service.py +++ b/cli/kaos_cli/services/state_service.py @@ -54,5 +54,7 @@ def full_delete(dir_build): shutil.rmtree(dir_build, ignore_errors=True) def write(self): + if not os.path.exists(KAOS_STATE_DIR): + os.mkdir(KAOS_STATE_DIR) with open(CONFIG_PATH, 'wb') as f: self.config.write(f) diff --git a/cli/kaos_cli/utils/validators.py b/cli/kaos_cli/utils/validators.py index c77a115..4233bf3 100644 --- a/cli/kaos_cli/utils/validators.py +++ b/cli/kaos_cli/utils/validators.py @@ -3,6 +3,7 @@ import sys import socket from json import JSONDecodeError +import shutil import click import textdistance @@ -32,18 +33,18 @@ def initialize(cls, cloud, env) -> "EnvironmentState": env_dir = f"{env_state.cloud}/{env_state.env}" if env_state.cloud not in [DOCKER, MINIKUBE] \ else f"{env_state.cloud}" - infra_root = PROVIDER_DICT.get(cloud) + infra_root = f"{PROVIDER_DICT.get(cloud)}" if is_cloud_provider(cloud): build_dir = f"{infra_root}/__working_{env}" - print("build_dir", build_dir) tf_state_path = f"{build_dir}/.terraform/terraform.tfstate" provider_directory = f"{infra_root}/{env}" env_state.provider_directory = provider_directory else: - build_dir = os.path.join(KAOS_TF_PATH, env_dir) - tf_state_path = os.path.join(KAOS_TF_PATH, env_dir, f"{TF_STATE}") + build_dir = f"{infra_root}" + tf_state_path = f"{infra_root}/terraform.tfstate" + env_state.provider_directory = infra_root env_state.if_build_dir_exists = os.path.exists(build_dir) env_state.if_tfstate_exists = os.path.exists(tf_state_path) @@ -52,11 +53,11 @@ def initialize(cls, cloud, env) -> "EnvironmentState": return env_state - def validate_if_build_dir_exits(self) -> "EnvironmentState": + def validate_if_tf_state_exits(self) -> "EnvironmentState": """ Ensure existence of kaos backend dir (tf_path) and throw appropriate warnings if it does not exist """ - if not self.if_build_dir_exists: + if not self.if_tfstate_exists: if self.env: click.echo("{} - {} [{}] backend in {} has not been deployed!".format( click.style("Warning", bold=True, fg='yellow'), @@ -90,6 +91,13 @@ def set_build_env(self) -> "EnvironmentState": else: self.env = 'prod' if not self.env else self.env # default = prod + def remove_terraform_files(self) -> "EnvironmentState": + """ + Simple method to remove existing terraform state for existing deployments + """ + tf_dir_path = f"{self.build_dir}/.terraform" + shutil.rmtree(tf_dir_path, ignore_errors=True) + def validate_index(n: int, ind: int, command: str): """ From 6290969967fd6b4a0c6f4992b9f7d47b89fb378c Mon Sep 17 00:00:00 2001 From: Shankar M S Date: Mon, 20 Jan 2020 11:47:03 +0100 Subject: [PATCH 4/9] build deploy style correction --- cli/kaos_cli/commands/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/kaos_cli/commands/build.py b/cli/kaos_cli/commands/build.py index b6d2e26..0814b90 100644 --- a/cli/kaos_cli/commands/build.py +++ b/cli/kaos_cli/commands/build.py @@ -27,7 +27,7 @@ def build(): @build.command(name='deploy', short_help='{}'.format( - click.style('Build the kaos backend', bold=True, fg='black'))) + click.style('Build the kaos backend', bold=False, fg='white'))) @click.option('-c', '--cloud', type=click.Choice([DOCKER, MINIKUBE, AWS, GCP]), help='selected provider', required=True) @click.option('-e', '--env', type=click.Choice(['prod', 'stage', 'dev']), From 505ea2887f58bce478b226a7d0ab2f2a0901f8e5 Mon Sep 17 00:00:00 2001 From: KI-labs Date: Mon, 20 Jan 2020 14:47:38 +0000 Subject: [PATCH 5/9] Version updated to: 1.1.4 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9c1218c..1b87bcd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.3 \ No newline at end of file +1.1.4 \ No newline at end of file From 5f01d0cfb62ab403d631117c1cf2bf8a2d05f449 Mon Sep 17 00:00:00 2001 From: Shankar M S Date: Fri, 24 Jan 2020 09:43:34 +0100 Subject: [PATCH 6/9] descriptive kaos home (fixed color scheme bug) --- cli/kaos_cli/commands/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/kaos_cli/commands/build.py b/cli/kaos_cli/commands/build.py index d5c669f..785f430 100644 --- a/cli/kaos_cli/commands/build.py +++ b/cli/kaos_cli/commands/build.py @@ -16,7 +16,7 @@ # BUILD group # ============= @click.group(name='build', cls=CustomHelpOrder, - short_help='{} and its {}: {}, {} and {}'.format( + short_help='{} and its {}: {}, {}, {} and {}'.format( click.style('Infrastructure deployments', bold=True), click.style('sub-commands', bold=False), click.style('deploy', bold=True), From 36a16651562da99fbde7f6b106cc328a32acb2c1 Mon Sep 17 00:00:00 2001 From: KI-labs Date: Fri, 24 Jan 2020 08:44:08 +0000 Subject: [PATCH 7/9] Version updated to: 1.1.4 From 16794bd88fb42c128524ad6d715ae70f566adea4 Mon Sep 17 00:00:00 2001 From: Shankar M S Date: Tue, 28 Jan 2020 15:33:43 +0100 Subject: [PATCH 8/9] write_aws_auth_config var removed to resolve aws build fails after merging #23 --- cli/kaos_cli/commands/build.py | 6 ++++-- cli/kaos_cli/facades/backend_facade.py | 12 +++--------- cli/kaos_cli/utils/validators.py | 6 +++--- infrastructure/aws/envs/dev/main.tf | 1 - infrastructure/aws/envs/prod/main.tf | 1 - infrastructure/aws/envs/stage/main.tf | 1 - 6 files changed, 10 insertions(+), 17 deletions(-) diff --git a/cli/kaos_cli/commands/build.py b/cli/kaos_cli/commands/build.py index 785f430..94b919a 100644 --- a/cli/kaos_cli/commands/build.py +++ b/cli/kaos_cli/commands/build.py @@ -7,6 +7,7 @@ from kaos_cli.facades.backend_facade import BackendFacade, is_cloud_provider from typing import Optional +from kaos_cli.utils.helpers import build_dir from kaos_cli.utils.custom_classes import CustomHelpOrder, NotRequiredIf from kaos_cli.utils.decorators import build_env_check, pass_obj from kaos_cli.utils.validators import validate_unused_port, validate_inputs, EnvironmentState @@ -31,8 +32,9 @@ def build(): @build.command(name='deploy', - short_help='{}'.format( - click.style('Build the kaos backend', bold=False, fg='white'))) + short_help='{} the {} backend'.format( + click.style('Build', bold=True), + click.style('kaos', bold=True))) @click.option('-c', '--cloud', type=click.Choice([DOCKER, MINIKUBE, AWS, GCP]), help='selected provider', required=True) @click.option('-e', '--env', type=click.Choice(['prod', 'stage', 'dev']), diff --git a/cli/kaos_cli/facades/backend_facade.py b/cli/kaos_cli/facades/backend_facade.py index 2fd1501..cf63526 100644 --- a/cli/kaos_cli/facades/backend_facade.py +++ b/cli/kaos_cli/facades/backend_facade.py @@ -129,13 +129,14 @@ def set_context_by_index(self, index): def build(self, provider, env, local_backend=False, verbose=False): env_state = EnvironmentState.initialize(provider, env) + if not os.path.exists(env_state.build_dir): build_dir(env_state.build_dir) auth_token = uuid.uuid4() extra_vars = self._get_vars(provider, env_state.build_dir, auth_token) - self.tf_service.cd_dir(env_state.build_dir) + self.tf_service.cd_dir(env_state.build_dir) self.tf_service.set_verbose(verbose) self._tf_init(env_state, provider, env, local_backend, destroying=False) self.tf_service.plan(env_state.build_dir, extra_vars) @@ -148,22 +149,16 @@ def build(self, provider, env, local_backend=False, verbose=False): if env_state.if_tfstate_exists: url, kubeconfig = self._parse_config(env_state.build_dir) - current_context = provider if provider in [DOCKER, MINIKUBE] else f"{provider}_{env}" - self.state_service.set(DEFAULT, user=USER) - self._set_context_list(current_context) self._set_active_context(current_context) self.state_service.set(current_context) - self.state_service.set_section(current_context, BACKEND, url=url, token=auth_token) self.state_service.set_section(current_context, INFRASTRUCTURE, kubeconfig=kubeconfig) - self.state_service.write() - return True, env_state return False, env_state @@ -218,7 +213,6 @@ def _tf_init(self, env_state, provider, env, local_backend, destroying=False): self.tf_service.init(env_state.build_dir) self.tf_service.new_workspace(env_state.build_dir, env) self.tf_service.select_workspace(env_state.build_dir, env) - else: copy_tree(env_state.provider_directory, env_state.build_dir) self.tf_service.init(env_state.build_dir) @@ -310,7 +304,7 @@ def _parse_config(dir_build): @staticmethod def _get_vars(provider, dir_build, auth_token=None): - extra_vars = f"--var config_dir={dir_build} --var token={auth_token}" + extra_vars = f"--var config_dir={dir_build} --var token={auth_token} " if provider == AWS: KEY_ID = os.getenv("AWS_ACCESS_KEY_ID") diff --git a/cli/kaos_cli/utils/validators.py b/cli/kaos_cli/utils/validators.py index 538f721..2b2892e 100644 --- a/cli/kaos_cli/utils/validators.py +++ b/cli/kaos_cli/utils/validators.py @@ -9,7 +9,7 @@ import textdistance from kaos_cli.exceptions.exceptions import MissingArgumentError -from ..constants import KAOS_STATE_DIR, DOCKER, MINIKUBE, KAOS_TF_PATH, TF_STATE, PROVIDER_DICT +from ..constants import KAOS_STATE_DIR, DOCKER, MINIKUBE, PROVIDER_DICT class EnvironmentState: @@ -34,14 +34,14 @@ def initialize(cls, cloud, env) -> "EnvironmentState": infra_root = f"{PROVIDER_DICT.get(cloud)}" if is_cloud_provider(cloud): + provider_directory = f"{infra_root}/{env}" build_dir = f"{infra_root}/__working_{env}" tf_state_path = f"{build_dir}/.terraform/terraform.tfstate" - provider_directory = f"{infra_root}/{env}" env_state.provider_directory = provider_directory else: build_dir = f"{infra_root}" - tf_state_path = f"{infra_root}/terraform.tfstate" + tf_state_path = f"{build_dir}/terraform.tfstate" env_state.provider_directory = infra_root env_state.if_build_dir_exists = os.path.exists(build_dir) diff --git a/infrastructure/aws/envs/dev/main.tf b/infrastructure/aws/envs/dev/main.tf index c857fe2..e78b394 100644 --- a/infrastructure/aws/envs/dev/main.tf +++ b/infrastructure/aws/envs/dev/main.tf @@ -29,7 +29,6 @@ module "eks" { cluster_delete_timeout = "15m" kubeconfig_name = local.prefix config_output_path = "${path.module}/" - write_aws_auth_config = true workers_additional_policies = module.policies.worker_additional_policies tags = merge(local.tags, local.cluster_autoscaler_tags) diff --git a/infrastructure/aws/envs/prod/main.tf b/infrastructure/aws/envs/prod/main.tf index 458351c..6dbe2ef 100644 --- a/infrastructure/aws/envs/prod/main.tf +++ b/infrastructure/aws/envs/prod/main.tf @@ -29,7 +29,6 @@ module "eks" { cluster_delete_timeout = "15m" kubeconfig_name = local.prefix config_output_path = "${path.module}/" - write_aws_auth_config = true workers_additional_policies = module.policies.worker_additional_policies tags = merge(local.tags, local.cluster_autoscaler_tags) diff --git a/infrastructure/aws/envs/stage/main.tf b/infrastructure/aws/envs/stage/main.tf index 786dd13..b2c29e2 100644 --- a/infrastructure/aws/envs/stage/main.tf +++ b/infrastructure/aws/envs/stage/main.tf @@ -29,7 +29,6 @@ module "eks" { cluster_delete_timeout = "15m" kubeconfig_name = local.prefix config_output_path = "${path.module}/" - write_aws_auth_config = true workers_additional_policies = module.policies.worker_additional_policies tags = merge(local.tags, local.cluster_autoscaler_tags) From 42db0b4d5c813ce628a1a32fd85194d2a92aa4e8 Mon Sep 17 00:00:00 2001 From: KI-labs Date: Tue, 28 Jan 2020 14:34:20 +0000 Subject: [PATCH 9/9] Version updated to: 1.1.4