From c57c96e6e83554126f05a535031a1d2f9e8b92bf Mon Sep 17 00:00:00 2001 From: Tung Nguyen Date: Mon, 2 Nov 2020 18:55:59 +0000 Subject: [PATCH] mix layering support, variables, custom helpers, plugin helpers - mix layering support: evaluate DSL so layering can be mixed between YAML and DSL - custom variables support - custom helpers support - plugins helpers support - generators: new resource, new helper, new variable - setup autoloader earlier. removes need for shims - auth login for gcr also - fix cli -h when not within Kubes project --- CHANGELOG.md | 10 ++ README.md | 52 +++++++ docs/_docs/dsl/resources/secret.md | 21 ++- docs/_docs/generators.md | 8 +- docs/_docs/helpers.md | 18 ++- docs/_docs/helpers/aws/advanced.md | 10 ++ docs/_docs/helpers/aws/advanced/secrets.md | 131 +++++++++++++++++ docs/_docs/helpers/aws/advanced/ssm.md | 78 ++++++++++ docs/_docs/helpers/aws/secrets.md | 106 +++----------- docs/_docs/helpers/aws/ssm.md | 58 +++----- docs/_docs/helpers/google/advanced.md | 10 ++ docs/_docs/helpers/google/advanced/secrets.md | 78 ++++++++++ docs/_docs/helpers/google/secrets.md | 45 +++--- docs/_docs/layering.md | 2 + docs/_docs/layering/mix.md | 99 +++++++++++++ docs/_docs/patterns/multiple-envs.md | 55 +++++++ docs/_docs/variables.md | 23 +++ docs/_docs/variables/advanced.md | 62 ++++++++ docs/_docs/variables/basic.md | 137 ++++++++++++++++++ docs/_includes/layering/layers.md | 6 +- docs/_includes/sidebar.html | 31 ++++ docs/_reference/kubes-new-help.md | 15 ++ docs/_reference/kubes-new-helper.md | 25 ++++ docs/_reference/kubes-new-resource.md | 56 +++++++ docs/_reference/kubes-new-variable.md | 20 +++ docs/_reference/kubes-new.md | 44 +----- kubes.gemspec | 4 +- lib/kubes.rb | 7 +- lib/kubes/auth.rb | 5 +- lib/kubes/auth/base.rb | 21 +++ lib/kubes/auth/ecr.rb | 16 +- lib/kubes/auth/gcr.rb | 24 +++ lib/kubes/cli/help/new/helper.md | 4 + .../cli/help/{new.md => new/resource.md} | 6 +- lib/kubes/cli/new.rb | 106 ++------------ lib/kubes/cli/new/helper.rb | 24 +++ lib/kubes/cli/new/resource.rb | 97 +++++++++++++ lib/kubes/cli/new/variable.rb | 16 ++ lib/kubes/command.rb | 2 +- lib/kubes/compiler/decorator/base.rb | 2 +- lib/kubes/compiler/dsl/core/base.rb | 15 +- lib/kubes/compiler/layering.rb | 28 +++- lib/kubes/compiler/shared/custom_variables.rb | 38 +++++ lib/kubes/compiler/shared/plugin_helpers.rb | 14 ++ lib/kubes/compiler/strategy.rb | 13 +- lib/kubes/compiler/strategy/base.rb | 61 +++++++- lib/kubes/compiler/strategy/dsl.rb | 29 ---- lib/kubes/compiler/strategy/erb.rb | 32 ++-- lib/kubes/compiler/util/normalize.rb | 9 +- lib/kubes/compiler/util/yaml_dump.rb | 8 +- lib/kubes/plugin.rb | 14 ++ lib/kubes/util/sh.rb | 2 +- lib/templates/new/helper/file.rb | 2 + .../new/{ => resource}/dsl/backend_config.rb | 0 .../new/{ => resource}/dsl/config_map.rb | 0 .../new/{ => resource}/dsl/daemon_set.rb | 0 .../new/{ => resource}/dsl/deployment.rb | 0 .../new/{ => resource}/dsl/ingress.rb | 0 lib/templates/new/{ => resource}/dsl/job.rb | 0 .../{ => resource}/dsl/managed_certificate.rb | 0 .../new/{ => resource}/dsl/namespace.rb | 0 .../new/{ => resource}/dsl/network_policy.rb | 0 lib/templates/new/{ => resource}/dsl/pod.rb | 0 lib/templates/new/{ => resource}/dsl/role.rb | 0 .../new/{ => resource}/dsl/role_binding.rb | 0 .../new/{ => resource}/dsl/secret.rb | 0 .../new/{ => resource}/dsl/service.rb | 0 .../new/{ => resource}/dsl/service_account.rb | 0 .../{ => resource}/yaml/backend_config.yaml | 0 .../new/{ => resource}/yaml/config_map.yaml | 0 .../new/{ => resource}/yaml/daemon_set.yaml | 0 .../new/{ => resource}/yaml/deployment.yaml | 0 .../new/{ => resource}/yaml/ingress.yaml | 0 .../new/{ => resource}/yaml/job.yaml | 0 .../yaml/managed_certificate.yaml | 0 .../new/{ => resource}/yaml/namespace.yaml | 0 .../{ => resource}/yaml/network_policy.yaml | 0 .../new/{ => resource}/yaml/pod.yaml | 0 .../new/{ => resource}/yaml/role.yaml | 0 .../new/{ => resource}/yaml/role_binding.yaml | 0 .../new/{ => resource}/yaml/secret.yaml | 0 .../new/{ => resource}/yaml/service.yaml | 0 .../{ => resource}/yaml/service_account.yaml | 0 lib/templates/new/variable/file.rb | 1 + .../resources/web}/deployment-1.rb | 0 .../resources/web}/deployment-2.rb | 0 .../.kubes/resources/{ => web}/deployment.rb | 0 .../resources/{foobar.rb => web/empty.rb} | 0 .../.kubes/resources/{ => web}/service.rb | 2 +- .../resources/web}/network_policy.rb | 0 .../syntax/{ => .kubes/resources/web}/pod.rb | 0 spec/kubes/compiler/strategy/dsl_spec.rb | 4 +- spec/kubes/compiler_spec.rb | 2 +- spec/kubes/dsl/network_policy_spec.rb | 2 +- spec/kubes/dsl/pod_spec.rb | 2 +- 95 files changed, 1400 insertions(+), 412 deletions(-) create mode 100644 docs/_docs/helpers/aws/advanced.md create mode 100644 docs/_docs/helpers/aws/advanced/secrets.md create mode 100644 docs/_docs/helpers/aws/advanced/ssm.md create mode 100644 docs/_docs/helpers/google/advanced.md create mode 100644 docs/_docs/helpers/google/advanced/secrets.md create mode 100644 docs/_docs/layering/mix.md create mode 100644 docs/_docs/patterns/multiple-envs.md create mode 100644 docs/_docs/variables.md create mode 100644 docs/_docs/variables/advanced.md create mode 100644 docs/_docs/variables/basic.md create mode 100644 docs/_reference/kubes-new-help.md create mode 100644 docs/_reference/kubes-new-helper.md create mode 100644 docs/_reference/kubes-new-resource.md create mode 100644 docs/_reference/kubes-new-variable.md create mode 100644 lib/kubes/auth/base.rb create mode 100644 lib/kubes/auth/gcr.rb create mode 100644 lib/kubes/cli/help/new/helper.md rename lib/kubes/cli/help/{new.md => new/resource.md} (85%) create mode 100644 lib/kubes/cli/new/helper.rb create mode 100644 lib/kubes/cli/new/resource.rb create mode 100644 lib/kubes/cli/new/variable.rb create mode 100644 lib/kubes/compiler/shared/custom_variables.rb create mode 100644 lib/kubes/compiler/shared/plugin_helpers.rb create mode 100644 lib/kubes/plugin.rb create mode 100644 lib/templates/new/helper/file.rb rename lib/templates/new/{ => resource}/dsl/backend_config.rb (100%) rename lib/templates/new/{ => resource}/dsl/config_map.rb (100%) rename lib/templates/new/{ => resource}/dsl/daemon_set.rb (100%) rename lib/templates/new/{ => resource}/dsl/deployment.rb (100%) rename lib/templates/new/{ => resource}/dsl/ingress.rb (100%) rename lib/templates/new/{ => resource}/dsl/job.rb (100%) rename lib/templates/new/{ => resource}/dsl/managed_certificate.rb (100%) rename lib/templates/new/{ => resource}/dsl/namespace.rb (100%) rename lib/templates/new/{ => resource}/dsl/network_policy.rb (100%) rename lib/templates/new/{ => resource}/dsl/pod.rb (100%) rename lib/templates/new/{ => resource}/dsl/role.rb (100%) rename lib/templates/new/{ => resource}/dsl/role_binding.rb (100%) rename lib/templates/new/{ => resource}/dsl/secret.rb (100%) rename lib/templates/new/{ => resource}/dsl/service.rb (100%) rename lib/templates/new/{ => resource}/dsl/service_account.rb (100%) rename lib/templates/new/{ => resource}/yaml/backend_config.yaml (100%) rename lib/templates/new/{ => resource}/yaml/config_map.yaml (100%) rename lib/templates/new/{ => resource}/yaml/daemon_set.yaml (100%) rename lib/templates/new/{ => resource}/yaml/deployment.yaml (100%) rename lib/templates/new/{ => resource}/yaml/ingress.yaml (100%) rename lib/templates/new/{ => resource}/yaml/job.yaml (100%) rename lib/templates/new/{ => resource}/yaml/managed_certificate.yaml (100%) rename lib/templates/new/{ => resource}/yaml/namespace.yaml (100%) rename lib/templates/new/{ => resource}/yaml/network_policy.yaml (100%) rename lib/templates/new/{ => resource}/yaml/pod.yaml (100%) rename lib/templates/new/{ => resource}/yaml/role.yaml (100%) rename lib/templates/new/{ => resource}/yaml/role_binding.yaml (100%) rename lib/templates/new/{ => resource}/yaml/secret.yaml (100%) rename lib/templates/new/{ => resource}/yaml/service.yaml (100%) rename lib/templates/new/{ => resource}/yaml/service_account.yaml (100%) create mode 100644 lib/templates/new/variable/file.rb rename spec/fixtures/multiple-files/{ => .kubes/resources/web}/deployment-1.rb (100%) rename spec/fixtures/multiple-files/{ => .kubes/resources/web}/deployment-2.rb (100%) rename spec/fixtures/project/.kubes/resources/{ => web}/deployment.rb (100%) rename spec/fixtures/project/.kubes/resources/{foobar.rb => web/empty.rb} (100%) rename spec/fixtures/project/.kubes/resources/{ => web}/service.rb (58%) rename spec/fixtures/syntax/{ => .kubes/resources/web}/network_policy.rb (100%) rename spec/fixtures/syntax/{ => .kubes/resources/web}/pod.rb (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c48930e..dd050c7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,16 @@ All notable changes to this project will be documented in this file. This project *loosely tries* to adhere to [Semantic Versioning](http://semver.org/), even before v1.0. +## [UNRELEASED] +- mix layering support: evaluate DSL so layering can be mixed between YAML and DSL +- custom variables support +- custom helpers support +- plugins helpers support +- generators: new resource, new helper, new variable +- setup autoloader earlier. removes need for shims +- auth login for gcr also +- fix cli -h when not within Kubes project + ## [0.5.1] - fix deployment generator diff --git a/README.md b/README.md index af695ed8..c3e38442 100644 --- a/README.md +++ b/README.md @@ -55,10 +55,62 @@ The deploy command, does all 3 steps: builds the docker image, compiles the `.ku kubes deploy +## Multiple Enviroments + +You can easily create multiple environments with the same YAML configs. Example: + + KUBES_ENV=dev kubes deploy + KUBES_ENV=prod kubes deploy + +See: [Multiple Enviroments Pattern](https://kubes.guru/docs/patterns/multiple-envs/) + +## Generators: Stop Writing Boilerplate + +Your time is precious. Why are we copying and pasting boilerplate structure in this day and age? + +Kubes provides generators to help you get going right away. + +Resources examples: + + $ kubes new resource secret + create .kubes/resources/shared/secret.yaml + $ kubes new resource service_account + create .kubes/resources/shared/service_account.yaml + +Kubes components examples: + + $ kubes new helper + create .kubes/helpers/custom_helper.rb + $ kubes new variable + create .kubes/variables/dev.rb + $ + +## Features + +* Automation: [Builds the Docker image](https://kubes.guru/docs/config/docker/) and updates the compiled YAML files +* Syntactic Sugar: Use an [ERB/YAML](https://kubes.guru/docs/yaml/) or a [DSL](https://kubes.guru/docs/dsl/) to write your Kubernetes YAML files. You can use a mix of DSL and YAML definitions in the `.kubes/resources` folder. +* Layering: Use the same Kubernetes YAML to build multiple environments like dev and prod with [layering](https://kubes.guru/docs/layering/). +* Generators: Kubes ships with a few generators to help you get building with Kubernetes quickly. See: [Generator Docs](https://kubes.guru/docs/generators/). +* CLI Customizations: You can customize the [cli args](https://kubes.guru/docs/config/args/kubectl/). +* Hooks: You can also run [hooks](https://kubes.guru/docs/config/hooks/) before and after [kubes](https://kubes.guru/docs/config/hooks/kubes/) and [kubectl](https://kubes.guru/docs/config/hooks/kubectl/) commands. +* Automated Suffix Hashes: Automatically appends a suffix hash to ConfigMap and Secret resources. More details in [ConfigMap](https://kubes.guru/docs/dsl/resources/config_map/) and [Secret](https://kubes.guru/docs/dsl/resources/secret/) docs. +* Kustomize Support: If you're a kustomization user, you can use it with Kubes. More details in [Kustomize Support Docs](https://kubes.guru/docs/misc/kustomize/). +* Auto Context Switching: Map dev to a specific kubectl context and prod to another kubectl context and Kubes can switch them automatically so you won't have to remember. More details in [Auto Context Docs](https://kubes.guru/docs/misc/auto-context/). +* Ordering: Kubes run kubectl apply to create resources in the [correct order](https://kubes.guru/docs/intro/ordering/). For deleting, it kubes will run `kubectl delete` in the correct reverse order. The order is also [customizable](https://kubes.guru/docs/intro/ordering/custom/). + ## Installation Install with: gem install kubes +## Comparison + +Here are some useful comparisons to help you compare Kubes vs other tools in the ecosystem: + +* Blog Post: [Kustomize vs Helm vs Kubes: Kubernetes Deploy Tools](https://blog.boltops.com/2020/11/05/kustomize-vs-helm-vs-kubes-kubernetes-deploy-tools) +* [Kubes vs Custom Solution](https://kubes.guru/docs/vs/custom/) +* [Kubes vs Helm](https://kubes.guru/docs/vs/helm/) +* [Kubes vs Kustomize](https://kubes.guru/docs/vs/kustomize/) + For more info: [kubes.guru](https://kubes.guru) diff --git a/docs/_docs/dsl/resources/secret.md b/docs/_docs/dsl/resources/secret.md index de01b404..9a8a4995 100644 --- a/docs/_docs/dsl/resources/secret.md +++ b/docs/_docs/dsl/resources/secret.md @@ -19,7 +19,7 @@ data( Produces: -.kubes/output/shared/service.yaml +.kubes/output/shared/secret.yaml ```yaml apiVersion: v1 @@ -38,6 +38,23 @@ data: {% include dsl/rolling_deployment.md kind="Secret" %} +.kubes/resources/web/deployment.yaml: + +```yaml +# .. +spec: + template: + spec: + containers: + - name: demo + image: nginx + envFrom: + - secretRef: + name: demo-secret +``` + +Produces: + .kubes/output/web/deployment.yaml: ```yaml @@ -46,7 +63,7 @@ spec: template: spec: containers: - - name: demo-shared + - name: demo image: nginx envFrom: - secretRef: diff --git a/docs/_docs/generators.md b/docs/_docs/generators.md index 51083c41..548ab4c8 100644 --- a/docs/_docs/generators.md +++ b/docs/_docs/generators.md @@ -8,15 +8,15 @@ Kubes ships with a few generators to help you get building with Kubernetes quick Here are a few examples: - $ kubes new ingress + $ kubes new resource ingress create .kubes/resources/web/ingress.yaml - $ kubes new service_account + $ kubes new resource service_account create .kubes/resources/shared/service_account.yaml $ Use `-h` to see the cli options: - kubes new -h + kubes new resource -h ## Supported Resources @@ -38,4 +38,4 @@ Here's a list of some of the supported resources. service_account service -Refer to the [source code](https://github.com/boltops-tools/kubes/blob/master/lib/templates/new/yaml) to all the resources that the generator supports. +Refer to the [source code](https://github.com/boltops-tools/kubes/blob/master/lib/templates/new/resource/yaml) to all the resources that the generator supports. diff --git a/docs/_docs/helpers.md b/docs/_docs/helpers.md index fe0d96a6..232c8c5f 100644 --- a/docs/_docs/helpers.md +++ b/docs/_docs/helpers.md @@ -6,10 +6,10 @@ Kubes provides some helper methods to help write Kubernetes YAML files. Here's Helper | Description --- | --- -decode64 | Basey64 decode a string. +decode64 | Base64 decode a string. docker_image | Method refers to the latest Docker image built by Kubes. This spares you from having to update the image manually in the deployment resource. Note, this can be overridden with the `--image` cli option or the `Kubes.config.image` setting. See: [Docker Image]({% link _docs/intro/docker-image.md %}) dockerfile_port | Exposed port extracted from the Dockerfile of the project. -encode64 | Basey64 encode a string. Also available as `base64` method. +encode64 | Base64 encode a string. Also available as `base64` method. extra | The `KUBES_EXTRA` value. with_extra | Appends the `KUBES_EXTRA` value to a string if it's set. It's covered in the [Extra Env Docs]({% link _docs/extra-env.md %}). @@ -25,3 +25,17 @@ There are also provider-specific helpers: * [AWS Helpers]({% link _docs/helpers/aws.md %}) * [Google Helpers]({% link _docs/helpers/google.md %}) + +## Generator + +To help you get started quickly, you can generate starter helper code. + + $ kubes new helper custom + create .kubes/helpers/custom_helper.rb + +.kubes/helpers/custom_helper.rb + +```ruby +module CustomHelper +end +``` \ No newline at end of file diff --git a/docs/_docs/helpers/aws/advanced.md b/docs/_docs/helpers/aws/advanced.md new file mode 100644 index 00000000..ca05972b --- /dev/null +++ b/docs/_docs/helpers/aws/advanced.md @@ -0,0 +1,10 @@ +--- +title: Advanced AWS Helpers +nav_text: Advanced +categories: helpers-aws +--- + +{% assign docs = site.docs | where: "categories","advanced-helpers-aws" %} +{% for doc in docs -%} + * [{{ doc.nav_text }}]({{ doc.url }}) +{% endfor %} diff --git a/docs/_docs/helpers/aws/advanced/secrets.md b/docs/_docs/helpers/aws/advanced/secrets.md new file mode 100644 index 00000000..89ec8a61 --- /dev/null +++ b/docs/_docs/helpers/aws/advanced/secrets.md @@ -0,0 +1,131 @@ +--- +title: AWS Secrets Advanced +nav_text: Secrets +categories: advanced-helpers-aws +--- + +This covers an advanced way so that Kubernetes Secrets are created from AWS Secrets Manager in a conventional way. + +## Simple Values + +For example if you have these secret values: + + $ aws secretsmanager get-secret-value --secret-id demo/dev/db_user | jq '.SecretString' + user + $ aws secretsmanager get-secret-value --secret-id demo/dev/db_pass | jq '.SecretString' + pass + +Set up a [Kubes hook](https://kubes.guru/docs/config/hooks/kubes/). + +.kubes/config/hooks/kubes.rb + +```ruby +secrets = KubesAws::Secrets.new(upcase: true, prefix: "demo/dev/") +before("compile", + label: "Get secrets from AWS Secrets Manager", + execute: secrets, +) +``` + +Then set the secrets in the YAML: + +.kubes/resources/shared/secret.yaml + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: demo + labels: + app: demo +data: +<% KubesAws::Secrets.data.each do |k,v| -%> + <%= k %>: <%= base64(v) %> +<% end -%> +``` + +This results in AWS secrets with the prefix the `demo/dev/` being added to the Kubernetes secret data. The values are automatically base64 encoded. Produces: + +.kubes/output/shared/secret.yaml + +```yaml +metadata: + namespace: demo + name: demo-2a78a13682 + labels: + app: demo +apiVersion: v1 +kind: Secret +data: + db_pass: dGVzdDEK + db_user: dGVzdDIK +``` + +## JSON Values + +For example if you have these secret values: + + $ aws secretsmanager get-secret-value --secret-id demo/dev/k2 | jq '.SecretString' + {\"a\":1,\"b\":2}" + +Set up a [Kubes hook](https://kubes.guru/docs/config/hooks/kubes/). + +.kubes/config/hooks/kubes.rb + +```ruby +secrets = KubesAws::Secrets.new(prefix: "rails/dev/") +before("compile", + label: "Get secrets from AWS Secrets Manager", + execute: secrets, +) +``` + +Then set the secrets in the YAML: + +.kubes/resources/shared/secret.yaml + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: demo + labels: + app: demo +data: +<% k2 = JSON.load(KubesAws::Secrets.data["k2"]) %> + a: <%= base64(k2["a"]) %> + b: <%= base64(k2["b"]) %> +``` + +Produces: + +```yaml +metadata: + namespace: demo-dev + name: demo-a4cd604a95 + labels: + app: demo +apiVersion: v1 +kind: Secret +data: + a: MQ== + b: Mg== +``` + +## Variables + +These environment variables can be set: + +Name | Description +---|--- +AWS_SECRET_PREFIX | Prefixed used to list and filter AWS secrets. IE: `demo/dev/`. + +Secrets#initialize options: + +Variable | Description | Default +---|---|--- +base64 | Automatically base64 encode the values. | false +upcase | Automatically upcase the Kubernetes secret data keys. | false +prefix | Prefixed used to list and filter AWS secrets. IE: `demo/dev/`. Can also be set with the `AWS_SECRET_PREFIX` env variable. The env variable takes the highest precedence. | nil + +{% include helpers/base64.md %} diff --git a/docs/_docs/helpers/aws/advanced/ssm.md b/docs/_docs/helpers/aws/advanced/ssm.md new file mode 100644 index 00000000..80704f86 --- /dev/null +++ b/docs/_docs/helpers/aws/advanced/ssm.md @@ -0,0 +1,78 @@ +--- +title: AWS SSM Parameters Advanced +nav_text: SSM +categories: advanced-helpers-aws +--- + +This covers an advanced way so that Kubernetes Secrets are created from AWS SSM Parameter Store in a conventional way. + +For example if you have these secret values: + + $ aws ssm get-parameter --name /demo/development/db_user --with-decryption | jq '.Parameter.Value' + user + $ aws ssm get-parameter --name /demo/development/db_pass --with-decryption | jq '.Parameter.Value' + pass + +Set up a [Kubes hook](https://kubes.guru/docs/config/hooks/kubes/). + +.kubes/config/hooks/kubes.rb + +```ruby +ssm = KubesAws::SSM.new(upcase: true, prefix: "/demo/development/") +before("compile", + label: "Get secrets from AWS SSM Manager", + execute: ssm, +) +``` + +Then set the secrets in the YAML: + +.kubes/resources/shared/secret.yaml + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: demo + labels: + app: demo +data: +<% KubesAws::SSM.data.each do |k,v| -%> + <%= k %>: <%= base64(v) %> +<% end -%> +``` + +This results in AWS secrets with the prefix the `demo/dev/` being added to the Kubernetes secret data. The values are automatically base64 encoded. Produces: + +.kubes/output/shared/secret.yaml + +```yaml +metadata: + namespace: demo + name: demo-2a78a13682 + labels: + app: demo +apiVersion: v1 +kind: Secret +data: + db_pass: dGVzdDEK + db_user: dGVzdDIK +``` + +## Variables + +These environment variables can be set: + +Name | Description +---|--- +AWS_SSM_PREFIX | Prefixed used to list and filter AWS SSM Parameters. IE: `demo/dev/`. + +Secrets#initialize options: + +Variable | Description | Default +---|---|--- +base64 | Automatically base64 encode the values. | false +upcase | Automatically upcase the Kubernetes secret data keys. | false +prefix | Prefixed used to list and filter AWS secrets. IE: `demo/dev/`. Can also be set with the `AWS_SECRET_PREFIX` env variable. The env variable takes the highest precedence. | nil + +{% include helpers/base64.md %} \ No newline at end of file diff --git a/docs/_docs/helpers/aws/secrets.md b/docs/_docs/helpers/aws/secrets.md index ae16009e..3e42232d 100644 --- a/docs/_docs/helpers/aws/secrets.md +++ b/docs/_docs/helpers/aws/secrets.md @@ -4,28 +4,9 @@ nav_text: Secrets categories: helpers-aws --- -## Simple Values +The `aws_secret` helper fetches secret data from AWS Secrets Manager. -For example if you have these secret values: - - $ aws secretsmanager get-secret-value --secret-id demo/dev/db_user | jq '.SecretString' - user - $ aws secretsmanager get-secret-value --secret-id demo/dev/db_pass | jq '.SecretString' - pass - -Set up a [Kubes hook](https://kubes.guru/docs/config/hooks/kubes/). - -.kubes/config/hooks/kubes.rb - -```ruby -secrets = KubesAws::Secrets.new(upcase: true, prefix: "demo/dev/") -before("compile", - label: "Get secrets from AWS Secrets Manager", - execute: secrets, -) -``` - -Then set the secrets in the YAML: +## Example .kubes/resources/shared/secret.yaml @@ -37,12 +18,17 @@ metadata: labels: app: demo data: -<% KubesAws::Secrets.data.each do |k,v| -%> - <%= k %>: <%= base64(v) %> -<% end -%> + PASS: <%= aws_secret("demo-#{Kubes.env}-PASS") %> + USER: <%= aws_secret("demo-#{Kubes.env}-USER") %> ``` -This results in AWS secrets with the prefix the `demo/dev/` being added to the Kubernetes secret data. The values are automatically base64 encoded. Produces: +For example if you have these secret values: + + $ aws secretsmanager get-secret-value --secret-id demo-dev-PASS | jq '.SecretString' + test1 + $ aws secretsmanager get-secret-value --secret-id demo-dev-USER | jq '.SecretString' + test2 + $ .kubes/output/shared/secret.yaml @@ -55,75 +41,19 @@ metadata: apiVersion: v1 kind: Secret data: - db_pass: dGVzdDEK - db_user: dGVzdDIK + PASS: dGVzdDEK + USER: dGVzdDIK ``` -## JSON Values +The values are automatically base64 encoded. -For example if you have these secret values: +## Base64 Option - $ aws secretsmanager get-secret-value --secret-id demo/dev/k2 | jq '.SecretString' - {\"a\":1,\"b\":2}" - -Set up a [Kubes hook](https://kubes.guru/docs/config/hooks/kubes/). - -.kubes/config/hooks/kubes.rb +The value is automatically base64 encoded. You can set the `base64` option to turn on and off the automated base64 encoding. ```ruby -secrets = KubesAws::Secrets.new(prefix: "rails/dev/") -before("compile", - label: "Get secrets from AWS Secrets Manager", - execute: secrets, -) +aws_secret("demo-#{Kubes.env}-USER", base64: true) # default is base64=true +aws_secret("demo-#{Kubes.env}-PASS", base64: false) ``` -Then set the secrets in the YAML: - -.kubes/resources/shared/secret.yaml - -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: demo - labels: - app: demo -data: -<% k2 = JSON.load(KubesAws::Secrets.data["k2"]) %> - a: <%= base64(k2["a"]) %> - b: <%= base64(k2["b"]) %> -``` - -Produces: - -```yaml -metadata: - namespace: demo-dev - name: demo-a4cd604a95 - labels: - app: demo -apiVersion: v1 -kind: Secret -data: - a: MQ== - b: Mg== -``` - -## Variables - -These environment variables can be set: - -Name | Description ----|--- -AWS_SECRET_PREFIX | Prefixed used to list and filter AWS secrets. IE: `demo/dev/`. - -Secrets#initialize options: - -Variable | Description | Default ----|---|--- -base64 | Automatically base64 encode the values. | false -upcase | Automatically upcase the Kubernetes secret data keys. | false -prefix | Prefixed used to list and filter AWS secrets. IE: `demo/dev/`. Can also be set with the `AWS_SECRET_PREFIX` env variable. The env variable takes the highest precedence. | nil - {% include helpers/base64.md %} diff --git a/docs/_docs/helpers/aws/ssm.md b/docs/_docs/helpers/aws/ssm.md index de9677c5..a03a6552 100644 --- a/docs/_docs/helpers/aws/ssm.md +++ b/docs/_docs/helpers/aws/ssm.md @@ -4,26 +4,9 @@ nav_text: SSM categories: helpers-aws --- -For example if you have these secret values: +The `aws_ssm` helper fetches data from AWS SSM Parameter Store. - $ aws ssm get-parameter --name /demo/development/db_user --with-decryption | jq '.Parameter.Value' - user - $ aws ssm get-parameter --name /demo/development/db_pass --with-decryption | jq '.Parameter.Value' - pass - -Set up a [Kubes hook](https://kubes.guru/docs/config/hooks/kubes/). - -.kubes/config/hooks/kubes.rb - -```ruby -ssm = KubesAws::SSM.new(upcase: true, prefix: "/demo/development/") -before("compile", - label: "Get secrets from AWS SSM Manager", - execute: ssm, -) -``` - -Then set the secrets in the YAML: +## Example .kubes/resources/shared/secret.yaml @@ -35,12 +18,16 @@ metadata: labels: app: demo data: -<% KubesAws::SSM.data.each do |k,v| -%> - <%= k %>: <%= base64(v) %> -<% end -%> + PASS: <%= aws_ssm("/demo/#{Kubes.env}/PASS") %> + USER: <%= aws_ssm("/demo/#{Kubes.env}/USER") %> ``` -This results in AWS secrets with the prefix the `demo/dev/` being added to the Kubernetes secret data. The values are automatically base64 encoded. Produces: +For example if you have these ssm parameter values: + + $ aws ssm get-parameter --name /demo/dev/PASS --with-decryption | jq '.Parameter.Value' + test1 + $ aws ssm get-parameter --name /demo/dev/USER --with-decryption | jq '.Parameter.Value' + test2 .kubes/output/shared/secret.yaml @@ -53,24 +40,19 @@ metadata: apiVersion: v1 kind: Secret data: - db_pass: dGVzdDEK - db_user: dGVzdDIK + PASS: dGVzdDEK + USER: dGVzdDIK ``` -## Variables - -These environment variables can be set: +The values are automatically base64 encoded. -Name | Description ----|--- -AWS_SSM_PREFIX | Prefixed used to list and filter AWS SSM Parameters. IE: `demo/dev/`. +## Base64 Option -Secrets#initialize options: +The value is automatically base64 encoded. You can set the `base64` option to turn on and off the automated base64 encoding. -Variable | Description | Default ----|---|--- -base64 | Automatically base64 encode the values. | false -upcase | Automatically upcase the Kubernetes secret data keys. | false -prefix | Prefixed used to list and filter AWS secrets. IE: `demo/dev/`. Can also be set with the `AWS_SECRET_PREFIX` env variable. The env variable takes the highest precedence. | nil +```ruby +aws_ssm("/demo/#{Kubes.env}/USER", base64: true) # default is base64=true +aws_ssm("/demo/#{Kubes.env}/PASS", base64: false) +``` -{% include helpers/base64.md %} \ No newline at end of file +{% include helpers/base64.md %} diff --git a/docs/_docs/helpers/google/advanced.md b/docs/_docs/helpers/google/advanced.md new file mode 100644 index 00000000..790be995 --- /dev/null +++ b/docs/_docs/helpers/google/advanced.md @@ -0,0 +1,10 @@ +--- +title: Advanced Google Helpers +nav_text: Advanced +categories: helpers-google +--- + +{% assign docs = site.docs | where: "categories","advanced-helpers-google" %} +{% for doc in docs -%} + * [{{ doc.nav_text }}]({{ doc.url }}) +{% endfor %} diff --git a/docs/_docs/helpers/google/advanced/secrets.md b/docs/_docs/helpers/google/advanced/secrets.md new file mode 100644 index 00000000..9567fb96 --- /dev/null +++ b/docs/_docs/helpers/google/advanced/secrets.md @@ -0,0 +1,78 @@ +--- +title: Advanced Google Secrets +nav_text: Secrets +categories: advanced-helpers-google +--- + +This covers an advanced way so that Kubernetes Secrets are created from Google Secrets in a conventional way. + +Set up a [Kubes hook](https://kubes.guru/docs/config/hooks/kubes/). + +.kubes/config/hooks/kubes.rb + +```ruby +before("compile", + execute: KubesGoogle::Secrets.new(upcase: true, prefix: 'projects/686010496118/secrets/demo-dev-') +) +``` + +Then set the secrets in the YAML: + +.kubes/resources/shared/secret.yaml + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: demo + labels: + app: demo +data: +<% KubesGoogle::Secrets.data.each do |k,v| -%> + <%= k %>: <%= base64(v) %> +<% end -%> +``` + +This results in Google secrets with the prefix the `demo-dev-` being added to the Kubernetes secret data. The values are automatically base64 encoded. + +For example if you have these secret values: + + $ gcloud secrets versions access latest --secret demo-dev-db_user + test1 + $ gcloud secrets versions access latest --secret demo-dev-db_pass + test2 + $ + +.kubes/output/shared/secret.yaml + +```yaml +metadata: + namespace: demo + name: demo-2a78a13682 + labels: + app: demo +apiVersion: v1 +kind: Secret +data: + db_pass: dGVzdDEK + db_user: dGVzdDIK +``` + +## Variables + +These environment variables can be set: + +Name | Description +---|--- +GCP_SECRET_PREFIX | Prefixed used to list and filter Google secrets. IE: `projects/686010496118/secrets/demo-dev-`. +GOOGLE_PROJECT | Google project id. + +Secrets#initialize options: + +Variable | Description | Default +---|---|--- +base64 | Automatically base64 encode the values. | false +upcase | Automatically upcase the Kubernetes secret data keys. | false +prefix | Prefixed used to list and filter Google secrets. IE: `projects/686010496118/secrets/demo-dev-`. Can also be set with the `GCP_SECRET_PREFIX` env variable. The env variable takes the highest precedence. | nil + +{% include helpers/base64.md %} \ No newline at end of file diff --git a/docs/_docs/helpers/google/secrets.md b/docs/_docs/helpers/google/secrets.md index a2194aa4..1e9e09d2 100644 --- a/docs/_docs/helpers/google/secrets.md +++ b/docs/_docs/helpers/google/secrets.md @@ -4,17 +4,9 @@ nav_text: Secrets categories: helpers-google --- -Set up a [Kubes hook](https://kubes.guru/docs/config/hooks/kubes/). +The `google_secret` helper fetches secret data from Google Secrets. -.kubes/config/hooks/kubes.rb - -```ruby -before("compile", - execute: KubesGoogle::Secrets.new(upcase: true, prefix: 'projects/686010496118/secrets/demo-dev-') -) -``` - -Then set the secrets in the YAML: +## Example .kubes/resources/shared/secret.yaml @@ -26,18 +18,17 @@ metadata: labels: app: demo data: -<% KubesGoogle::Secrets.data.each do |k,v| -%> - <%= k %>: <%= base64(v) %> -<% end -%> + PASS: <%= google_secret("demo-#{Kubes.env}-PASS") %> + USER: <%= google_secret("demo-#{Kubes.env}-USER") %> ``` -This results in Google secrets with the prefix the `demo-dev-` being added to the Kubernetes secret data. The values are automatically base64 encoded. +The values are automatically base64 encoded. For example if you have these secret values: - $ gcloud secrets versions access latest --secret demo-dev-db_user + $ gcloud secrets versions access latest --secret demo-dev-USER test1 - $ gcloud secrets versions access latest --secret demo-dev-db_pass + $ gcloud secrets versions access latest --secret demo-dev-PASS test2 $ @@ -52,8 +43,8 @@ metadata: apiVersion: v1 kind: Secret data: - db_pass: dGVzdDEK - db_user: dGVzdDIK + PASS: dGVzdDEK + USER: dGVzdDIK ``` ## Variables @@ -62,15 +53,15 @@ These environment variables can be set: Name | Description ---|--- -GCP_SECRET_PREFIX | Prefixed used to list and filter Google secrets. IE: `projects/686010496118/secrets/demo-dev-`. -GOOGLE_PROJECT | Google project id. +GOOGLE_PROJECT | Google project id. This is required. -Secrets#initialize options: +## Base64 Option -Variable | Description | Default ----|---|--- -base64 | Automatically base64 encode the values. | false -upcase | Automatically upcase the Kubernetes secret data keys. | false -prefix | Prefixed used to list and filter Google secrets. IE: `projects/686010496118/secrets/demo-dev-`. Can also be set with the `GCP_SECRET_PREFIX` env variable. The env variable takes the highest precedence. | nil +The value is automatically base64 encoded. You can set the `base64` option to turn on and off the automated base64 encoding. + +```ruby +google_secret("demo-#{Kubes.env}-USER", base64: true) # default is base64=true +google_secret("demo-#{Kubes.env}-PASS", base64: false) +``` -{% include helpers/base64.md %} \ No newline at end of file +{% include helpers/base64.md %} diff --git a/docs/_docs/layering.md b/docs/_docs/layering.md index 87ca325b..5e3a0798 100644 --- a/docs/_docs/layering.md +++ b/docs/_docs/layering.md @@ -6,3 +6,5 @@ Kubes supports layering files together so you can use the same Kubernetes files * [YAML Layering]({% link _docs/layering/yaml.md %}) * [DSL Layering]({% link _docs/layering/dsl.md %}) +* [Mix Layering]({% link _docs/layering/mix.md %}) +* [Merge Behavior]({% link _docs/layering/merge.md %}) diff --git a/docs/_docs/layering/mix.md b/docs/_docs/layering/mix.md new file mode 100644 index 00000000..f64409e3 --- /dev/null +++ b/docs/_docs/layering/mix.md @@ -0,0 +1,99 @@ +--- +title: Mixed Layering +--- + +You can mix and match the YAML and DSL forms together and layering still works. + +## Project Structure + +Here's an example structure, so we can understand how layering works. + + .kubes/resources/ + ├── base + │ ├── all.yaml + │ └── deployment.yaml + └── web + ├── deployment + │ ├── dev.yaml + │ └── prod.yaml + ├── deployment.rb + └── service.yaml + +Notice, how deployment.rb is defined as a DSL. The layers will still be merged like so: + + .kubes/resources/base/all.yaml + .kubes/resources/base/deployment.yaml + .kubes/resources/web/deployment.rb + .kubes/resources/web/deployment/dev.rb + +## Resources Files + +.kubes/resources/base/all.yaml + +```yaml +metadata: + namespace: demo-<%= Kubes.env %> + labels: + app: demo +``` + +.kubes/resources/base/deployment.yaml + +```yaml +metadata: + labels: + app: demo +spec: + selector: + matchLabels: + app: demo + template: + metadata: + labels: + app: demo +``` + +.kubes/resources/web/deployment.rb + +```ruby +name "web" +labels(role: "web") +image "nginx" +``` + +.kubes/resources/web/deployment/dev.yaml + +```yaml +spec: + replicas: 2 +``` + +## Output + +The result is the merge layered files. + +```yaml +metadata: + namespace: demo-dev + labels: + app: demo + role: web + name: web +spec: + selector: + matchLabels: + app: demo + role: web + template: + metadata: + labels: + app: demo + role: web + spec: + containers: + - image: nginx + name: web + replicas: 2 +apiVersion: apps/v1 +kind: Deployment +``` diff --git a/docs/_docs/patterns/multiple-envs.md b/docs/_docs/patterns/multiple-envs.md new file mode 100644 index 00000000..b4901724 --- /dev/null +++ b/docs/_docs/patterns/multiple-envs.md @@ -0,0 +1,55 @@ +--- +title: Multiple Enviroments with Layering +nav_text: Multiple Enviroments +categories: patterns +--- + +You can use Kubes to easily create multiple enviroments with the same YAML configs. This is thanks to [Kubes Layering]({% link _docs/layering.md %}). We'll walk through an example to help understand how it works. + +## Creating Multiple Enviroments + +To create multiple enviroments like dev and prod just change KUBES_ENV. Example: + + KUBES_ENV=dev kubes deploy + KUBES_ENV=prod kubes deploy + +Different env files will be layered and merged to produce YAML files specific to each environment. + +## Project Structure + +Here's an example structure, so we can understand how layering works to create multiple enviroments. + + .kubes/resources/ + ├── base + │ ├── all.yaml + │ └── deployment.yaml + └── web + ├── deployment + │ ├── dev.yaml + │ └── prod.yaml + ├── deployment.yaml + └── service.yaml + +## Concrete Example + +Let's look at a concrete web/deployment.yaml. + +Here are the files that get layered when `KUBES_ENV=dev`: + + .kubes/resources/base/all.yaml + .kubes/resources/base/deployment.yaml + .kubes/resources/web/deployment.yaml + .kubes/resources/web/deployment/dev.yaml + +And when `KUBES_ENV=prod`: + + .kubes/resources/base/all.yaml + .kubes/resources/base/deployment.yaml + .kubes/resources/web/deployment.yaml + .kubes/resources/web/deployment/prod.yaml + +Layering allows us to have common settings that are processed before your main `.kubes/resources/web/deployment.yaml` YAML manifest. And then add **environment** specific YAML files that get merged. + +## Variables and Helpers + +Additional, you can use variables and helpers to provide environment specific values. diff --git a/docs/_docs/variables.md b/docs/_docs/variables.md new file mode 100644 index 00000000..f50f27d0 --- /dev/null +++ b/docs/_docs/variables.md @@ -0,0 +1,23 @@ +--- +title: Variables +--- + +You can set variables to be made available to the templates. Generally, it is recommended to use Basic layering. + +{% assign docs = site.docs | where: "categories","variables" %} +{% for doc in docs -%} +* [{{ doc.title }}]({{ doc.url }}) +{% endfor %} + +## Generator + +To help you get started quickly, you can generate starter variable code. + + $ kubes new variable + create .kubes/variables/dev.rb + +.kubes/variables/dev.rb + +```ruby +@example = "dev-value" +``` diff --git a/docs/_docs/variables/advanced.md b/docs/_docs/variables/advanced.md new file mode 100644 index 00000000..65dc23c6 --- /dev/null +++ b/docs/_docs/variables/advanced.md @@ -0,0 +1,62 @@ +--- +title: Advanced Variables +nav_text: Advanced +categories: variables +--- + +Basic variables layering should provides enough flexibility and is generally recommended. This page covers more advanced variables layering. + +## Advanced Layering Example + +Here's a more complex structure to demonstrate advanced layering. + + .kubes/variables + ├── base + │ └── deployment.rb + ├── base.rb + ├── dev.rb + ├── prod.rb + └── web + ├── deployment + │ ├── dev.rb + │ └── prod.rb + └── deployment.rb + +## Concrete Example + +Let's look at a concrete web/deployment.yaml. + +Here are the files that get layered when `KUBES_ENV=dev`: + + .kubes/variables/base.rb + .kubes/variables/dev.rb + .kubes/variables/base/deployment.rb + .kubes/variables/web/deployment.rb + .kubes/variables/web/deployment/dev.rb + +And when `KUBES_ENV=prod`: + + .kubes/variables/base.rb + .kubes/variables/prod.rb + .kubes/variables/base/deployment.rb + .kubes/variables/web/deployment.rb + .kubes/variables/web/deployment/prod.rb + +With advanced layering you can target a specific role and kind. So variables are only scoped to the resources you want. + +## Full Layering Table + +Here's a table showing the the full layering. + +Folder/Pattern | Example +------------------|-------------------------------------------- +base.rb | base.rb +ENV.rb | dev.rb +base/all.rb | base/all.rb +base/all/ENV.rb | base/all/dev.rb +base/KIND.rb | base/deployment.rb +base/KIND/base.rb | base/deployment/base.rb +base/KIND/ENV.rb | base/deployment/dev.rb +ROLE/KIND.rb | web/deployment.rb +ROLE/KIND/base.rb | web/deployment/base.rb +ROLE/KIND/ENV.rb | web/deployment/dev.rb diff --git a/docs/_docs/variables/basic.md b/docs/_docs/variables/basic.md new file mode 100644 index 00000000..5dc8fed6 --- /dev/null +++ b/docs/_docs/variables/basic.md @@ -0,0 +1,137 @@ +--- +title: Basic Variables +nav_text: Basic +categories: variables +--- + +## Basic Layering Example + +Here's an example variables directory structure: + + .kubes/variables + ├── base.rb + ├── dev.rb + └── prod.rb + +base.rb + +```ruby +@endpoint = "base-endpoint" # overriden by ENV specific variable files. +``` + +dev.rb + +```ruby +@endpoint = "dev-endpoint" +``` + +prod.rb + +```ruby +@endpoint = "prod-endpoint" +``` + +The `@endpoint` value will be overriden by the ENV specific variable files. + +## Deployment YAML + +Here's an example deployment.yaml + +.kubes/resources/web/deployment.yaml + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web + labels: + role: web +spec: + replicas: 1 # overridden on a env basis + selector: + matchLabels: + role: web + template: + metadata: + labels: + role: web + spec: + containers: + - name: web + image: <%= docker_image %> + env: + - name: endpoint + value: <%= @endpoint %> +``` + +## Deploy + +When you deploy you can use `KUBES_ENV` to and the ENV specific variables will be used: + + KUBES_ENV=dev kubes deploy + +Results in: + +```yaml +metadata: + namespace: demo-dev + labels: + app: demo + role: web + name: web +spec: + selector: + matchLabels: + app: demo + role: web + template: + metadata: + labels: + app: demo + role: web + spec: + containers: + - name: web + image: gcr.io/GOOGLE_PROJECT/demo:kubes-2020-11-07T22-29-02 + env: + - name: endpoint + value: dev-endpoint + replicas: 1 +apiVersion: apps/v1 +kind: Deployment +``` + +When using `KUBES_ENV=prod` the endpoint will use the `variables/prod.rb` values. + + KUBES_ENV=prod kubes deploy + +Results in: + +```yaml +metadata: + namespace: demo-prod + labels: + app: demo + role: web + name: web +spec: + selector: + matchLabels: + app: demo + role: web + template: + metadata: + labels: + app: demo + role: web + spec: + containers: + - name: web + image: gcr.io/GOOGLE_PROJECT/demo:kubes-2020-11-07T22-29-02 + env: + - name: endpoint + value: prod-endpoint + replicas: 1 +apiVersion: apps/v1 +kind: Deployment +``` diff --git a/docs/_includes/layering/layers.md b/docs/_includes/layering/layers.md index fd567d17..a4835ae4 100644 --- a/docs/_includes/layering/layers.md +++ b/docs/_includes/layering/layers.md @@ -1,6 +1,6 @@ Kubes Layering in it's full form allows you to keep your resource definitions DRY and create different environments with the same code. -## Structure +## Project Structure Here's an example structure, so we can understand how layering works. @@ -25,9 +25,7 @@ To explain the layering, here's the general processing order that Kubes takes. 2. Then Kubes will process your `.kubes/resources/ROLE` definitions. 3. Lastly, Kubes processes any post-layers in the `.kubes/resources/ROLE/KIND` folders. -Note, both YAML and DSL forms support layering. - -Layering only combines resources definitions with the same form. For example, the DSL form `base/all.rb` will not be combined with YAML form `web/deployment.yaml`. +Note, both YAML and DSL forms support layering. They can be mixed together. ## Full Layering diff --git a/docs/_includes/sidebar.html b/docs/_includes/sidebar.html index cced7bbd..f0e84906 100644 --- a/docs/_includes/sidebar.html +++ b/docs/_includes/sidebar.html @@ -91,6 +91,7 @@

Docs

@@ -107,6 +108,14 @@

Docs

  • Multiple Resources
    • +
  • Variables + +
  • Helpers