Merge pull request #110 from AirHelp/feature/DOT-665-templates-with-k…

…ey-value-interpolations

[DOT-665] Add ability to use variables in templates

.gitignore

@@ -2,3 +2,4 @@ treasury
 pkg
 test/output
 vendor/
+tmp

README.md

@@ -5,32 +5,36 @@
 Treasury is a very simple tool for managing secrets. It uses Amazon S3 or SSM ([Systems Manager Parameter Store](https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-paramstore.html)) service to store secrets. By default, Treasury uses SSM as a backend. The secrets are encrypted before saving them on disks in Amazon data centers and decrypted when being read. Treasury uses Server-Side Encryption with AWS KMS-Managed Keys ([SSE-KMS](http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html)).
 
 - [treasury](#treasury)
- - [Architecture](#architecture)
- - [Command Line interface (CLI)](#command-line-interface-cli)
- - [Configuration](#configuration)
- - [AWS Credentials](#aws-credentials)
- - [SSM store configuration](#ssm-store-configuration)
- - [S3 store configuration](#s3-store-configuration)
- - [AWS Region configuration](#aws-region-configuration)
- - [Installation](#installation)
- - [CLI Usage](#cli-usage)
- - [Write secret](#write-secret)
- - [Read secret](#read-secret)
- - [List secrets](#list-secrets)
- - [Import secrets](#import-secrets)
- - [Export secrets](#export-secrets)
- - [Teamplate usage](#teamplate-usage)
- - [read](#read)
- - [export](#export)
- - [exportMap](#exportmap)
- - [Setting up the infrastructure](#setting-up-the-infrastructure)
- - [IAM Policy for S3 store](#iam-policy-for-s3-store)
- - [IAM Policy for SSM Store](#iam-policy-for-ssm-store)
- - [Treasury as a user vault](#treasury-as-a-user-vault)
- - [Go Client](#go-client)
- - [Development](#development)
- - [Build for development](#build-for-development)
- - [Tests](#tests)
+ - [Architecture](#architecture)
+ - [Command Line interface (CLI)](#command-line-interface-cli)
+ - [Configuration](#configuration)
+ - [AWS Credentials](#aws-credentials)
+ - [SSM store configuration](#ssm-store-configuration)
+ - [S3 store configuration](#s3-store-configuration)
+ - [AWS Region configuration](#aws-region-configuration)
+ - [Installation](#installation)
+ - [CLI Usage](#cli-usage)
+ - [Write secret](#write-secret)
+ - [Write file content](#write-file-content)
+ - [Read secret](#read-secret)
+ - [List secrets](#list-secrets)
+ - [Import secrets](#import-secrets)
+ - [Export secrets](#export-secrets)
+ - [Teamplate usage](#teamplate-usage)
+ - [Template usage with string append to secret value](#template-usage-with-string-append-to-secret-value)
+ - [Template usage with variables interpolation](#template-usage-with-variables-interpolation)
+ - [read](#read)
+ - [readFromEnv](#readfromenv)
+ - [export](#export)
+ - [exportMap](#exportmap)
+ - [Setting up the infrastructure](#setting-up-the-infrastructure)
+ - [IAM Policy for S3 store](#iam-policy-for-s3-store)
+ - [IAM Policy for SSM Store](#iam-policy-for-ssm-store)
+ - [Treasury as a user vault](#treasury-as-a-user-vault)
+ - [Go Client](#go-client)
+ - [Development](#development)
+ - [Build for development](#build-for-development)
+ - [Tests](#tests)
 
 ## Architecture
 
@@ -193,20 +197,47 @@ treasury template --src /tmp/template.tpl --dst /tmp/result --append key1:v2
 ```
 This command ends up with output file where the value of variable key1 has a string "v2" appended.
 
+#### Template usage with variables interpolation
+
+Example template:
+
+```
+APPNAME={{ .AppName }}
+API_PASSWORD={{ .ApiPassword }}
+```
+
+```bash
+treasury template --src /tmp/template.tpl --dst /tmp/result.env --env AppName=test,ApiPassword=qwerty12345
+```
+
 Supported actions:
 
-#### read 
-Returns single value for given key
+#### read
+**DEPRECATED (please use [readFromEnv](#readfromenv))** Returns single value for given key
 
 ```
 {{ read "ENVIRONMENT/APPLICATION/SECRET_NAME" }}
 ```
 
 Example:
+
 ```
 COCKPIT_API_PASSWORD={{ read "production/cockpit/cockpit_api_password" }}
 ```
 
+#### readFromEnv
+
+Returns single value for given key in specified environment
+
+```
+{{ readFromEnv "ENVIRONMENT" "APPLICATION/SECRET_NAME" }}
+
+# example using interpolation:
+
+{{ readFromEnv .Environment "app/API_PASSWORD" }}
+```
+
+
 #### export
 Returns all values for a given path in `key=value` format
 

client/read.go

@@ -1,6 +1,8 @@
 package client
 
 import (
+ "fmt"
+
  "github.com/AirHelp/treasury/types"
  "github.com/AirHelp/treasury/utils"
 )
@@ -34,6 +36,10 @@ func (c *Client) ReadValue(key string) (string, error) {
  return secret.Value, nil
 }
 
+func (c *Client) ReadFromEnv(env, key string) (string, error) {
+ return c.ReadValue(fmt.Sprintf("%s/%s", env, key))
+}
+
 // ReadGroup returns list of secrets for given key prefix
 func (c *Client) ReadGroup(keyPrefix string) ([]*Secret, error) {
  if err := utils.ValidateInputKeyPattern(keyPrefix); err != nil {

client/read_test.go

@@ -70,6 +70,52 @@ func TestReadValue(t *testing.T) {
  }
 }
 
+func TestReadFromEnv(t *testing.T) {
+ dummyClientOptions := &client.Options{
+ Backend: &test.MockBackendClient{},
+ S3BucketName: "fake_s3_bucket",
+ }
+ treasury, err := client.New(dummyClientOptions)
+ if err != nil {
+ t.Error(err)
+ }
+
+ tests := []struct {
+ name string
+ env string
+ key string
+ want string
+ wantErr bool
+ }{
+ {
+ name: "test valid key",
+ env: "test",
+ key: test.Key1NoEnv,
+ want: test.KeyValueMap[test.Key1],
+ wantErr: false,
+ },
+ {
+ name: "test non existing key",
+ env: "test",
+ key: "nonExistingKey",
+ want: "",
+ wantErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := treasury.ReadFromEnv(tt.env, tt.key)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Client.ReadFromEnv() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ if got != tt.want {
+ t.Errorf("Client.ReadFromEnv() = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
+
 func TestReadGroup(t *testing.T) {
  dummyClientOptions := &client.Options{
  Backend: &test.MockBackendClient{},

client/template.go

@@ -24,12 +24,13 @@ func readTemplate(filePath string) (string, error) {
 }
 
 // Render template
-func (c *Client) renderTemplate(templateText string, appendMap map[string]string) (templateResultBuffer bytes.Buffer, err error) {
+func (c *Client) renderTemplate(templateText string, appendMap, envMap map[string]string) (templateResultBuffer bytes.Buffer, err error) {
  // Create a FuncMap with which to register the function.
  funcMap := template.FuncMap{
  // The name "read" is what the function will be called in the template text.
- "read": c.ReadValue,
- "exportMap": c.ExportMap,
+ "read": c.ReadValue,
+ "readFromEnv": c.ReadFromEnv,
+ "exportMap": c.ExportMap,
  "export": func(key string) (string, error) {
  return c.ExportToTemplate(key, appendMap)
  },
@@ -40,7 +41,7 @@ func (c *Client) renderTemplate(templateText string, appendMap map[string]string
  return
  }
  // Run the template.
- err = tmpl.Execute(&templateResultBuffer, nil)
+ err = tmpl.Execute(&templateResultBuffer, envMap)
  return
 }
 
@@ -76,12 +77,12 @@ func writeTemplateResults(destinationFilePath string, templateResultBuffer bytes
 }
 
 // Template generates a file with secrets from given template
-func (c *Client) Template(sourceFilePath, destinationFilePath string, perms os.FileMode, appendMap map[string]string) error {
+func (c *Client) Template(sourceFilePath, destinationFilePath string, perms os.FileMode, appendMap, envMap map[string]string) error {
  templateText, err := readTemplate(sourceFilePath)
  if err != nil {
  return err
  }
- templateResultBuffer, err := c.renderTemplate(templateText, appendMap)
+ templateResultBuffer, err := c.renderTemplate(templateText, appendMap, envMap)
  if err != nil {
  return err
  }

client/template_test.go

@@ -23,9 +23,16 @@ func TestTemplate(t *testing.T) {
  if err != nil {
  t.Error(err)
  }
- if err := treasury.Template(templateTestSourceFile, templateTestDestinationFile, 0, map[string]string{}); err != nil {
+
+ envMap := map[string]string{
+ "Environment": "test",
+ "Name": "some_testing_template",
+ }
+
+ if err := treasury.Template(templateTestSourceFile, templateTestDestinationFile, 0, map[string]string{}, envMap); err != nil {
  t.Error("Could not generate secret file from template. Error: ", err.Error())
  }
+
  _, err = os.Stat(templateTestDestinationParentDir)
  if err != nil {
  t.Error("Destination directory does not exist. Error: ", err.Error())

cmd/template.go

@@ -3,10 +3,11 @@ package cmd
 import (
  "errors"
  "fmt"
- "github.com/AirHelp/treasury/client"
- "github.com/spf13/cobra"
  "os"
  "strings"
+
+ "github.com/AirHelp/treasury/client"
+ "github.com/spf13/cobra"
 )
 
 const (
@@ -17,6 +18,8 @@ const (
  templateCommandDestinationFileArgument = "dst"
  templateCommandPermissionFileArgument = "perms"
  templateCommandAppendArgument = "append"
+ templateEnvKeyValueArgument = "env"
+ templateEnvKeyValueArgumentShort = "e"
 )
 
 var (
@@ -35,6 +38,7 @@ func init() {
  templateCmd.PersistentFlags().String(templateCommandDestinationFileArgument, "", "destination file path")
  templateCmd.PersistentFlags().Int(templateCommandPermissionFileArgument, 0, "destination file permission, e.g.: 0644")
  templateCmd.PersistentFlags().StringArray(templateCommandAppendArgument, []string{}, "variable suffix, e.g: --append \"DATABASE_URL:?pool=10\"")
+ templateCmd.PersistentFlags().StringToStringP(templateEnvKeyValueArgument, templateEnvKeyValueArgumentShort, map[string]string{}, "key-value parameters for template, e.g. -e Environment=staging,AppName=someapp")
 }
 
 func template(cmd *cobra.Command, args []string) error {
@@ -74,6 +78,11 @@ func template(cmd *cobra.Command, args []string) error {
  }
  }
 
+ envMap, err := cmd.Flags().GetStringToString(templateEnvKeyValueArgument)
+ if err != nil {
+ return err
+ }
+
  treasury, err := client.New(&client.Options{
  Region: s3Region,
  S3BucketName: treasuryS3,
@@ -82,7 +91,7 @@ func template(cmd *cobra.Command, args []string) error {
  if err != nil {
  return err
  }
- if err := treasury.Template(sourceFilePath, destinationFilePath, os.FileMode(perms), appendMap); err != nil {
+ if err := treasury.Template(sourceFilePath, destinationFilePath, os.FileMode(perms), appendMap, envMap); err != nil {
  return err
  }
  fmt.Println(templateSuccessMsg)

test/backend/test.go

@@ -10,21 +10,29 @@ import (
 )
 
 const (
- Key1 = "test/webapp/cocpit_api_pass"
- ShortKey1 = "cocpit_api_pass"
+ Key1 = "test/webapp/cockpit_api_pass"
+ Key1NoEnv = "webapp/cockpit_api_pass"
+ ShortKey1 = "cockpit_api_pass"
  Key2 = "test/webapp/user_api_pass"
+ Key2NoEnv = "webapp/user_api_pass"
  ShortKey2 = "user_api_pass"
  Key3 = "test/cockpit/user_api_pass"
+ Key3NoEnv = "cockpit/user_api_pass"
  ShortKey3 = "user_api_pass"
  Key4 = "test/webapp/some_key"
+ Key4NoEnv = "webapp/some_key"
  ShortKey4 = "some_key"
  Key5 = "test/airmail/DATABASE_URL"
+ Key5NoEnv = "airmail/DATABASE_URL"
  ShortKey5 = "DATABASE_URL"
  Key6 = "test/airmail/user_api_pass"
+ Key6NoEnv = "airmail/user_api_pass"
  ShortKey6 = "user_api_pass"
  Key7 = "test/aircom/TWILIO_AUTH_TOKEN"
+ Key7NoEnv = "aircom/TWILIO_AUTH_TOKEN"
  ShortKey7 = "TWILIO_AUTH_TOKEN"
  Key8 = "test/aircom/NEW_RELIC_LICENSE_KEY"
+ Key8NoEnv = "aircom/NEW_RELIC_LICENSE_KEY"
  ShortKey8 = "NEW_RELIC_LICENSE_KEY"
 )
 

test/bats/tests.bats

@@ -128,7 +128,7 @@ invalid_aws_region=us-west-1
  [ $status -eq 0 ]
  run grep "key1=secret1treasury" test/output/bats-output.secret
  [ $status -eq 0 ]
-} 
+}
 
 @test "template-and-var-append-multiple-variables" {
  run $treasury template --src test/resources/bats-source.secret.tpl --dst test/output/bats-output.secret --append 'key1:treasury' --append 'key2:?pool=20'
@@ -139,6 +139,30 @@ invalid_aws_region=us-west-1
  [ $status -eq 0 ]
 }
 
+@test "template-and-var-interpolation-multiple" {
+ run $treasury template --src test/resources/bats-source-interpolation.secret.tpl --dst test/output/bats-output.secret -e Name=someapp,Environment=development
+ [ $status -eq 0 ]
+ run grep "APPLICATION_SECRET_KEY=secret2" test/output/bats-output.secret
+ [ $status -eq 0 ]
+ run grep "NAME=someapp" test/output/bats-output.secret
+ [ $status -eq 0 ]
+}
+
+@test "template-and-var-interpolation-multiple-alternate-syntax" {
+ run $treasury template --src test/resources/bats-source-interpolation.secret.tpl --dst test/output/bats-output.secret -e Name=someapp -e Environment=development
+ [ $status -eq 0 ]
+ run grep "APPLICATION_SECRET_KEY=secret2" test/output/bats-output.secret
+ [ $status -eq 0 ]
+ run grep "NAME=someapp" test/output/bats-output.secret
+ [ $status -eq 0 ]
+}
+
+@test "template-and-var-interpolation-variable-not-provided" {
+ run $treasury template --src test/resources/bats-source-interpolation.secret.tpl --dst test/output/bats-output.secret -e Name=someapp
+ [ $status -eq 255 ]
+ [[ ${lines[0]} =~ "Error" ]]
+}
+
 @test "template-and-var-append-bad-input" {
  run $treasury template --src test/resources/bats-source.secret.tpl --dst test/output/bats-output.secret --append 'key1::treasury'
  [ $status -eq 0 ]
@@ -155,7 +179,7 @@ invalid_aws_region=us-west-1
 @test "write file content to treasury key" {
  run $treasury write development/treasury/key5 test/resources/test_file --file
  [ $status -eq 0 ]
- run $treasury read development/treasury/key5 
+ run $treasury read development/treasury/key5
  [[ ${lines[0]} =~ "H4sIAAAAAAAA/yopSk0sLi2q5OICBAAA///FZR9LCgAAAA==" ]]
 }
 

test/resources/bats-source-interpolation.secret.tpl

@@ -0,0 +1,9 @@
+APPLICATION_SECRET_KEY={{ readFromEnv .Environment "treasury/key2" }}
+NAME={{ .Name }}
+
+# export secrets in flexible way
+{{ range $key, $value := exportMap "development/treasury/" }}
+{{ $key }}={{ $value }}{{ end }}
+
+# export secrets as key=value
+{{ export "development/treasury/" }}

test/resources/import.env.test

@@ -1,2 +1,2 @@
-cocpit_api_pass=value@$!#A&*()+-1=
+cockpit_api_pass=value@$!#A&*()+-1=
 user_api_pass=value2