Merge pull request #7 from mhlias/tf_095_support

Tf 095 support

README.md

@@ -11,8 +11,11 @@ This tool wraps terraform execution forcing a specific structure while providing
  - Gets STS tokens and uses them for the current account.
  - Creates an S3 bucket in the current account and enables versioning.
  - Configures remote terraform state on the created S3 bucket.
+ - Creates a DynamoDB lock table and uses it to lock remote S3 state (Terraform 0.9.x only)
  - Provides management of remote git/github terraform modules using the Terrafile concept.
  - Provides plan and apply functionality with resources target support and keeps the local & remote states in sync.
+ - Support for Terraform 0.9.x and legacy mode with version autodetection
+ - Support for Terraform state environments (created if not exist already) with Terraform 0.9.x versions
 
 
 ### Setup Requirements
@@ -24,6 +27,15 @@ Configuration input required:
  - Name of your project config yaml file: This file will reside on the root directory of your project and needs to be `%name.yaml` which `%name` you specify in this stage.
  - Directory name of your terraform modules: This will be always created a level down of the root of your project and will be used by the Terrafile concept to store your project's modules. Will also be the modules source in your terraform templates.
  - Root profile, is your `$HOME/.aws/credentials` profile name that can assume roles on your AWS accounts
+ - With Terraform 0.9.x you need to include in a .tf file the following terraform block:
+
+ ```
+ terraform {
+  required_version = ">= 0.9.0"
+  backend "s3" {}
+ }
+
+ ```
 
 
 From the files mentioned above here are some examples of what their contents need to be:
@@ -96,11 +108,12 @@ The tool accepts the following parameters:
 ```
  -a Terraform Apply Plan
  -c Force reconfiguration of Tholos
+ -e Terraform state environment to use
  -o Display Terraform outputs
  -p Terraform Plan
  -s Sync remote S3 state
+ -t Terraform resources to target only, (-t resourcetype.resource resourcetype2.resource2)
  -u Fetch and update modules from remote repo
- -t Terraform resources to target only, (-t resourcetype.resource resourcetype2.resource2)
 
 ```
 

aws_helper/conn.go

@@ -8,13 +8,15 @@ import (
  "github.com/aws/aws-sdk-go/aws"
  "github.com/aws/aws-sdk-go/aws/credentials"
  "github.com/aws/aws-sdk-go/aws/session"
+ "github.com/aws/aws-sdk-go/service/dynamodb"
  "github.com/aws/aws-sdk-go/service/s3"
  "github.com/aws/aws-sdk-go/service/sts"
 )
 
 type AWSClient struct {
  stsconn *sts.STS
  S3conn *s3.S3
+ Dynconn *dynamodb.DynamoDB
  region string
 }
 
@@ -108,6 +110,7 @@ func (c *Config) Connect() interface{} {
 
  log.Println("[INFO] Initializing S3 Connection")
  client.S3conn = s3.New(sess)
+ client.Dynconn = dynamodb.New(sess)
 
  return &client
 
@@ -127,6 +130,7 @@ func (c *Config) assumeConnect(sts *sts.AssumeRoleOutput) interface{} {
 
  log.Println("[INFO] Initializing S3 Connection")
  client.S3conn = s3.New(sess)
+ client.Dynconn = dynamodb.New(sess)
 
  return &client
 

main.go

@@ -57,6 +57,7 @@ func main() {
  modulesPtr := flag.Bool("u", false, "Fetch and update modules from remote repo")
  outputsPtr := flag.Bool("o", false, "Display Terraform outputs")
  configPtr := flag.Bool("c", false, "Force reconfiguration of Tholos")
+ envPtr := flag.String("e", "", "Terraform state environment to use")
  flag.Var(&targetsTF, "t", "Terraform resources to target only, (-t resourcetype.resource resourcetype2.resource2)")
 
  flag.Parse()
@@ -128,16 +129,24 @@ func main() {
 
  if ver_int > 8 {
  tf_legacy = false
+ if len(*envPtr) > 0 {
+ log.Printf("[INFO] Will be working on STATE ENVIRONMENT: %s", *envPtr)
+ // Sleep for 5 seconds let the user stop execution if wrong state environment
+ time.Sleep(5 * time.Second)
+ }
  } else {
  log.Printf("[WARN] Running in legacy mode, current Terraform version: %s, install >=0.9.x for full features.\n", tf_version)
  }
 
  state_config := &tf_helper.Config{Bucket_name: fmt.Sprintf("%s-%s-%s-tfstate", project_config.Project, project_config.account, project_config.environment),
  State_filename: fmt.Sprintf("%s-%s-%s.tfstate", project_config.Project, project_config.account, project_config.environment),
+ Lock_table: fmt.Sprintf("%s-%s-%s-locktable", project_config.Project, project_config.account, project_config.environment),
  Versioning: true,
+ Region: project_config.Region,
  Encrypt_s3_state: project_config.Encrypt_s3_state,
  TargetsTF: targetsTF,
  TFlegacy: tf_legacy,
+ TFenv: *envPtr,
  }
 
  var tf_parallelism int16 = 10
@@ -177,6 +186,30 @@ func main() {
 
  }
 
+ if *syncPtr || *planPtr || *applyPtr {
+
+ if !state_config.TFlegacy {
+
+ if len(*envPtr) > 0 {
+ state_config.Switch_env()
+ }
+
+ for j := 1; j <= retries; j++ {
+
+ if !state_config.Create_locktable(client) {
+ log.Printf("[WARN] DynamoDB table %s failed to be created. Retrying.\n", state_config.Lock_table)
+ } else {
+ log.Printf("[INFO] DynamoDB table %s created.\n", state_config.Lock_table)
+ break
+ }
+
+ time.Sleep(time.Duration(j) * time.Second)
+
+ }
+ }
+
+ }
+
  if *syncPtr {
 
  bucket_created := false

tf_helper/state.go

@@ -1,21 +1,28 @@
 package tf_helper
 
 import (
+ "bytes"
  "fmt"
  "log"
+ "os/exec"
+ "strings"
 
  "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/service/dynamodb"
  "github.com/aws/aws-sdk-go/service/s3"
  "github.com/mhlias/tholos/aws_helper"
 )
 
 type Config struct {
  Bucket_name string
  State_filename string
+ Lock_table string
  Encrypt_s3_state bool
  Versioning bool
  TargetsTF []string
  TFlegacy bool
+ TFenv string
+ Region string
 }
 
 func (c *Config) Create_bucket(client interface{}) bool {
@@ -99,7 +106,108 @@ func (c *Config) enable_versioning(client interface{}) bool {
 
 }
 
-func (c *Config) setup_lock_DB() {
+func (c *Config) Create_locktable(client interface{}) bool {
+
+ params := &dynamodb.ListTablesInput{}
+
+ resp, err := client.(*aws_helper.AWSClient).Dynconn.ListTables(params)
+
+ if err != nil {
+ fmt.Println(err.Error())
+ return false
+ }
+
+ for _, dt := range resp.TableNames {
+ if *dt == c.Lock_table {
+ return true
+ }
+ }
+
+ params2 := &dynamodb.CreateTableInput{
+ AttributeDefinitions: []*dynamodb.AttributeDefinition{
+ {
+ AttributeName: aws.String("LockID"),
+ AttributeType: aws.String("S"),
+ },
+ },
+ KeySchema: []*dynamodb.KeySchemaElement{
+ {
+ AttributeName: aws.String("LockID"),
+ KeyType: aws.String("HASH"),
+ },
+ },
+ ProvisionedThroughput: &dynamodb.ProvisionedThroughput{
+ ReadCapacityUnits: aws.Int64(5),
+ WriteCapacityUnits: aws.Int64(5),
+ },
+ TableName: aws.String(c.Lock_table),
+ }
+ _, err2 := client.(*aws_helper.AWSClient).Dynconn.CreateTable(params2)
+
+ if err2 != nil {
+ fmt.Println(err2.Error())
+ return false
+ }
+
+ return true
+
+}
+
+func (c *Config) Switch_env() {
+
+ var args []string
+ env_exists := false
+
+ cmdList := exec.Command("terraform", "env", "list")
+ var out bytes.Buffer
+ cmdList.Stdout = &out
+ err := cmdList.Run()
+ if err != nil {
+ log.Fatal("Failed to get Terraform state environments list:", err)
+ }
+
+ out_str := out.String()
+
+ tfenvs := strings.Split(out_str, "\n")
+
+ for _, e := range tfenvs {
+ if c.TFenv == strings.Trim(e, "* ") {
+ env_exists = true
+ break
+ }
+ }
+
+ if !env_exists {
+
+ cmdCreate := "terraform"
+
+ args = []string{
+ "env",
+ "new",
+ c.TFenv,
+ }
+
+ if ExecCmd(cmdCreate, args) {
+ log.Printf("[INFO] Terraform state environment %s created.", c.TFenv)
+ } else {
+ log.Fatal("[ERROR] Failed create Terraform state environment. Aborting.\n")
+ }
+
+ }
+
+ cmdSelect := "terraform"
+
+ args = []string{
+ "env",
+ "select",
+ c.TFenv,
+ }
+
+ if ExecCmd(cmdSelect, args) {
+ log.Printf("[INFO] Terraform state environment %s selected.", c.TFenv)
+ } else {
+ log.Fatal("[ERROR] Failed select Terraform state environment. Aborting.\n")
+ }
 
 }
 
@@ -109,18 +217,36 @@ func (c *Config) Setup_remote_state() {
 
  cmdName := "terraform"
 
- args := []string{"remote",
- "config",
- "-backend=S3",
- fmt.Sprintf("-backend-config=bucket=%s", c.Bucket_name),
- fmt.Sprintf("-backend-config=key=%s", c.State_filename),
- fmt.Sprintf("-backend-config=encrypt=%t", c.Encrypt_s3_state),
+ var args []string
+
+ if c.TFlegacy {
+
+ args = []string{"remote",
+ "config",
+ "-backend=S3",
+ fmt.Sprintf("-backend-config=bucket=%s", c.Bucket_name),
+ fmt.Sprintf("-backend-config=key=%s", c.State_filename),
+ fmt.Sprintf("-backend-config=encrypt=%t", c.Encrypt_s3_state),
+ }
+
+ } else {
+
+ args = []string{"init",
+ "-backend=true",
+ fmt.Sprintf("-backend-config=bucket=%s", c.Bucket_name),
+ fmt.Sprintf("-backend-config=key=%s", c.State_filename),
+ fmt.Sprintf("-backend-config=region=%s", c.Region),
+ fmt.Sprintf("-backend-config=lock_table=%s", c.Lock_table),
+ fmt.Sprintf("-backend-config=encrypt=%t", c.Encrypt_s3_state),
+ "-force-copy",
+ }
+
  }
 
  if ExecCmd(cmdName, args) {
  log.Println("[INFO] Remote State was set up successfully.")
  } else {
- log.Fatal("[INFO] Remote state failed to be set up. Aborting.\n")
+ log.Fatal("[ERROR] Remote state failed to be set up. Aborting.\n")
  }
 
 }