feat: remote state files (#648)

This change enhances helmfile to accept terraform-module-like URLs in nested state files a.k.a sub-helmfiles.

```yaml
helmfiles:
- # Terraform-module-like URL for importing a remote directory and use a file in it as a nested-state file
  # The nested-state file is locally checked-out along with the remote directory containing it.
  # Therefore all the local paths in the file are resolved relative to the file
  path: git::https://github.com/cloudposse/helmfiles.git@releases/kiam.yaml?ref=0.40.0
```

The URL isn't equivalent to terraform module sources. The difference is that we use `@` to distinguish between (1) the path to the repository and directory containing the state file and (2) the path to the state file being loaded. This distinction provides us enough fleibiity to instruct helmfile to check-out necessary and sufficient directory to make the state file works.

Under the hood, it uses [hashicorp/go-getter](https://github.com/hashicorp/go-getter), that is used for [terraform module sources](https://www.terraform.io/docs/modules/sources.html) as well.

Only the git provider without authentication like git-credentials helper is tested. But theoretically any go-getter providers should work. Please feel free to test the provider of your choice and contribute documentation or instruction to use it :)

Resolves #347

README.md

@@ -184,6 +184,10 @@ helmfiles:
 - # All the nested state files under `helmfiles:` is processed in the order of definition.
   # So it can be used for preparation for your main `releases`. An example would be creating CRDs required by `reelases` in the parent state file. 
   path: path/to/mycrd.helmfile.yaml
+- # Terraform-module-like URL for importing a remote directory and use a file in it as a nested-state file
+  # The nested-state file is locally checked-out along with the remote directory containing it.
+  # Therefore all the local paths in the file are resolved relative to the file
+  path: git::https://github.com/cloudposse/helmfiles.git@releases/kiam.yaml?ref=0.40.0
 
 #
 # Advanced Configuration: Environments

go.mod

@@ -8,16 +8,15 @@ require (
 	github.com/aokoli/goutils v1.0.1 // indirect
 	github.com/google/go-cmp v0.3.0
 	github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c // indirect
+	github.com/hashicorp/go-getter v1.3.0
 	github.com/huandu/xstrings v1.0.0 // indirect
 	github.com/imdario/mergo v0.3.6
-	github.com/mattn/go-runewidth v0.0.4 // indirect
 	github.com/pkg/errors v0.8.1 // indirect
 	github.com/tatsushid/go-prettytable v0.0.0-20141013043238-ed2d14c29939
 	github.com/urfave/cli v0.0.0-20160620154522-6011f165dc28
 	go.uber.org/atomic v1.3.2 // indirect
 	go.uber.org/multierr v1.1.0 // indirect
 	go.uber.org/zap v1.8.0
-	golang.org/x/crypto v0.0.0-20180403160946-b2aa35443fbc // indirect
 	gopkg.in/yaml.v2 v2.2.1
 	gotest.tools v2.2.0+incompatible
 )

pkg/app/app.go

@@ -3,6 +3,7 @@ package app
 import (
 	"fmt"
 	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/roboll/helmfile/pkg/remote"
 	"github.com/roboll/helmfile/pkg/state"
 	"io/ioutil"
 	"log"
@@ -42,6 +43,8 @@ type App struct {
 
 	getwd func() (string, error)
 	chdir func(string) error
+
+	remote *remote.Remote
 }
 
 func New(conf ConfigProvider) *App {
@@ -302,6 +305,14 @@ func (a *App) visitStates(fileOrDir string, defOpts LoadOpts, converge func(*sta
 				} else {
 					optsForNestedState.Selectors = m.Selectors
 				}
+
+				path, err := a.remote.Locate(m.Path)
+				if err != nil {
+					return appError(fmt.Sprintf("in .helmfiles[%d]", i), err)
+				}
+
+				m.Path = path
+
 				if err := a.visitStates(m.Path, optsForNestedState, converge); err != nil {
 					switch err.(type) {
 					case *NoMatchingHelmfileError:
@@ -373,6 +384,26 @@ func (a *App) VisitDesiredStatesWithReleasesFiltered(fileOrDir string, converge
 		opts.Environment.OverrideValues = envvals
 	}
 
+	var dir string
+	if a.directoryExistsAt(fileOrDir) {
+		dir = fileOrDir
+	} else {
+		dir = filepath.Dir(fileOrDir)
+	}
+
+	getter := &remote.GoGetter{Logger: a.Logger}
+
+	remote := &remote.Remote{
+		Logger:     a.Logger,
+		Home:       dir,
+		Getter:     getter,
+		ReadFile:   a.readFile,
+		DirExists:  a.directoryExistsAt,
+		FileExists: a.fileExistsAt,
+	}
+
+	a.remote = remote
+
 	err := a.visitStates(fileOrDir, opts, func(st *state.HelmState, helm helmexec.Interface) (bool, []error) {
 		if len(st.Selectors) > 0 {
 			err := st.FilterReleases()
@@ -388,6 +419,7 @@ func (a *App) VisitDesiredStatesWithReleasesFiltered(fileOrDir string, converge
 		type Key struct {
 			TillerNamespace, Name string
 		}
+
 		releaseNameCounts := map[Key]int{}
 		for _, r := range st.Releases {
 			tillerNamespace := st.HelmDefaults.TillerNamespace

pkg/app/app_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/roboll/helmfile/pkg/helmexec"
 	"github.com/roboll/helmfile/pkg/state"
+	"github.com/roboll/helmfile/pkg/testhelper"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -13,11 +14,11 @@ import (
 )
 
 func appWithFs(app *App, files map[string]string) *App {
-	fs := state.NewTestFs(files)
+	fs := testhelper.NewTestFs(files)
 	return injectFs(app, fs)
 }
 
-func injectFs(app *App, fs *state.TestFs) *App {
+func injectFs(app *App, fs *testhelper.TestFs) *App {
 	app.readFile = fs.ReadFile
 	app.glob = fs.Glob
 	app.abs = fs.Abs
@@ -52,7 +53,7 @@ releases:
   chart: stable/grafana
 `,
 	}
-	fs := state.NewTestFs(files)
+	fs := testhelper.NewTestFs(files)
 	fs.GlobFixtures["/path/to/helmfile.d/a*.yaml"] = []string{"/path/to/helmfile.d/a2.yaml", "/path/to/helmfile.d/a1.yaml"}
 	app := &App{
 		KubeContext: "default",
@@ -98,7 +99,7 @@ BAR: 2
 BAZ: 4
 `,
 	}
-	fs := state.NewTestFs(files)
+	fs := testhelper.NewTestFs(files)
 	fs.GlobFixtures["/path/to/env.*.yaml"] = []string{"/path/to/env.2.yaml", "/path/to/env.1.yaml"}
 	app := &App{
 		KubeContext: "default",
@@ -137,7 +138,7 @@ releases:
   chart: stable/zipkin
 `,
 	}
-	fs := state.NewTestFs(files)
+	fs := testhelper.NewTestFs(files)
 	app := &App{
 		KubeContext: "default",
 		Logger:      helmexec.NewLogger(os.Stderr, "debug"),
@@ -190,7 +191,7 @@ releases:
   chart: stable/zipkin
 `, testcase.handler, testcase.filePattern),
 			}
-			fs := state.NewTestFs(files)
+			fs := testhelper.NewTestFs(files)
 			app := &App{
 				KubeContext: "default",
 				Logger:      helmexec.NewLogger(os.Stderr, "debug"),
@@ -251,7 +252,7 @@ releases:
 	}
 
 	for _, testcase := range testcases {
-		fs := state.NewTestFs(files)
+		fs := testhelper.NewTestFs(files)
 		fs.GlobFixtures["/path/to/helmfile.d/a*.yaml"] = []string{"/path/to/helmfile.d/a2.yaml", "/path/to/helmfile.d/a1.yaml"}
 		app := &App{
 			KubeContext: "default",
@@ -1077,7 +1078,7 @@ releases:
     stage: post
   <<: *default
 `
-	testFs := state.NewTestFs(map[string]string{
+	testFs := testhelper.NewTestFs(map[string]string{
 		yamlFile: yamlContent,
 		"/path/to/base.yaml": `environments:
   default:
@@ -1158,7 +1159,7 @@ releases:
     stage: post
   <<: *default
 `
-	testFs := state.NewTestFs(map[string]string{
+	testFs := testhelper.NewTestFs(map[string]string{
 		yamlFile: yamlContent,
 		"/path/to/base.yaml": `environments:
   default:
@@ -1235,7 +1236,7 @@ releases:
 - name: myrelease0
   chart: mychart0
 `
-	testFs := state.NewTestFs(map[string]string{
+	testFs := testhelper.NewTestFs(map[string]string{
 		yamlFile: yamlContent,
 		"/path/to/base.yaml": `environments:
   default:
@@ -1295,7 +1296,7 @@ releases:
 - name: myrelease0
   chart: mychart0
 `
-	testFs := state.NewTestFs(map[string]string{
+	testFs := testhelper.NewTestFs(map[string]string{
 		yamlFile: yamlContent,
 		"/path/to/base.yaml": `environments:
   default:
@@ -1372,7 +1373,7 @@ releases:
     stage: post
   <<: *default
 `
-	testFs := state.NewTestFs(map[string]string{
+	testFs := testhelper.NewTestFs(map[string]string{
 		yamlFile: yamlContent,
 		"/path/to/base.yaml": `environments:
   test:
@@ -1458,7 +1459,7 @@ releases:
   chart: mychart3
   <<: *default
 `
-	testFs := state.NewTestFs(map[string]string{
+	testFs := testhelper.NewTestFs(map[string]string{
 		yamlFile: yamlContent,
 		"/path/to/yaml/templates.yaml": `templates:
   default: &default
@@ -1515,7 +1516,7 @@ releases:
 - name: {{ .Environment.Values.foo | quote }}
   chart: {{ .Environment.Values.bar | quote }}
 `
-	testFs := state.NewTestFs(map[string]string{
+	testFs := testhelper.NewTestFs(map[string]string{
 		statePath:         stateContent,
 		"/path/to/1.yaml": `bar: ["bar"]`,
 		"/path/to/2.yaml": `bar: ["BAR"]`,
@@ -1568,7 +1569,7 @@ releases:
 - name: {{ .Environment.Values.foo | quote }}
   chart: {{ .Environment.Values.bar | quote }}
 `
-		testFs := state.NewTestFs(map[string]string{
+		testFs := testhelper.NewTestFs(map[string]string{
 			statePath:         stateContent,
 			"/path/to/1.yaml": `bar: ["bar"]`,
 			"/path/to/2.yaml": `bar: ["BAR"]`,
@@ -1653,7 +1654,7 @@ releases:
 		tc := testcases[i]
 		statePath := "/path/to/helmfile.yaml"
 		stateContent := fmt.Sprintf(tc.state, tc.expr)
-		testFs := state.NewTestFs(map[string]string{
+		testFs := testhelper.NewTestFs(map[string]string{
 			statePath:         stateContent,
 			"/path/to/1.yaml": `foo: FOO`,
 			"/path/to/2.yaml": `bar: { "baz": "BAZ" }

pkg/app/two_pass_renderer_test.go

@@ -3,15 +3,16 @@ package app
 import (
 	"github.com/roboll/helmfile/pkg/helmexec"
 	"github.com/roboll/helmfile/pkg/state"
+	"github.com/roboll/helmfile/pkg/testhelper"
 	"os"
 	"strings"
 	"testing"
 
 	"gopkg.in/yaml.v2"
 )
 
-func makeLoader(files map[string]string, env string) (*desiredStateLoader, *state.TestFs) {
-	testfs := state.NewTestFs(files)
+func makeLoader(files map[string]string, env string) (*desiredStateLoader, *testhelper.TestFs) {
+	testfs := testhelper.NewTestFs(files)
 	return &desiredStateLoader{
 		env:        env,
 		namespace:  "namespace",