diff --git a/Makefile b/Makefile index 959e216..efbd6b6 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ NAME=sigil ARCH=$(shell uname -m) ORG=gliderlabs -VERSION=0.3.3 +VERSION=0.4.0 build: glu build darwin,linux ./cmd diff --git a/builtin/builtin.go b/builtin/builtin.go index 7a7d1bf..76a36a4 100644 --- a/builtin/builtin.go +++ b/builtin/builtin.go @@ -10,46 +10,127 @@ import ( "path/filepath" "reflect" "strconv" + "os/exec" + "net/http" "strings" "text/template" "github.com/dustin/go-jsonpointer" "github.com/gliderlabs/sigil" "gopkg.in/yaml.v2" + "github.com/flynn/go-shlex" ) func init() { sigil.Register(template.FuncMap{ - "seq": Seq, - "default": Default, - "join": Join, - "split": Split, + // templating + "include": Include, + "default": Default, + "var": Var, + // strings "capitalize": Capitalize, "lower": Lower, "upper": Upper, "replace": Replace, "trim": Trim, - "file": File, - "json": Json, - "yaml": Yaml, - "pointer": Pointer, - "include": Include, "indent": Indent, - "var": Var, "match": Match, "render": Render, - "exists": Exists, - "dir": Dir, - "dirs": Dirs, - "files": Files, - "uniq": Uniq, - "drop": Drop, - "append": Append, "stdin": Stdin, + // filesystem + "file": File, + "exists": Exists, + "dir": Dir, + "dirs": Dirs, + "files": Files, + "text": Text, + // external + "sh": Shell, + "httpget": HttpGet, + // structured data + "pointer": Pointer, + "json": Json, + "tojson": ToJson, + "yaml": Yaml, + "toyaml": ToYaml, + "uniq": Uniq, + "drop": Drop, + "append": Append, + "seq": Seq, + "join": Join, + "joinkv": JoinKv, + "split": Split, + "splitkv": SplitKv, }) } -func Seq(i interface{}) ([]string, error) { +func Shell(in interface{}) (interface{}, error) { + in_, _, ok := sigil.String(in) + if !ok { + return "", fmt.Errorf("sh must be given a string") + } + args, err := shlex.Split(in_) + if err != nil { + return "", err + } + path, err := exec.LookPath(args[0]) + if err != nil { + return "", err + } + out, err := exec.Command(path, args[1:]...).Output() + if err != nil { + return "", err + } + return string(out), nil +} + +func HttpGet(in interface{}) (interface{}, error) { + in_, _, ok := sigil.String(in) + if !ok { + return "", fmt.Errorf("httpget must be given a string") + } + resp, err := http.Get(in_) + if err != nil { + return "", err + } + return sigil.NamedReader{resp.Body, "<"+in_+">"}, nil +} + + +func JoinKv(sep string, in interface{}) ([]interface{}, error) { + m, ok := in.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("joinkv must be given a string map of strings") + } + var elements []interface{} + for k, v := range m { + s, ok := v.(string) + if !ok { + return nil, fmt.Errorf("joinkv must be given a string map of strings") + } + elements = append(elements, strings.Join([]string{k, s}, sep)) + } + return elements, nil +} + +func SplitKv(sep string, in []interface{}) (interface{}, error) { + out := make(map[string]interface{}) + for i := range in { + v, ok := in[i].(string) + if !ok { + return nil, fmt.Errorf("joinkv must be given a string map of strings") + } + parts := strings.SplitN(v, sep, 2) + if len(parts) == 2 { + out[parts[0]] = parts[1] + } else { + out[v] = true + } + } + return out, nil +} + +func Seq(i interface{}) ([]interface{}, error) { var num int var err error var valid bool @@ -65,7 +146,7 @@ func Seq(i interface{}) ([]string, error) { if !valid { return nil, fmt.Errorf("seq must be given an integer or numeric string") } - var el []string + var el []interface{} for i, _ := range make([]bool, num) { el = append(el, strconv.Itoa(i)) } @@ -82,7 +163,7 @@ func Default(value, in interface{}) interface{} { return in } -func Join(delim string, in []interface{}) string { +func Join(delim string, in []interface{}) interface{} { var elements []string for _, el := range in { str, ok := el.(string) @@ -93,38 +174,66 @@ func Join(delim string, in []interface{}) string { return strings.Join(elements, delim) } -func Split(delim string, in string) []string { - return strings.Split(in, delim) +func Split(delim string, in interface{}) ([]interface{}, error) { + in_, _, ok := sigil.String(in) + if !ok { + return nil, fmt.Errorf("split must be given a string") + } + var elements []interface{} + for _, v := range strings.Split(in_, delim) { + elements = append(elements, v) + } + return elements, nil } -func Capitalize(in string) string { - return strings.Title(in) +func Capitalize(in interface{}) (interface{}, error) { + in_, _, ok := sigil.String(in) + if !ok { + return "", fmt.Errorf("capitalize must be given a string") + } + return strings.Title(in_), nil } -func Lower(in string) string { - return strings.ToLower(in) +func Lower(in interface{}) (interface{}, error) { + in_, _, ok := sigil.String(in) + if !ok { + return "", fmt.Errorf("lower must be given a string") + } + return strings.ToLower(in_), nil } -func Upper(in string) string { - return strings.ToUpper(in) +func Upper(in interface{}) (interface{}, error) { + in_, _, ok := sigil.String(in) + if !ok { + return "", fmt.Errorf("upper must be given a string") + } + return strings.ToUpper(in_), nil } -func Replace(old, new, in string) string { - return strings.Replace(in, old, new, -1) +func Replace(old, new string, in interface{}) (interface{}, error) { + in_, _, ok := sigil.String(in) + if !ok { + return "", fmt.Errorf("replace must be given a string") + } + return strings.Replace(in_, old, new, -1), nil } -func Trim(in string) string { - return strings.Trim(in, " \n") +func Trim(in interface{}) (interface{}, error) { + in_, _, ok := sigil.String(in) + if !ok { + return "", fmt.Errorf("trim must be given a string") + } + return strings.Trim(in_, " \n"), nil } func read(file interface{}) ([]byte, error) { - stdin, ok := file.(stdinStr) + reader, ok := file.(sigil.NamedReader) if ok { - return []byte(stdin), nil + return ioutil.ReadAll(reader) } - path, ok := file.(string) + path, _, ok := sigil.String(file) if !ok { - return []byte{}, fmt.Errorf("file must be path string or stdin") + return []byte{}, fmt.Errorf("file must be stream or path string") } filepath, err := sigil.LookPath(path) if err != nil { @@ -137,11 +246,19 @@ func read(file interface{}) ([]byte, error) { return data, nil } -func File(filename string) (string, error) { +func File(filename interface{}) (interface{}, error) { str, err := read(filename) return string(str), err } +func Text(file interface{}) (interface{}, error) { + f, err := read(file) + if err != nil { + return nil, err + } + return string(f), nil +} + func Json(file interface{}) (interface{}, error) { var obj interface{} f, err := read(file) @@ -155,6 +272,14 @@ func Json(file interface{}) (interface{}, error) { return obj, nil } +func ToJson(obj interface{}) (interface{}, error) { + data, err := json.Marshal(obj) + if err != nil { + return nil, err + } + return string(data), nil +} + func Yaml(file interface{}) (interface{}, error) { var obj interface{} f, err := read(file) @@ -168,6 +293,14 @@ func Yaml(file interface{}) (interface{}, error) { return obj, nil } +func ToYaml(obj interface{}) (interface{}, error) { + data, err := yaml.Marshal(obj) + if err != nil { + return nil, err + } + return string(data), nil +} + func Pointer(path string, in interface{}) (interface{}, error) { m := make(map[string]interface{}) switch val := in.(type) { @@ -185,7 +318,7 @@ func Pointer(path string, in interface{}) (interface{}, error) { return jsonpointer.Get(m, path), nil } -func Render(args ...interface{}) (string, error) { +func Render(args ...interface{}) (interface{}, error) { if len(args) == 0 { fmt.Errorf("render cannot be used without arguments") } @@ -199,9 +332,9 @@ func Render(args ...interface{}) (string, error) { } func render(data []byte, args []interface{}, name string) (bytes.Buffer, error) { - vars := make(map[string]string) + vars := make(map[string]interface{}) for _, arg := range args { - mv, ok := arg.(map[string]string) + mv, ok := arg.(map[string]interface{}) if ok { for k, v := range mv { vars[k] = v @@ -220,7 +353,7 @@ func render(data []byte, args []interface{}, name string) (bytes.Buffer, error) return sigil.Execute(data, vars, name) } -func Include(filename string, args ...interface{}) (string, error) { +func Include(filename string, args ...interface{}) (interface{}, error) { path, err := sigil.LookPath(filename) if err != nil { return "", err @@ -235,9 +368,13 @@ func Include(filename string, args ...interface{}) (string, error) { return render.String(), err } -func Indent(indent, in string) string { +func Indent(indent string, in interface{}) (interface{}, error) { + in_, _, ok := sigil.String(in) + if !ok { + return "", fmt.Errorf("indent must be given a string") + } var indented []string - lines := strings.Split(in, "\n") + lines := strings.Split(in_, "\n") indented = append(indented, lines[0]) if len(lines) > 1 { for _, line := range lines[1:] { @@ -248,26 +385,38 @@ func Indent(indent, in string) string { } } } - return strings.Join(indented, "\n") + return strings.Join(indented, "\n"), nil } -func Var(name string) string { +func Var(name string) interface{} { return os.Getenv(name) } -func Match(pattern string, str string) (bool, error) { +func Match(pattern string, in interface{}) (bool, error) { + str, _, ok := sigil.String(in) + if !ok { + return false, fmt.Errorf("match must be given a string") + } return path.Match(pattern, str) } -func Exists(filename string) bool { +func Exists(in interface{}) (bool, error) { + filename, _, ok := sigil.String(in) + if !ok { + return false, fmt.Errorf("exists must be given a string") + } _, err := sigil.LookPath(filename) if err != nil { - return false + return false, nil } - return true + return true, nil } -func Dir(path string) ([]interface{}, error) { +func Dir(in interface{}) ([]interface{}, error) { + path, _, ok := sigil.String(in) + if !ok { + return nil, fmt.Errorf("dir must be given a string") + } var files []interface{} dir, err := ioutil.ReadDir(path) if err != nil { @@ -279,7 +428,11 @@ func Dir(path string) ([]interface{}, error) { return files, nil } -func Dirs(path string) ([]interface{}, error) { +func Dirs(in interface{}) ([]interface{}, error) { + path, _, ok := sigil.String(in) + if !ok { + return nil, fmt.Errorf("dirs must be given a string") + } var dirs []interface{} dir, err := ioutil.ReadDir(path) if err != nil { @@ -293,7 +446,11 @@ func Dirs(path string) ([]interface{}, error) { return dirs, nil } -func Files(path string) ([]interface{}, error) { +func Files(in interface{}) ([]interface{}, error) { + path, _, ok := sigil.String(in) + if !ok { + return nil, fmt.Errorf("files must be given a string") + } var files []interface{} dir, err := ioutil.ReadDir(path) if err != nil { @@ -321,14 +478,8 @@ func Uniq(in ...[]interface{}) []interface{} { return uniq } -type stdinStr string - -func Stdin() (stdinStr, error) { - data, err := ioutil.ReadAll(os.Stdin) - if err != nil { - return "", err - } - return stdinStr(data), nil +func Stdin() (interface{}, error) { + return sigil.NamedReader{os.Stdin, ""}, nil } func Append(item interface{}, items []interface{}) []interface{} { diff --git a/circle.yml b/circle.yml index f091dd8..1e8fff3 100644 --- a/circle.yml +++ b/circle.yml @@ -4,6 +4,7 @@ machine: dependencies: pre: + - make deps - docker run --rm gliderlabs/glu | tar xC /home/ubuntu/bin - glu circleci - glu container up diff --git a/cmd/sigil.go b/cmd/sigil.go index 781122b..1f498b4 100644 --- a/cmd/sigil.go +++ b/cmd/sigil.go @@ -52,7 +52,7 @@ func main() { if os.Getenv("SIGIL_PATH") != "" { sigil.TemplatePath = strings.Split(os.Getenv("SIGIL_PATH"), ":") } - vars := make(map[string]string) + vars := make(map[string]interface{}) for _, arg := range flag.Args() { parts := strings.SplitN(arg, "=", 2) if len(parts) == 2 { diff --git a/sigil.go b/sigil.go index 3de5c74..77c9c29 100644 --- a/sigil.go +++ b/sigil.go @@ -3,6 +3,8 @@ package sigil import ( "bytes" "fmt" + "io" + "io/ioutil" "os" "path/filepath" "strings" @@ -14,10 +16,46 @@ import ( var ( TemplatePath []string PosixPreprocess bool + leftDelim string + rightDelim string ) var fnMap = template.FuncMap{} +func init() { + leftDelim = os.Getenv("SIGIL_LEFT_DELIM") + if leftDelim == "" { + leftDelim = "{{" + } + rightDelim = os.Getenv("SIGIL_RIGHT_DELIM") + if rightDelim == "" { + rightDelim = "}}" + } +} + +type NamedReader struct { + io.Reader + Name string +} + +func String(in interface{}) (string, string, bool) { + switch obj := in.(type) { + case string: + return obj, "", true + case NamedReader: + data, err := ioutil.ReadAll(obj) + if err != nil { + // TODO: better overall error/panic handling + panic(err) + } + return string(data), obj.Name, true + case fmt.Stringer: + return obj.String(), "", true + default: + return "", "", false + } +} + func Register(fm template.FuncMap) { for k, v := range fm { fnMap[k] = v @@ -55,17 +93,18 @@ func restoreEnv(env []string) { } } -func Execute(input []byte, vars map[string]string, name string) (bytes.Buffer, error) { +func Execute(input []byte, vars map[string]interface{}, name string) (bytes.Buffer, error) { var tmplVars string var err error defer restoreEnv(os.Environ()) - for k, v := range vars { - err := os.Setenv(k, v) - if err != nil { - return bytes.Buffer{}, err + for k, iv := range vars { + if v, ok := iv.(string); ok { + err := os.Setenv(k, v) + if err != nil { + return bytes.Buffer{}, err + } } - escaped := strings.Replace(v, "\"", "\\\"", -1) - tmplVars = tmplVars + fmt.Sprintf("{{ $%s := \"%s\" }}", k, escaped) + tmplVars = tmplVars + fmt.Sprintf("%s $%s := .%s %s", leftDelim, k, k, rightDelim) } inputStr := string(input) if PosixPreprocess { @@ -74,8 +113,14 @@ func Execute(input []byte, vars map[string]string, name string) (bytes.Buffer, e return bytes.Buffer{}, err } } - inputStr = strings.Replace(inputStr, "\\}}\n{{", "}}{{", -1) - tmpl, err := template.New(name).Funcs(fnMap).Parse(tmplVars + inputStr) + + inputStr = strings.Replace( + inputStr, + fmt.Sprintf("\\%s\n%s", rightDelim, leftDelim), + fmt.Sprintf("%s%s", rightDelim, leftDelim), + -1, + ) + tmpl, err := template.New(name).Funcs(fnMap).Delims(leftDelim, rightDelim).Parse(tmplVars + inputStr) if err != nil { return bytes.Buffer{}, err } diff --git a/tests/sigil.bash b/tests/sigil.bash index a81dc0e..dcf2d41 100644 --- a/tests/sigil.bash +++ b/tests/sigil.bash @@ -60,3 +60,40 @@ T_XXX() { result=$(echo 'XXX' | $SIGIL) [[ "$result" == "XXX" ]] } + +T_split_join() { + result=$(echo 'one,two,three' | $SIGIL -i '{{ stdin | split "," | join ":" }}') + [[ "$result" == "one:two:three" ]] +} + +T_splitkv_joinkv() { + result=$(echo 'one:two,three:four' | $SIGIL -i '{{ stdin | split "," | splitkv ":" | joinkv "=" | join "," }}') + [[ "$result" == "one=two,three=four" ]] +} + +T_json() { + result=$(echo '{"one": "two"}' | $SIGIL -i '{{ stdin | json | tojson }}') + [[ "$result" == "{\"one\":\"two\"}" ]] +} + +T_yaml() { + yaml="$(echo -e "one: two\nthree:\n- four\n- five")" + result="$(echo -e "$yaml" | $SIGIL -i '{{ stdin | yaml | toyaml }}')" + [[ "$result" == "$yaml" ]] +} + +T_shell() { + result="$($SIGIL -i '{{ sh "date +%m-%d-%Y" }}')" + [[ "$result" == "$(date +%m-%d-%Y)" ]] +} + +T_httpget() { + result="$($SIGIL -i '{{ httpget "https://httpbin.org/get" | json | pointer "/url" }}')" + [[ "$result" == "https://httpbin.org/get" ]] +} + +T_custom_delim() { + result="$(SIGIL_LEFT_DELIM={{{ SIGIL_RIGHT_DELIM=}}} $SIGIL -i '{{ hello {{{ $name }}} }}' name=packer)" + [[ "$result" == "{{ hello packer }}" ]] + +}