Add external plugin

README.md

@@ -1104,6 +1104,58 @@ local secretsmanager_arn = std.native('secretsmanager_arn');
 }
 ```
 
+### Execute external commands
+
+The `external` plugin introduces functions to execute any external commands.
+
+For example, `jq -n "{ Now: now | todateiso8601" }` returns the current date in ISO8601 format as a JSON object.
+
+```console
+$ jq -n "{ Now: now | todateiso8601 }"
+{
+  "Now": "2024-10-25T16:13:22Z"
+}
+```
+
+You can use this command as a template function in the definition files.
+
+First, define the plugin in the configuration file.
+
+ecspresso.yml
+```yaml
+plugins:
+  - name: external
+    config:
+      name: jq
+      command: ["jq", "-n"]
+      num_args: 1
+      timeout: 5
+```
+
+The `config` section defines the following parameters:
+
+- `name`: template function name
+- `command`: external command and arguments (array)
+  - The command must return a JSON string or any strings to stdout.
+- `num_args`: number of arguments (optional, default 0)
+- `parser`: parser type "json" or "string" (optional, default "json")
+- `timeout`: command execution timeout seconds (optional, default never timeout)
+
+And use the template function in the definition files as follows.
+
+```jsonnet
+local jq = std.native('jq');
+{
+  today: jq('{ Now: now | todateiso8601 }').Now,
+}
+```
+
+```json
+{
+  "today": "{{ (jq `{Now: now | todateiso8601}`).Now }}"
+}
+```
+
 ## LICENSE
 
 MIT

external/external.go

@@ -0,0 +1,117 @@
+package external
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"os/exec"
+	"syscall"
+	"text/template"
+	"time"
+
+	"github.com/google/go-jsonnet"
+	"github.com/google/go-jsonnet/ast"
+)
+
+type Plugin struct {
+	Config *Config
+}
+
+type Config struct {
+	Name    string   `json:"name" yaml:"name"`
+	Command []string `json:"command" yaml:"command"`
+	NumArgs int      `json:"num_args" yaml:"num_args"`
+	Parser  string   `json:"parser" yaml:"parser"`
+	Timeout int64    `json:"timeout" yaml:"timeout"`
+}
+
+func NewPlugin(ctx context.Context, cfg *Config) (*Plugin, error) {
+	if len(cfg.Command) == 0 {
+		return nil, fmt.Errorf("command is required")
+	}
+	if cfg.Name == "" {
+		return nil, fmt.Errorf("name is required")
+	}
+	if cfg.Parser == "" {
+		cfg.Parser = "json" // default parser
+	}
+	if cfg.Parser != "json" && cfg.Parser != "string" {
+		return nil, fmt.Errorf("unsupported parser: %s", cfg.Parser)
+	}
+	return &Plugin{Config: cfg}, nil
+}
+
+func (p *Plugin) Exec(ctx context.Context, extraArgs []string) (any, error) {
+	cmd, args := p.Config.Command[0], p.Config.Command[1:]
+	args = append(args, extraArgs...)
+	stdout := new(bytes.Buffer)
+	stderr := new(bytes.Buffer)
+
+	if p.Config.Timeout > 0 {
+		to := time.Duration(p.Config.Timeout) * time.Second
+		_ctx, cancel := context.WithTimeout(ctx, to)
+		defer cancel()
+		ctx = _ctx
+	}
+	c := exec.CommandContext(ctx, cmd, args...)
+	c.Stdout = stdout
+	c.Stderr = stderr
+	var timedOut bool
+	c.Cancel = func() error {
+		timedOut = true
+		return c.Process.Signal(syscall.SIGTERM)
+	}
+	c.WaitDelay = 5 * time.Second // SIGKILL after 5 seconds
+	if err := c.Run(); err != nil {
+		return nil, fmt.Errorf("failed to run command: %w stdout:%s stderr:%s", err, stdout.String(), stderr.String())
+	}
+	if timedOut {
+		return nil, fmt.Errorf("command timed out: %s %v", cmd, args)
+	}
+	switch p.Config.Parser {
+	case "json", "":
+		var result any
+		if err := json.NewDecoder(stdout).Decode(&result); err != nil {
+			return nil, fmt.Errorf("failed to decode json: %w", err)
+		}
+		return result, nil
+	case "string":
+		return stdout.String(), nil
+	default:
+		return nil, fmt.Errorf("unsupported parser: %s", p.Config.Parser)
+	}
+}
+
+func (p *Plugin) FuncMap(ctx context.Context) template.FuncMap {
+	funcs := template.FuncMap{
+		p.Config.Name: func(args ...string) (any, error) {
+			return p.Exec(ctx, args)
+		},
+	}
+	return funcs
+}
+
+func (p *Plugin) JsonnetNativeFuncs(ctx context.Context) []*jsonnet.NativeFunction {
+	params := make([]ast.Identifier, p.Config.NumArgs)
+	for i := range params {
+		params[i] = ast.Identifier(fmt.Sprintf("arg%d", i))
+	}
+	return []*jsonnet.NativeFunction{
+		{
+			Name:   p.Config.Name,
+			Params: params,
+			Func: func(args []any) (any, error) {
+				pArgs := make([]string, len(args))
+				for i, arg := range args {
+					if s, ok := arg.(string); ok {
+						pArgs[i] = s
+					} else {
+						return nil, fmt.Errorf("arg%d must be string", i)
+					}
+				}
+				return p.Exec(ctx, pArgs)
+			},
+		},
+	}
+}

external/external_test.go

@@ -0,0 +1,94 @@
+package external_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/kayac/ecspresso/v2/external"
+)
+
+func TestExternalPlugin(t *testing.T) {
+	ctx := context.Background()
+	config := external.Config{
+		Name:    "test",
+		Command: []string{"jq", "-n"},
+		NumArgs: 1,
+		Timeout: 0,
+	}
+	p, err := external.NewPlugin(ctx, &config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	result, err := p.Exec(ctx, []string{`{Now: now}`})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result == nil {
+		t.Fatal("result is nil")
+	}
+	if m, ok := result.(map[string]any); ok {
+		unix, ok := m["Now"]
+		if !ok {
+			t.Fatal("Now is not found")
+		}
+		ts, ok := unix.(float64)
+		if !ok {
+			t.Fatalf("Now is not float64: %T", unix)
+		}
+		now := time.Unix(int64(ts), 0)
+		goNow := time.Now()
+		if now.Before(goNow.Add(-1*time.Second)) || now.After(goNow.Add(1*time.Second)) {
+			t.Fatalf("Now is not current time: %s expected: %s", now, goNow)
+		}
+	} else {
+		t.Fatalf("result is not map: %T", result)
+	}
+}
+
+func TestExternalPluginTimeout(t *testing.T) {
+	ctx := context.Background()
+	config := external.Config{
+		Name:    "test",
+		Command: []string{"sh", "-c", "sleep 2; echo 123"},
+		Timeout: 1,
+	}
+	p, err := external.NewPlugin(ctx, &config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := p.Exec(ctx, nil); err == nil {
+		t.Fatal("timeout is not working")
+	} else {
+		t.Log(err)
+	}
+}
+
+func TestExternalPluginString(t *testing.T) {
+	ctx := context.Background()
+	config := external.Config{
+		Name:    "echo",
+		Command: []string{"echo", "-n"},
+		NumArgs: 1,
+		Parser:  "string",
+		Timeout: 0,
+	}
+	p, err := external.NewPlugin(ctx, &config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	result, err := p.Exec(ctx, []string{`Hello World`})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result == nil {
+		t.Fatal("result is nil")
+	}
+	if m, ok := result.(string); ok {
+		if m != "Hello World" {
+			t.Fatalf("unexpected result: %s", m)
+		}
+	} else {
+		t.Fatalf("result is not a string: %T", result)
+	}
+}

plugin.go

@@ -2,6 +2,7 @@ package ecspresso
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"path/filepath"
@@ -13,6 +14,7 @@ import (
 	"github.com/fujiwara/ssm-lookup/ssm"
 	"github.com/fujiwara/tfstate-lookup/tfstate"
 	"github.com/google/go-jsonnet"
+	"github.com/kayac/ecspresso/v2/external"
 	"github.com/kayac/ecspresso/v2/secretsmanager"
 	"github.com/samber/lo"
 )
@@ -35,6 +37,8 @@ func (p ConfigPlugin) Setup(ctx context.Context, c *Config) error {
 		return setupPluginSSM(ctx, p, c)
 	case "secretsmanager":
 		return setupPluginSecretsManager(ctx, p, c)
+	case "external":
+		return setupPluginExternal(ctx, p, c)
 	default:
 		return fmt.Errorf("plugin %s is not available", p.Name)
 	}
@@ -144,3 +148,22 @@ func setupPluginSecretsManager(ctx context.Context, p ConfigPlugin, c *Config) e
 	}
 	return nil
 }
+
+func setupPluginExternal(ctx context.Context, p ConfigPlugin, c *Config) error {
+	extCfg := &external.Config{}
+	b, _ := json.Marshal(p.Config)
+	if err := json.Unmarshal(b, extCfg); err != nil {
+		return fmt.Errorf("failed to unmarshal external plugin config: %w", err)
+	}
+	ext, err := external.NewPlugin(ctx, extCfg)
+	if err != nil {
+		return err
+	}
+	if err := p.AppendFuncMap(c, ext.FuncMap(ctx)); err != nil {
+		return err
+	}
+	if err := p.AppendJsonnetNativeFuncs(c, ext.JsonnetNativeFuncs(ctx)); err != nil {
+		return err
+	}
+	return nil
+}

tests/ci/Makefile

@@ -3,7 +3,6 @@
 export CLUSTER := ecspresso-test
 export SERVICE := nginx
 # export TFSTATE_BUCKET := ecspresso-test
-export NOW := $(shell date +%Y%m%d%H%M%S)
 export CLIENT_TOKEN := $(shell uuidgen)
 export DEPLOYMENT_CONTROLLER ?= ECS
 ECSPRESSO := ecspresso --envfile envfile

tests/ci/ecs-service-def.jsonnet

@@ -1,6 +1,8 @@
 local env = std.native('env');
 local must_env = std.native('must_env');
 local isCodeDeploy = env('DEPLOYMENT_CONTROLLER', 'ECS') == 'CODE_DEPLOY';
+local jq = std.native('jq');
+local cat = std.native('cat');
 {
   capacityProviderStrategy: [
     {
@@ -63,7 +65,15 @@ local isCodeDeploy = env('DEPLOYMENT_CONTROLLER', 'ECS') == 'CODE_DEPLOY';
   tags: [
     {
       key: 'deployed_at',
-      value: '{{ env `NOW` `` }}',
+      value: '{{ jq `now | todateiso8601` }}',
+    },
+    {
+      key: 'deployed_at_localtime',
+      value: jq('now | localtime | todateiso8601'),
+    },
+    {
+      key: 'envfile_content',
+      value: cat('./envfile'),
     },
     {
       key: 'cluster',

tests/ci/ecspresso.yml

@@ -8,6 +8,17 @@ ignore:
   tags:
     - cost-category
 plugins:
+  - name: external
+    config:
+      name: jq
+      command: ["jq", "-n"]
+      num_args: 1
+  - name: external
+    config:
+      name: cat
+      command: ["cat"]
+      parser: string
+      num_args: 1
   - name: tfstate
     config:
       url: "./terraform.tfstate"