From 3ff804d0930e3110ad9658a7a1f34fced91e3632 Mon Sep 17 00:00:00 2001 From: fujiwara Date: Sat, 26 Oct 2024 01:17:05 +0900 Subject: [PATCH 1/3] Add external plugin external plugin runs any external commands as template functions. --- README.md | 51 +++++++++++++++ external/external.go | 103 +++++++++++++++++++++++++++++++ external/external_test.go | 65 +++++++++++++++++++ plugin.go | 23 +++++++ tests/ci/Makefile | 1 - tests/ci/ecs-service-def.jsonnet | 7 ++- tests/ci/ecspresso.yml | 5 ++ 7 files changed, 253 insertions(+), 2 deletions(-) create mode 100644 external/external.go create mode 100644 external/external_test.go diff --git a/README.md b/README.md index 7858b804..2dc9e9ca 100644 --- a/README.md +++ b/README.md @@ -1104,6 +1104,57 @@ local secretsmanager_arn = std.native('secretsmanager_arn'); } ``` +### Execute external commands + +The `external` plugin introduces functions to execute any external commands. + +For example, `jq -n "{ Now: now | todateiso8601" }` returns the current date in ISO8601 format as a JSON object. + +```console +$ jq -n "{ Now: now | todateiso8601 }" +{ + "Now": "2024-10-25T16:13:22Z" +} +``` + +You can use this command as a template function in the definition files. + +First, define the plugin in the configuration file. + +ecspresso.yml +```yaml +plugins: + - name: external + config: + name: jq + command: ["jq", "-n"] + num_args: 1 + timeout: 5 +``` + +The `config` section defines the following parameters: + +- `name`: template function name +- `command`: external command and arguments (array) + - The command must return a JSON string to stdout. +- `num_args`: number of arguments (optional, default 0) +- `timeout`: command execution timeout seconds (optional, default never timeout) + +And use the template function in the definition files as follows. + +```jsonnet +local jq = std.native('jq'); +{ + today: jq('{ Now: now | todateiso8601 }').Now, +} +``` + +```json +{ + "today": "{{ (jq `{Now: now | todateiso8601}`).Now }}" +} +``` + ## LICENSE MIT diff --git a/external/external.go b/external/external.go new file mode 100644 index 00000000..32dde553 --- /dev/null +++ b/external/external.go @@ -0,0 +1,103 @@ +package external + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os/exec" + "syscall" + "text/template" + "time" + + "github.com/google/go-jsonnet" + "github.com/google/go-jsonnet/ast" +) + +type Plugin struct { + Config *Config +} + +type Config struct { + Name string `json:"name" yaml:"name"` + Command []string `json:"command" yaml:"command"` + NumArgs int `json:"num_args" yaml:"num_args"` + Timeout int64 `json:"timeout" yaml:"timeout"` +} + +func NewPlugin(ctx context.Context, cfg *Config) (*Plugin, error) { + if len(cfg.Command) == 0 { + return nil, fmt.Errorf("command is required") + } + if cfg.Name == "" { + return nil, fmt.Errorf("name is required") + } + return &Plugin{Config: cfg}, nil +} + +func (p *Plugin) Exec(ctx context.Context, extraArgs []string) (any, error) { + cmd, args := p.Config.Command[0], p.Config.Command[1:] + args = append(args, extraArgs...) + stdout := new(bytes.Buffer) + stderr := new(bytes.Buffer) + + if p.Config.Timeout > 0 { + to := time.Duration(p.Config.Timeout) * time.Second + _ctx, cancel := context.WithTimeout(ctx, to) + defer cancel() + ctx = _ctx + } + c := exec.CommandContext(ctx, cmd, args...) + c.Stdout = stdout + c.Stderr = stderr + var timedOut bool + c.Cancel = func() error { + timedOut = true + return c.Process.Signal(syscall.SIGTERM) + } + c.WaitDelay = 5 * time.Second // SIGKILL after 5 seconds + if err := c.Run(); err != nil { + return nil, fmt.Errorf("failed to run command: %w stdout:%s stderr:%s", err, stdout.String(), stderr.String()) + } + if timedOut { + return nil, fmt.Errorf("command timed out: %s %v", cmd, args) + } + var result any + if err := json.NewDecoder(stdout).Decode(&result); err != nil { + return nil, fmt.Errorf("failed to decode json: %w", err) + } + return result, nil +} + +func (p *Plugin) FuncMap(ctx context.Context) template.FuncMap { + funcs := template.FuncMap{ + p.Config.Name: func(args ...string) (any, error) { + return p.Exec(ctx, args) + }, + } + return funcs +} + +func (p *Plugin) JsonnetNativeFuncs(ctx context.Context) []*jsonnet.NativeFunction { + params := make([]ast.Identifier, p.Config.NumArgs) + for i := range params { + params[i] = ast.Identifier(fmt.Sprintf("arg%d", i)) + } + return []*jsonnet.NativeFunction{ + { + Name: p.Config.Name, + Params: params, + Func: func(args []any) (any, error) { + pArgs := make([]string, len(args)) + for i, arg := range args { + if s, ok := arg.(string); ok { + pArgs[i] = s + } else { + return nil, fmt.Errorf("arg%d must be string", i) + } + } + return p.Exec(ctx, pArgs) + }, + }, + } +} diff --git a/external/external_test.go b/external/external_test.go new file mode 100644 index 00000000..e3a12c17 --- /dev/null +++ b/external/external_test.go @@ -0,0 +1,65 @@ +package external_test + +import ( + "context" + "testing" + "time" + + "github.com/kayac/ecspresso/v2/external" +) + +func TestExternalPlugin(t *testing.T) { + ctx := context.Background() + config := external.Config{ + Name: "test", + Command: []string{"jq", "-n"}, + NumArgs: 1, + Timeout: 0, + } + p, err := external.NewPlugin(ctx, &config) + if err != nil { + t.Fatal(err) + } + result, err := p.Exec(ctx, []string{`{Now: now}`}) + if err != nil { + t.Fatal(err) + } + if result == nil { + t.Fatal("result is nil") + } + if m, ok := result.(map[string]any); ok { + unix, ok := m["Now"] + if !ok { + t.Fatal("Now is not found") + } + ts, ok := unix.(float64) + if !ok { + t.Fatalf("Now is not float64: %T", unix) + } + now := time.Unix(int64(ts), 0) + goNow := time.Now() + if now.Before(goNow.Add(-1*time.Second)) || now.After(goNow.Add(1*time.Second)) { + t.Fatalf("Now is not current time: %s expected: %s", now, goNow) + } + } else { + t.Fatalf("result is not map: %T", result) + } +} + +func TestExternalPluginTimeout(t *testing.T) { + ctx := context.Background() + config := external.Config{ + Name: "test", + Command: []string{"sh", "-c", "sleep 2; echo 123"}, + Timeout: 1, + } + p, err := external.NewPlugin(ctx, &config) + if err != nil { + t.Fatal(err) + } + if _, err := p.Exec(ctx, nil); err == nil { + t.Fatal("timeout is not working") + } else { + t.Log(err) + } +} diff --git a/plugin.go b/plugin.go index 933d9504..40c26c0c 100644 --- a/plugin.go +++ b/plugin.go @@ -2,6 +2,7 @@ package ecspresso import ( "context" + "encoding/json" "errors" "fmt" "path/filepath" @@ -13,6 +14,7 @@ import ( "github.com/fujiwara/ssm-lookup/ssm" "github.com/fujiwara/tfstate-lookup/tfstate" "github.com/google/go-jsonnet" + "github.com/kayac/ecspresso/v2/external" "github.com/kayac/ecspresso/v2/secretsmanager" "github.com/samber/lo" ) @@ -35,6 +37,8 @@ func (p ConfigPlugin) Setup(ctx context.Context, c *Config) error { return setupPluginSSM(ctx, p, c) case "secretsmanager": return setupPluginSecretsManager(ctx, p, c) + case "external": + return setupPluginExternal(ctx, p, c) default: return fmt.Errorf("plugin %s is not available", p.Name) } @@ -144,3 +148,22 @@ func setupPluginSecretsManager(ctx context.Context, p ConfigPlugin, c *Config) e } return nil } + +func setupPluginExternal(ctx context.Context, p ConfigPlugin, c *Config) error { + extCfg := &external.Config{} + b, _ := json.Marshal(p.Config) + if err := json.Unmarshal(b, extCfg); err != nil { + return fmt.Errorf("failed to unmarshal external plugin config: %w", err) + } + ext, err := external.NewPlugin(ctx, extCfg) + if err != nil { + return err + } + if err := p.AppendFuncMap(c, ext.FuncMap(ctx)); err != nil { + return err + } + if err := p.AppendJsonnetNativeFuncs(c, ext.JsonnetNativeFuncs(ctx)); err != nil { + return err + } + return nil +} diff --git a/tests/ci/Makefile b/tests/ci/Makefile index 9001e531..4f50e809 100644 --- a/tests/ci/Makefile +++ b/tests/ci/Makefile @@ -3,7 +3,6 @@ export CLUSTER := ecspresso-test export SERVICE := nginx # export TFSTATE_BUCKET := ecspresso-test -export NOW := $(shell date +%Y%m%d%H%M%S) export CLIENT_TOKEN := $(shell uuidgen) export DEPLOYMENT_CONTROLLER ?= ECS ECSPRESSO := ecspresso --envfile envfile diff --git a/tests/ci/ecs-service-def.jsonnet b/tests/ci/ecs-service-def.jsonnet index a952920a..dbe2b9af 100644 --- a/tests/ci/ecs-service-def.jsonnet +++ b/tests/ci/ecs-service-def.jsonnet @@ -1,6 +1,7 @@ local env = std.native('env'); local must_env = std.native('must_env'); local isCodeDeploy = env('DEPLOYMENT_CONTROLLER', 'ECS') == 'CODE_DEPLOY'; +local jq = std.native('jq'); { capacityProviderStrategy: [ { @@ -63,7 +64,11 @@ local isCodeDeploy = env('DEPLOYMENT_CONTROLLER', 'ECS') == 'CODE_DEPLOY'; tags: [ { key: 'deployed_at', - value: '{{ env `NOW` `` }}', + value: '{{ jq `now | todateiso8601` }}', + }, + { + key: 'deployed_at_localtime', + value: jq('now | localtime | todateiso8601'), }, { key: 'cluster', diff --git a/tests/ci/ecspresso.yml b/tests/ci/ecspresso.yml index 7e9f2a8a..7c13e175 100644 --- a/tests/ci/ecspresso.yml +++ b/tests/ci/ecspresso.yml @@ -8,6 +8,11 @@ ignore: tags: - cost-category plugins: + - name: external + config: + name: jq + command: ["jq", "-n"] + num_args: 1 - name: tfstate config: url: "./terraform.tfstate" From 4353e80c9776da47a870445ee56c83e58401b555 Mon Sep 17 00:00:00 2001 From: fujiwara Date: Mon, 28 Oct 2024 10:51:14 +0900 Subject: [PATCH 2/3] add parser config for external plugin --- README.md | 1 + external/external.go | 22 ++++++++++++++++++---- external/external_test.go | 29 +++++++++++++++++++++++++++++ tests/ci/ecs-service-def.jsonnet | 5 +++++ tests/ci/ecspresso.yml | 6 ++++++ 5 files changed, 59 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2dc9e9ca..fb2d236a 100644 --- a/README.md +++ b/README.md @@ -1138,6 +1138,7 @@ The `config` section defines the following parameters: - `command`: external command and arguments (array) - The command must return a JSON string to stdout. - `num_args`: number of arguments (optional, default 0) +- `parser`: parser type "json" or "string" (optional, default "json") - `timeout`: command execution timeout seconds (optional, default never timeout) And use the template function in the definition files as follows. diff --git a/external/external.go b/external/external.go index 32dde553..a8957fa0 100644 --- a/external/external.go +++ b/external/external.go @@ -22,6 +22,7 @@ type Config struct { Name string `json:"name" yaml:"name"` Command []string `json:"command" yaml:"command"` NumArgs int `json:"num_args" yaml:"num_args"` + Parser string `json:"parser" yaml:"parser"` Timeout int64 `json:"timeout" yaml:"timeout"` } @@ -32,6 +33,12 @@ func NewPlugin(ctx context.Context, cfg *Config) (*Plugin, error) { if cfg.Name == "" { return nil, fmt.Errorf("name is required") } + if cfg.Parser == "" { + cfg.Parser = "json" // default parser + } + if cfg.Parser != "json" && cfg.Parser != "string" { + return nil, fmt.Errorf("unsupported parser: %s", cfg.Parser) + } return &Plugin{Config: cfg}, nil } @@ -62,11 +69,18 @@ func (p *Plugin) Exec(ctx context.Context, extraArgs []string) (any, error) { if timedOut { return nil, fmt.Errorf("command timed out: %s %v", cmd, args) } - var result any - if err := json.NewDecoder(stdout).Decode(&result); err != nil { - return nil, fmt.Errorf("failed to decode json: %w", err) + switch p.Config.Parser { + case "json", "": + var result any + if err := json.NewDecoder(stdout).Decode(&result); err != nil { + return nil, fmt.Errorf("failed to decode json: %w", err) + } + return result, nil + case "string": + return stdout.String(), nil + default: + return nil, fmt.Errorf("unsupported parser: %s", p.Config.Parser) } - return result, nil } func (p *Plugin) FuncMap(ctx context.Context) template.FuncMap { diff --git a/external/external_test.go b/external/external_test.go index e3a12c17..d9a72d36 100644 --- a/external/external_test.go +++ b/external/external_test.go @@ -63,3 +63,32 @@ func TestExternalPluginTimeout(t *testing.T) { t.Log(err) } } + +func TestExternalPluginString(t *testing.T) { + ctx := context.Background() + config := external.Config{ + Name: "echo", + Command: []string{"echo", "-n"}, + NumArgs: 1, + Parser: "string", + Timeout: 0, + } + p, err := external.NewPlugin(ctx, &config) + if err != nil { + t.Fatal(err) + } + result, err := p.Exec(ctx, []string{`Hello World`}) + if err != nil { + t.Fatal(err) + } + if result == nil { + t.Fatal("result is nil") + } + if m, ok := result.(string); ok { + if m != "Hello World" { + t.Fatalf("unexpected result: %s", m) + } + } else { + t.Fatalf("result is not a string: %T", result) + } +} diff --git a/tests/ci/ecs-service-def.jsonnet b/tests/ci/ecs-service-def.jsonnet index dbe2b9af..037ca4d6 100644 --- a/tests/ci/ecs-service-def.jsonnet +++ b/tests/ci/ecs-service-def.jsonnet @@ -2,6 +2,7 @@ local env = std.native('env'); local must_env = std.native('must_env'); local isCodeDeploy = env('DEPLOYMENT_CONTROLLER', 'ECS') == 'CODE_DEPLOY'; local jq = std.native('jq'); +local cat = std.native('cat'); { capacityProviderStrategy: [ { @@ -70,6 +71,10 @@ local jq = std.native('jq'); key: 'deployed_at_localtime', value: jq('now | localtime | todateiso8601'), }, + { + key: 'envfile_content', + value: cat('./envfile'), + }, { key: 'cluster', value: 'ecspresso-test', diff --git a/tests/ci/ecspresso.yml b/tests/ci/ecspresso.yml index 7c13e175..a101be4b 100644 --- a/tests/ci/ecspresso.yml +++ b/tests/ci/ecspresso.yml @@ -13,6 +13,12 @@ plugins: name: jq command: ["jq", "-n"] num_args: 1 + - name: external + config: + name: cat + command: ["cat"] + parser: string + num_args: 1 - name: tfstate config: url: "./terraform.tfstate" From e08844eb087147c0f732d743da5cec5b671d7254 Mon Sep 17 00:00:00 2001 From: fujiwara Date: Mon, 28 Oct 2024 10:59:47 +0900 Subject: [PATCH 3/3] fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fb2d236a..dd61a5c0 100644 --- a/README.md +++ b/README.md @@ -1136,7 +1136,7 @@ The `config` section defines the following parameters: - `name`: template function name - `command`: external command and arguments (array) - - The command must return a JSON string to stdout. + - The command must return a JSON string or any strings to stdout. - `num_args`: number of arguments (optional, default 0) - `parser`: parser type "json" or "string" (optional, default "json") - `timeout`: command execution timeout seconds (optional, default never timeout)