Skip to content

Commit

Permalink
Owl 🦉: More validation and GCP-SM prototype (#693)
Browse files Browse the repository at this point in the history 
This is a work-in-progress milestone. However, before continuing a big
refactor. I'm merging this to baseline the next PR.
  • Loading branch information
@sourishkrout
sourishkrout authored Oct 29, 2024
1 parent 3077809 commit f238750
Show file tree
Hide file tree
Showing 17 changed files with 878 additions and 109 deletions.
16 changes: 14 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ toolchain go1.23.2
// replace github.com/stateful/godotenv => ../godotenv

require (
cloud.google.com/go/secretmanager v1.14.1
github.com/Masterminds/semver/v3 v3.3.0
github.com/Microsoft/go-winio v0.6.2
github.com/atotto/clipboard v0.1.4
Expand Down Expand Up @@ -52,6 +53,10 @@ require (
)

require (
cloud.google.com/go/auth v0.9.4 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
cloud.google.com/go/compute/metadata v0.5.2 // indirect
cloud.google.com/go/iam v1.2.1 // indirect
dario.cat/mergo v1.0.1 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/bufbuild/protocompile v0.14.1 // indirect
Expand All @@ -71,19 +76,26 @@ require (
github.com/go-logr/stdr v1.2.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/s2a-go v0.1.8 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
github.com/googleapis/gax-go/v2 v2.13.0 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/term v0.5.0 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 // indirect
go.opentelemetry.io/otel v1.31.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 // indirect
go.opentelemetry.io/otel/metric v1.31.0 // indirect
go.opentelemetry.io/otel/sdk v1.24.0 // indirect
go.opentelemetry.io/otel/trace v1.31.0 // indirect
golang.org/x/net v0.30.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/time v0.7.0 // indirect
google.golang.org/api v0.196.0 // indirect
google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect
gotest.tools/v3 v3.5.1 // indirect
)

Expand Down
111 changes: 104 additions & 7 deletions go.sum
View file

Large diffs are not rendered by default.

134 changes: 130 additions & 4 deletions internal/owl/graph.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,17 @@ package owl

import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"log"
"strings"

sm "cloud.google.com/go/secretmanager/apiv1"
smpb "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb"
exprlang "github.com/expr-lang/expr"
"github.com/graphql-go/graphql"
"github.com/pkg/errors"
)

// Constants representing different spec names.
Expand All @@ -34,6 +39,7 @@ var (

var EnvironmentType,
ValidateType,
ResolveType,
RenderType,
SpecTypeErrorsType *graphql.Object

Expand Down Expand Up @@ -248,6 +254,7 @@ func specResolver(mutator SpecResolverMutator) graphql.FieldResolveFn {

spec.Spec.Complex = complexName
spec.Spec.Namespace = complexNs
spec.Spec.Checked = true

// skip if last known status was DELETED
if valOk && val.Value.Status == "DELETED" {
Expand All @@ -272,9 +279,6 @@ func specResolver(mutator SpecResolverMutator) graphql.FieldResolveFn {
}

mutator(val, spec, insecure)
if specOk {
spec.Spec.Checked = true
}
}

return p.Source, nil
Expand Down Expand Up @@ -706,6 +710,122 @@ func init() {
}),
})

ResolveType = graphql.NewObject(graphql.ObjectConfig{
Name: "ResolveType",
Fields: (graphql.FieldsThunk)(func() graphql.Fields {
fields := graphql.Fields{
"transform": &graphql.Field{
Type: ResolveType,
Args: graphql.FieldConfigArgument{
"expr": &graphql.ArgumentConfig{
Type: graphql.NewNonNull(graphql.String),
},
},
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
var opSet *OperationSet
var complexOpSet *ComplexOperationSet

switch p.Source.(type) {
case *OperationSet:
opSet = p.Source.(*OperationSet)
case *ComplexOperationSet:
complexOpSet = p.Source.(*ComplexOperationSet)
opSet = complexOpSet.OperationSet
default:
return nil, errors.New("source does not contain an OperationSet")
}

expr, ok := p.Args["expr"].(string)
if !ok {
return nil, errors.New("transform without expr")
}

ctx := context.Background()
smClient, err := sm.NewClient(ctx)
if err != nil {
log.Fatalf("failed to setup client: %v", err)
}
defer smClient.Close()

for _, v := range opSet.values {
if v.Value.Status != "UNRESOLVED" {
v.Value.Status = "DELETED"
continue
}

env := map[string]string{"key": v.Var.Key}

program, err := exprlang.Compile(expr, exprlang.Env(env))
if err != nil {
return nil, errors.Wrap(err, "failed to compile transform program")
}

output, err := exprlang.Run(program, env)
if err != nil {
return nil, errors.Wrap(err, "failed to run transform program")
}

res, ok := output.(string)
if !ok {
return nil, errors.New("transform output is not a string")
}

spec, ok := opSet.specs[v.Var.Key]
if !ok {
return nil, fmt.Errorf("missing spec for %s", v.Var.Key)
}

_, aitem, err := complexOpSet.GetAtomicItem(spec)
if err != nil {
return nil, err
}

if aitem.Spec.Name != SpecNameSecret && aitem.Spec.Name != SpecNamePassword {
v.Value.Status = "DELETED"
continue
}

uri := fmt.Sprintf("projects/platform-staging-413816/secrets/%s", res)
// status := strings.ToLower(aitem.Value.Status)
// if aitem.Spec.Name == SpecNameSecret || aitem.Spec.Name == SpecNamePassword {
// _, _ = fmt.Println(status, aitem.Spec.Name, aitem.Var.Key, "via", uri)
// } else {
// _, _ = fmt.Println(status, aitem.Spec.Name, aitem.Var.Key)
// continue
// }

accessRequest := &smpb.AccessSecretVersionRequest{
Name: fmt.Sprintf("%s/versions/latest", uri),
}

result, err := smClient.AccessSecretVersion(ctx, accessRequest)
if err != nil {
return nil, errors.Errorf("failed to access secret version: %v", err)
}

if err := opSet.resolveValue(v.Var.Key, string(result.Payload.Data)); err != nil {
return nil, err
}
}

if complexOpSet != nil {
return complexOpSet, nil
}

return opSet, nil
},
},
"done": &graphql.Field{
Type: EnvironmentType,
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
return p.Source, nil
},
},
}
return fields
}),
})

OperationType := &graphql.Field{
Type: graphql.NewObject(graphql.ObjectConfig{
Name: "VariableOperationType",
Expand Down Expand Up @@ -1024,6 +1144,12 @@ func init() {
return p.Source, nil
},
},
"resolve": &graphql.Field{
Type: ResolveType,
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
return p.Source, nil
},
},
}
}),
})
Expand Down
18 changes: 9 additions & 9 deletions internal/owl/graph_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
"gopkg.in/yaml.v3"
)

func Test_Graph(t *testing.T) {
func TestGraph(t *testing.T) {
t.Run("introspect schema", func(t *testing.T) {
result := graphql.Do(graphql.Params{
Schema: Schema,
Expand All @@ -33,7 +33,7 @@ func Test_Graph(t *testing.T) {
})
}

func Test_QuerySpecs(t *testing.T) {
func TestQuerySpecs(t *testing.T) {
t.Run("query list of specs", func(t *testing.T) {
result := graphql.Do(graphql.Params{
Schema: Schema,
Expand Down Expand Up @@ -102,7 +102,7 @@ func (testCases fileTestCases) runAll(t *testing.T) {
}
}

func Test_ResolveEnv(t *testing.T) {
func TestResolveEnv(t *testing.T) {
t.Parallel()

testCases := fileTestCases{
Expand Down Expand Up @@ -163,7 +163,7 @@ func Test_ResolveEnv(t *testing.T) {
testCases.runAll(t)
}

func Test_Graph_Update(t *testing.T) {
func TestGraph_Update(t *testing.T) {
testCases := fileTestCases{
{
name: "Store update",
Expand All @@ -178,7 +178,7 @@ func Test_Graph_Update(t *testing.T) {
testCases.runAll(t)
}

func Test_Graph_Required(t *testing.T) {
func TestGraph_Required(t *testing.T) {
testCases := fileTestCases{
{
name: "Validate simple env",
Expand All @@ -199,7 +199,7 @@ func Test_Graph_Required(t *testing.T) {
testCases.runAll(t)
}

func Test_Graph_Reconcile(t *testing.T) {
func TestGraph_Reconcile(t *testing.T) {
testCases := fileTestCases{
{
name: "Reconcile operationless",
Expand All @@ -220,7 +220,7 @@ func Test_Graph_Reconcile(t *testing.T) {
testCases.runAll(t)
}

func Test_Graph_SensitiveKeys(t *testing.T) {
func TestGraph_SensitiveKeys(t *testing.T) {
testCases := fileTestCases{
{
name: "Sensitive keys",
Expand All @@ -241,7 +241,7 @@ func Test_Graph_SensitiveKeys(t *testing.T) {
testCases.runAll(t)
}

func Test_Graph_Get(t *testing.T) {
func TestGraph_Get(t *testing.T) {
testCases := fileTestCases{
{
name: "InsecureGet",
Expand All @@ -263,7 +263,7 @@ func Test_Graph_Get(t *testing.T) {
testCases.runAll(t)
}

func Test_Graph_DotEnv(t *testing.T) {
func TestGraph_DotEnv(t *testing.T) {
testCases := fileTestCases{
{
name: "Without prefix",
Expand Down
16 changes: 8 additions & 8 deletions internal/owl/parse_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,14 +159,14 @@ func TestValues(t *testing.T) {
require.Equal(t, map[string]string{
"ALLOWED_URL_PATTERNS": "Plain!",
"API_URL": "Plain!",
"AUTH0_AUDIENCE": "Plain!",
"AUTH0_CLIENT_ID": "Plain!",
"AUTH0_AUDIENCE": "Auth0!",
"AUTH0_CLIENT_ID": "Auth0!",
"AUTH0_COOKIE_DOMAIN": "Plain",
"AUTH0_DEV_ID": "Plain",
"AUTH0_DOMAIN": "Plain!",
"AUTH0_MANAGEMENT_AUDIENCE": "Plain!",
"AUTH0_MANAGEMENT_CLIENT_ID": "Plain!",
"AUTH0_MANAGEMENT_CLIENT_SECRET": "Secret!",
"AUTH0_DOMAIN": "Auth0!",
"AUTH0_MANAGEMENT_AUDIENCE": "Auth0Mgmt!",
"AUTH0_MANAGEMENT_CLIENT_ID": "Auth0Mgmt!",
"AUTH0_MANAGEMENT_CLIENT_SECRET": "Auth0Mgmt!",
"AUTH0_WEBHOOK_TOKEN": "Secret!",
"AUTH_DEV_SKIP_EXP": "Plain",
"CORS_ORIGINS": "Plain",
Expand All @@ -183,8 +183,8 @@ func TestValues(t *testing.T) {
"IDP_REDIRECT_URL": "Plain!",
"MIXPANEL_TOKEN": "Secret",
"NODE_ENV": "Plain",
"OPENAI_API_KEY": "Secret!",
"OPENAI_ORG_ID": "Secret!",
"OPENAI_API_KEY": "OpenAI!",
"OPENAI_ORG_ID": "OpenAI!",
"PORT": "Plain!",
"RATE_LIMIT_MAX": "Plain",
"RATE_LIMIT_TIME_WINDOW": "Plain",
Expand Down
Loading

0 comments on commit f238750

Please sign in to comment.