Owl 🦉 refactor spec (#695)

Refactor env spec definitions loading from instead of hard-coding them.

go.mod

@@ -47,6 +47,7 @@ require (
 	golang.org/x/oauth2 v0.23.0
 	golang.org/x/sys v0.26.0
 	golang.org/x/term v0.25.0
+	google.golang.org/api v0.196.0
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38
 	google.golang.org/protobuf v1.35.1
 	mvdan.cc/sh/v3 v3.10.0
@@ -93,7 +94,6 @@ require (
 	go.opentelemetry.io/otel/trace v1.31.0 // indirect
 	golang.org/x/net v0.30.0 // indirect
 	golang.org/x/time v0.7.0 // indirect
-	google.golang.org/api v0.196.0 // indirect
 	google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect
 	gotest.tools/v3 v3.5.1 // indirect

internal/cmd/environment.go

@@ -42,28 +42,126 @@ func environmentCmd() *cobra.Command {
 	return &cmd
 }
 
+type envStoreFlags struct {
+	serverAddr      string
+	sessionID       string
+	sessionStrategy string
+	tlsDir          string
+}
+
 func storeCmd() *cobra.Command {
+	var storeFlags envStoreFlags
+
 	cmd := cobra.Command{
 		Hidden: true,
 		Use:    "store",
 		Short:  "Owl store",
 		Long:   "Owl Store",
 	}
 
-	cmd.AddCommand(storeSnapshotCmd())
+	cmd.Flags().StringVar(&storeFlags.serverAddr, "server-address", os.Getenv("RUNME_SERVER_ADDR"), "The Server ServerAddress to connect to, i.e. 127.0.0.1:7865")
+	cmd.Flags().StringVar(&storeFlags.tlsDir, "tls-dir", os.Getenv("RUNME_TLS_DIR"), "Path to tls files")
+	cmd.Flags().StringVar(&storeFlags.sessionID, "session", os.Getenv("RUNME_SESSION"), "Session Id")
+	cmd.Flags().StringVar(&storeFlags.sessionStrategy, "session-strategy", func() string {
+		if val, ok := os.LookupEnv("RUNME_SESSION_STRATEGY"); ok {
+			return val
+		}
+		return "manual"
+	}(), "Strategy for session selection. Options are manual, recent. Defaults to manual")
+
+	cmd.AddCommand(storeSnapshotCmd(storeFlags))
+	cmd.AddCommand(storeSourceCmd(storeFlags))
 	cmd.AddCommand(storeCheckCmd())
 
 	return &cmd
 }
 
-func storeSnapshotCmd() *cobra.Command {
+func storeSourceCmd(storeFlags envStoreFlags) *cobra.Command {
 	var (
-		serverAddr      string
-		sessionID       string
-		sessionStrategy string
-		tlsDir          string
-		limit           int
-		all             bool
+		insecure  bool
+		export    bool
+		sessionID = ""
+	)
+
+	cmd := cobra.Command{
+		Use:   "source",
+		Short: "Source environment variables from session",
+		Long:  "Source environment variables from session",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			// discard any stderr in silent mode
+			if !insecure {
+				return errors.New("must be run in insecure mode to prevent misuse; enable by adding --insecure flag")
+			}
+
+			tlsConfig, err := runmetls.LoadClientConfigFromDir(storeFlags.tlsDir)
+			if err != nil {
+				return err
+			}
+
+			credentials := credentials.NewTLS(tlsConfig)
+			conn, err := grpc.NewClient(
+				storeFlags.serverAddr,
+				grpc.WithTransportCredentials(credentials),
+			)
+			if err != nil {
+				return errors.Wrap(err, "failed to connect")
+			}
+			defer conn.Close()
+
+			client := runnerv1.NewRunnerServiceClient(conn)
+
+			// todo(sebastian): would it be better to require a specific session?
+			if strings.ToLower(storeFlags.sessionStrategy) == "recent" {
+				req := &runnerv1.ListSessionsRequest{}
+				resp, err := client.ListSessions(cmd.Context(), req)
+				if err != nil {
+					return err
+				}
+				l := len(resp.Sessions)
+				if l == 0 {
+					return errors.New("no sessions found")
+				}
+				// potentially unreliable
+				sessionID = resp.Sessions[l-1].Id
+			}
+
+			req := &runnerv1.GetSessionRequest{Id: sessionID}
+			resp, err := client.GetSession(cmd.Context(), req)
+			if err != nil {
+				return err
+			}
+
+			for _, kv := range resp.Session.Envs {
+				parts := strings.Split(kv, "=")
+				if len(parts) < 2 {
+					return errors.Errorf("invalid key-value pair: %s", kv)
+				}
+
+				envVar := fmt.Sprintf("%s=%q", parts[0], strings.Join(parts[1:], "="))
+				if export {
+					envVar = fmt.Sprintf("export %s", envVar)
+				}
+
+				if _, err := fmt.Fprintf(cmd.OutOrStdout(), "%s\n", envVar); err != nil {
+					return err
+				}
+			}
+
+			return nil
+		},
+	}
+
+	cmd.Flags().BoolVarP(&export, "export", "", false, "export variables")
+	cmd.Flags().BoolVar(&insecure, "insecure", false, "Explicitly allow delicate operations to prevent misuse")
+
+	return &cmd
+}
+
+func storeSnapshotCmd(storeFlags envStoreFlags) *cobra.Command {
+	var (
+		limit  int
+		reveal bool
+		all    bool
 	)
 
 	cmd := cobra.Command{
@@ -72,14 +170,18 @@ func storeSnapshotCmd() *cobra.Command {
 		Short:  "Takes a snapshot of the smart env store",
 		Long:   "Connects with a running server to inspect the environment variables of a session and returns a snapshot of the smart env store.",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			tlsConfig, err := runmetls.LoadClientConfigFromDir(tlsDir)
+			tlsConfig, err := runmetls.LoadClientConfigFromDir(storeFlags.tlsDir)
 			if err != nil {
 				return err
 			}
 
+			if reveal && !fInsecure {
+				return errors.New("must be run in insecure mode to prevent misuse; enable by adding --insecure flag")
+			}
+
 			credentials := credentials.NewTLS(tlsConfig)
 			conn, err := grpc.NewClient(
-				serverAddr,
+				storeFlags.serverAddr,
 				grpc.WithTransportCredentials(credentials),
 			)
 			if err != nil {
@@ -90,7 +192,7 @@ func storeSnapshotCmd() *cobra.Command {
 			client := runnerv1.NewRunnerServiceClient(conn)
 
 			// todo(sebastian): this should move into API as part of v2
-			if strings.ToLower(sessionStrategy) == "recent" {
+			if strings.ToLower(storeFlags.sessionStrategy) == "recent" {
 				req := &runnerv1.ListSessionsRequest{}
 				resp, err := client.ListSessions(cmd.Context(), req)
 				if err != nil {
@@ -101,11 +203,11 @@ func storeSnapshotCmd() *cobra.Command {
 					return errors.New("no sessions found")
 				}
 				// potentially unreliable
-				sessionID = resp.Sessions[l-1].Id
+				storeFlags.sessionID = resp.Sessions[l-1].Id
 			}
 
 			req := &runnerv1.MonitorEnvStoreRequest{
-				Session: &runnerv1.Session{Id: sessionID},
+				Session: &runnerv1.Session{Id: storeFlags.sessionID},
 			}
 			meClient, err := client.MonitorEnvStore(cmd.Context(), req)
 			if err != nil {
@@ -119,25 +221,17 @@ func storeSnapshotCmd() *cobra.Command {
 			}
 
 			if msgData, ok := msg.Data.(*runnerv1.MonitorEnvStoreResponse_Snapshot); ok {
-				return errors.Wrap(printStore(cmd, msgData, limit, all), "failed to render")
+				insecureReveal := reveal && fInsecure
+				return errors.Wrap(printStore(cmd, msgData, limit, insecureReveal, all), "failed to render")
 			}
 
 			return nil
 		},
 	}
 
-	cmd.Flags().StringVar(&serverAddr, "server-address", os.Getenv("RUNME_SERVER_ADDR"), "The Server ServerAddress to connect to, i.e. 127.0.0.1:7865")
-	cmd.Flags().StringVar(&tlsDir, "tls-dir", os.Getenv("RUNME_TLS_DIR"), "Path to tls files")
-	cmd.Flags().StringVar(&sessionID, "session", os.Getenv("RUNME_SESSION"), "Session Id")
-	cmd.Flags().StringVar(&sessionStrategy, "session-strategy", func() string {
-		if val, ok := os.LookupEnv("RUNME_SESSION_STRATEGY"); ok {
-			return val
-		}
-
-		return "manual"
-	}(), "Strategy for session selection. Options are manual, recent. Defaults to manual")
 	cmd.Flags().IntVar(&limit, "limit", 50, "Limit the number of lines")
 	cmd.Flags().BoolVarP(&all, "all", "A", false, "Show all lines")
+	cmd.Flags().BoolVarP(&reveal, "reveal", "r", false, "Reveal hidden values")
 
 	return &cmd
 }
@@ -225,7 +319,7 @@ func environmentDumpCmd() *cobra.Command {
 	return &cmd
 }
 
-func printStore(cmd *cobra.Command, msgData *runnerv1.MonitorEnvStoreResponse_Snapshot, lines int, all bool) error {
+func printStore(cmd *cobra.Command, msgData *runnerv1.MonitorEnvStoreResponse_Snapshot, lines int, reveal, all bool) error {
 	term := term.FromIO(cmd.InOrStdin(), cmd.OutOrStdout(), cmd.ErrOrStderr())
 
 	width, _, err := term.Size()
@@ -238,7 +332,7 @@ func printStore(cmd *cobra.Command, msgData *runnerv1.MonitorEnvStoreResponse_Sn
 	table.AddField(strings.ToUpper("Value"))
 	table.AddField(strings.ToUpper("Description"))
 	table.AddField(strings.ToUpper("Spec"))
-	table.AddField(strings.ToUpper("Origin"))
+	table.AddField(strings.ToUpper("Source"))
 	table.AddField(strings.ToUpper("Updated"))
 	table.EndRow()
 
@@ -247,8 +341,8 @@ func printStore(cmd *cobra.Command, msgData *runnerv1.MonitorEnvStoreResponse_Sn
 			break
 		}
 
-		specless := msgData.Snapshot.Envs[0].Spec != owl.SpecNameOpaque
-		if !all && specless && env.Spec == owl.SpecNameOpaque {
+		specless := msgData.Snapshot.Envs[0].Spec != owl.AtomicNameOpaque
+		if !all && specless && env.Spec == owl.AtomicNameOpaque {
 			break
 		}
 
@@ -260,7 +354,10 @@ func printStore(cmd *cobra.Command, msgData *runnerv1.MonitorEnvStoreResponse_Sn
 		case runnerv1.MonitorEnvStoreResponseSnapshot_STATUS_MASKED:
 			value = "[masked]"
 		case runnerv1.MonitorEnvStoreResponseSnapshot_STATUS_HIDDEN:
-			value = "******"
+			value = "[hidden]"
+			if reveal {
+				value = env.GetOriginalValue()
+			}
 		}
 
 		strippedVal := strings.ReplaceAll(strings.ReplaceAll(value, "\n", " "), "\r", "")

internal/owl/envSpecDefs.defaults.yaml

@@ -0,0 +1,111 @@
+apiVersion: runme.stateful.com/v1beta1
+kind: EnvSpecDefinitions
+metadata:
+  name: runme
+  namespace: stateful
+  annotations:
+    github.com/repo-url: https://github.com/stateful/runme
+spec:
+  type: owl
+  envSpecs:
+    - name: Auth0
+      breaker: AUTH0
+      atomics:
+        - key: AUDIENCE
+          atomic: Plain
+          rules: url
+          required: true
+        - key: CLIENT_ID
+          atomic: Secret
+          rules: alphanum,min=32,max=32
+          required: true
+        - key: DOMAIN
+          atomic: Secret
+          rules: fqdn
+          required: true
+        - key: COOKIE_DOMAIN
+          atomic: Secret
+          rules: min=8
+          required: true
+    - name: Auth0Mgmt
+      breaker: AUTH0_MANAGEMENT
+      atomics:
+        - key: CLIENT_ID
+          atomic: Plain
+          rules: alphanum,min=32,max=32
+          required: true
+        - key: CLIENT_SECRET
+          atomic: Secret
+          rules: ascii,min=64,max=64
+          required: true
+        - key: AUDIENCE
+          atomic: Plain
+          rules: url
+          required: true
+    - name: DatabaseUrl
+      breaker: DATABASE
+      atomics:
+        - key: URL
+          atomic: Secret
+          rules: database_url
+          required: true
+    - name: OpenAI
+      breaker: OPENAI
+      atomics:
+        - key: ORG_ID
+          atomic: Opaque
+          rules: ascii,min=28,max=28,startswith=org-
+          required: true
+        - key: API_KEY
+          atomic: Secret
+          rules: ascii,min=34,startswith=sk-
+          required: true
+    - name: Redis
+      breaker: REDIS
+      atomics:
+        - key: HOST
+          atomic: Plain
+          rules: ip|hostname
+          required: true
+        - key: PORT
+          atomic: Plain
+          rules: number
+          required: true
+        - key: PASSWORD
+          atomic: Password
+          rules: min=18,max=32
+          required: false
+    - name: Slack
+      breaker: SLACK
+      atomics:
+        - key: CLIENT_ID
+          atomic: Plain
+          rules: min=24,max=24
+          required: true
+        - key: CLIENT_SECRET
+          atomic: Secret
+          rules: min=32,max=32
+          required: true
+        - key: REDIRECT_URL
+          atomic: Secret
+          rules: url
+          required: true
+    - name: UserHub
+      breaker: USERHUB
+      atomics:
+        - key: WEBHOOK_SECRET
+          atomic: Secret
+          rules: min=8,max=64
+          required: true
+        - key: CONN_PROVIDER_ID
+          atomic: Plain
+          rules:
+          required: true
+        - key: SKIP_CONNECTION
+          atomic: Plain
+          rules:
+          required: false
+        - key: API_KEY
+          atomic: Secret
+          rules: min=16,max=64
+          required: true