diff --git a/cmd/sops/common/common.go b/cmd/sops/common/common.go index 6d6fa0751..9d4d1abf6 100644 --- a/cmd/sops/common/common.go +++ b/cmd/sops/common/common.go @@ -18,6 +18,7 @@ import ( "github.com/getsops/sops/v3/stores/dotenv" "github.com/getsops/sops/v3/stores/ini" "github.com/getsops/sops/v3/stores/json" + "github.com/getsops/sops/v3/stores/toml" "github.com/getsops/sops/v3/stores/yaml" "github.com/getsops/sops/v3/version" "github.com/mitchellh/go-wordwrap" @@ -55,6 +56,10 @@ func newJsonStore(c *config.StoresConfig) Store { return json.NewStore(&c.JSON) } +func newTomlStore(c *config.StoresConfig) Store { + return toml.NewStore(&c.TOML) +} + func newYamlStore(c *config.StoresConfig) Store { return yaml.NewStore(&c.YAML) } @@ -64,6 +69,7 @@ var storeConstructors = map[Format]storeConstructor{ Dotenv: newDotenvStore, Ini: newIniStore, Json: newJsonStore, + Toml: newTomlStore, Yaml: newYamlStore, } diff --git a/cmd/sops/formats/formats.go b/cmd/sops/formats/formats.go index 83c5391e2..0f3eaf580 100644 --- a/cmd/sops/formats/formats.go +++ b/cmd/sops/formats/formats.go @@ -10,6 +10,7 @@ const ( Dotenv Ini Json + Toml Yaml ) @@ -18,6 +19,7 @@ var stringToFormat = map[string]Format{ "dotenv": Dotenv, "ini": Ini, "json": Json, + "toml": Toml, "yaml": Yaml, } diff --git a/config/config.go b/config/config.go index 511df1bc1..63bd4a663 100644 --- a/config/config.go +++ b/config/config.go @@ -111,6 +111,8 @@ type JSONBinaryStoreConfig struct { Indent int `yaml:"indent"` } +type TOMLStoreConfig struct{} + type YAMLStoreConfig struct { Indent int `yaml:"indent"` } @@ -120,6 +122,7 @@ type StoresConfig struct { INI INIStoreConfig `yaml:"ini"` JSONBinary JSONBinaryStoreConfig `yaml:"json_binary"` JSON JSONStoreConfig `yaml:"json"` + TOML TOMLStoreConfig `yaml:"toml"` YAML YAMLStoreConfig `yaml:"yaml"` } diff --git a/go.mod b/go.mod index 5bfcf8453..1be4030b5 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/go-wordwrap v1.0.1 github.com/ory/dockertest/v3 v3.12.0 + github.com/pelletier/go-toml/v2 v2.2.4 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.11.1 diff --git a/go.sum b/go.sum index 5ea6bf7ce..ce68b42af 100644 --- a/go.sum +++ b/go.sum @@ -293,6 +293,8 @@ github.com/opencontainers/runc v1.2.8 h1:RnEICeDReapbZ5lZEgHvj7E9Q3Eex9toYmaGBsb github.com/opencontainers/runc v1.2.8/go.mod h1:cC0YkmZcuvr+rtBZ6T7NBoVbMGNAdLa/21vIElJDOzI= github.com/ory/dockertest/v3 v3.12.0 h1:3oV9d0sDzlSQfHtIaB5k6ghUCVMVLpAY8hwrqoCyRCw= github.com/ory/dockertest/v3 v3.12.0/go.mod h1:aKNDTva3cp8dwOWwb9cWuX84aH5akkxXRvO7KCwWVjE= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/stores/stores.go b/stores/stores.go index 11e362a5d..b48e363e1 100644 --- a/stores/stores.go +++ b/stores/stores.go @@ -36,7 +36,7 @@ type SopsFile struct { // in the SOPS file by checking for nil. This way we can show the user a // helpful error message indicating that the metadata wasn't found, instead // of showing a cryptic parsing error - Metadata *Metadata `yaml:"sops" json:"sops" ini:"sops"` + Metadata *Metadata `yaml:"sops" json:"sops" ini:"sops" toml:"sops"` } // Metadata is stored in SOPS encrypted files, and it contains the information necessary to decrypt the file. @@ -44,83 +44,83 @@ type SopsFile struct { // in order to allow the binary format to stay backwards compatible over time, but at the same time allow the internal // representation SOPS uses to change over time. type Metadata struct { - ShamirThreshold int `yaml:"shamir_threshold,omitempty" json:"shamir_threshold,omitempty"` - KeyGroups []keygroup `yaml:"key_groups,omitempty" json:"key_groups,omitempty"` - KMSKeys []kmskey `yaml:"kms,omitempty" json:"kms,omitempty"` - GCPKMSKeys []gcpkmskey `yaml:"gcp_kms,omitempty" json:"gcp_kms,omitempty"` - HCKmsKeys []hckmskey `yaml:"hckms,omitempty" json:"hckms,omitempty"` - AzureKeyVaultKeys []azkvkey `yaml:"azure_kv,omitempty" json:"azure_kv,omitempty"` - VaultKeys []vaultkey `yaml:"hc_vault,omitempty" json:"hc_vault,omitempty"` - AgeKeys []agekey `yaml:"age,omitempty" json:"age,omitempty"` - LastModified string `yaml:"lastmodified" json:"lastmodified"` - MessageAuthenticationCode string `yaml:"mac" json:"mac"` - PGPKeys []pgpkey `yaml:"pgp,omitempty" json:"pgp,omitempty"` - UnencryptedSuffix string `yaml:"unencrypted_suffix,omitempty" json:"unencrypted_suffix,omitempty"` - EncryptedSuffix string `yaml:"encrypted_suffix,omitempty" json:"encrypted_suffix,omitempty"` - UnencryptedRegex string `yaml:"unencrypted_regex,omitempty" json:"unencrypted_regex,omitempty"` - EncryptedRegex string `yaml:"encrypted_regex,omitempty" json:"encrypted_regex,omitempty"` - UnencryptedCommentRegex string `yaml:"unencrypted_comment_regex,omitempty" json:"unencrypted_comment_regex,omitempty"` - EncryptedCommentRegex string `yaml:"encrypted_comment_regex,omitempty" json:"encrypted_comment_regex,omitempty"` - MACOnlyEncrypted bool `yaml:"mac_only_encrypted,omitempty" json:"mac_only_encrypted,omitempty"` - Version string `yaml:"version" json:"version"` + ShamirThreshold int `yaml:"shamir_threshold,omitempty" json:"shamir_threshold,omitempty" toml:"shamir_threshold,omitempty"` + KeyGroups []keygroup `yaml:"key_groups,omitempty" json:"key_groups,omitempty" toml:"key_groups,omitempty"` + KMSKeys []kmskey `yaml:"kms,omitempty" json:"kms,omitempty" toml:"kms,omitempty"` + GCPKMSKeys []gcpkmskey `yaml:"gcp_kms,omitempty" json:"gcp_kms,omitempty" toml:"gcp_kms,omitempty"` + HCKmsKeys []hckmskey `yaml:"hckms,omitempty" json:"hckms,omitempty" toml:"hckms,omitempty"` + AzureKeyVaultKeys []azkvkey `yaml:"azure_kv,omitempty" json:"azure_kv,omitempty" toml:"azure_kv,omitempty"` + VaultKeys []vaultkey `yaml:"hc_vault,omitempty" json:"hc_vault,omitempty" toml:"hc_vault,omitempty"` + AgeKeys []agekey `yaml:"age,omitempty" json:"age,omitempty" toml:"age,omitempty"` + LastModified string `yaml:"lastmodified" json:"lastmodified" toml:"lastmodified"` + MessageAuthenticationCode string `yaml:"mac" json:"mac" toml:"mac"` + PGPKeys []pgpkey `yaml:"pgp,omitempty" json:"pgp,omitempty" toml:"pgp,omitempty"` + UnencryptedSuffix string `yaml:"unencrypted_suffix,omitempty" json:"unencrypted_suffix,omitempty" toml:"unencrypted_suffix,omitempty"` + EncryptedSuffix string `yaml:"encrypted_suffix,omitempty" json:"encrypted_suffix,omitempty" toml:"encrypted_suffix,omitempty"` + UnencryptedRegex string `yaml:"unencrypted_regex,omitempty" json:"unencrypted_regex,omitempty" toml:"unencrypted_regex,omitempty"` + EncryptedRegex string `yaml:"encrypted_regex,omitempty" json:"encrypted_regex,omitempty" toml:"encrypted_regex,omitempty"` + UnencryptedCommentRegex string `yaml:"unencrypted_comment_regex,omitempty" json:"unencrypted_comment_regex,omitempty" toml:"unencrypted_comment_regex,omitempty"` + EncryptedCommentRegex string `yaml:"encrypted_comment_regex,omitempty" json:"encrypted_comment_regex,omitempty" toml:"encrypted_comment_regex,omitempty"` + MACOnlyEncrypted bool `yaml:"mac_only_encrypted,omitempty" json:"mac_only_encrypted,omitempty" toml:"mac_only_encrypted,omitempty"` + Version string `yaml:"version" json:"version" toml:"version"` } type keygroup struct { - PGPKeys []pgpkey `yaml:"pgp,omitempty" json:"pgp,omitempty"` - KMSKeys []kmskey `yaml:"kms,omitempty" json:"kms,omitempty"` - GCPKMSKeys []gcpkmskey `yaml:"gcp_kms,omitempty" json:"gcp_kms,omitempty"` - HCKmsKeys []hckmskey `yaml:"hckms,omitempty" json:"hckms,omitempty"` - AzureKeyVaultKeys []azkvkey `yaml:"azure_kv,omitempty" json:"azure_kv,omitempty"` - VaultKeys []vaultkey `yaml:"hc_vault" json:"hc_vault"` - AgeKeys []agekey `yaml:"age" json:"age"` + PGPKeys []pgpkey `yaml:"pgp,omitempty" json:"pgp,omitempty" toml:"pgp,omitempty"` + KMSKeys []kmskey `yaml:"kms,omitempty" json:"kms,omitempty" toml:"kms,omitempty"` + GCPKMSKeys []gcpkmskey `yaml:"gcp_kms,omitempty" json:"gcp_kms,omitempty" toml:"gcp_kms,omitempty"` + HCKmsKeys []hckmskey `yaml:"hckms,omitempty" json:"hckms,omitempty" toml:"hckms,omitempty"` + AzureKeyVaultKeys []azkvkey `yaml:"azure_kv,omitempty" json:"azure_kv,omitempty" toml:"azure_kv,omitempty"` + VaultKeys []vaultkey `yaml:"hc_vault" json:"hc_vault" toml:"hc_vault"` + AgeKeys []agekey `yaml:"age" json:"age" toml:"age"` } type pgpkey struct { - CreatedAt string `yaml:"created_at" json:"created_at"` - EncryptedDataKey string `yaml:"enc" json:"enc"` - Fingerprint string `yaml:"fp" json:"fp"` + CreatedAt string `yaml:"created_at" json:"created_at" toml:"created_at"` + EncryptedDataKey string `yaml:"enc" json:"enc" toml:"enc"` + Fingerprint string `yaml:"fp" json:"fp" toml:"fp"` } type kmskey struct { - Arn string `yaml:"arn" json:"arn"` - Role string `yaml:"role,omitempty" json:"role,omitempty"` - Context map[string]*string `yaml:"context,omitempty" json:"context,omitempty"` - CreatedAt string `yaml:"created_at" json:"created_at"` - EncryptedDataKey string `yaml:"enc" json:"enc"` - AwsProfile string `yaml:"aws_profile" json:"aws_profile"` + Arn string `yaml:"arn" json:"arn" toml:"arn"` + Role string `yaml:"role,omitempty" json:"role,omitempty" toml:"role,omitempty"` + Context map[string]*string `yaml:"context,omitempty" json:"context,omitempty" toml:"context,omitempty"` + CreatedAt string `yaml:"created_at" json:"created_at" toml:"created_at"` + EncryptedDataKey string `yaml:"enc" json:"enc" toml:"enc"` + AwsProfile string `yaml:"aws_profile" json:"aws_profile" toml:"aws_profile"` } type gcpkmskey struct { - ResourceID string `yaml:"resource_id" json:"resource_id"` - CreatedAt string `yaml:"created_at" json:"created_at"` - EncryptedDataKey string `yaml:"enc" json:"enc"` + ResourceID string `yaml:"resource_id" json:"resource_id" toml:"resource_id"` + CreatedAt string `yaml:"created_at" json:"created_at" toml:"created_at"` + EncryptedDataKey string `yaml:"enc" json:"enc" toml:"enc"` } type vaultkey struct { - VaultAddress string `yaml:"vault_address" json:"vault_address"` - EnginePath string `yaml:"engine_path" json:"engine_path"` - KeyName string `yaml:"key_name" json:"key_name"` - CreatedAt string `yaml:"created_at" json:"created_at"` - EncryptedDataKey string `yaml:"enc" json:"enc"` + VaultAddress string `yaml:"vault_address" json:"vault_address" toml:"vault_address"` + EnginePath string `yaml:"engine_path" json:"engine_path" toml:"engine_path"` + KeyName string `yaml:"key_name" json:"key_name" toml:"key_name"` + CreatedAt string `yaml:"created_at" json:"created_at" toml:"created_at"` + EncryptedDataKey string `yaml:"enc" json:"enc" toml:"enc"` } type azkvkey struct { - VaultURL string `yaml:"vault_url" json:"vault_url"` - Name string `yaml:"name" json:"name"` - Version string `yaml:"version" json:"version"` - CreatedAt string `yaml:"created_at" json:"created_at"` - EncryptedDataKey string `yaml:"enc" json:"enc"` + VaultURL string `yaml:"vault_url" json:"vault_url" toml:"vault_url"` + Name string `yaml:"name" json:"name" toml:"name"` + Version string `yaml:"version" json:"version" toml:"version"` + CreatedAt string `yaml:"created_at" json:"created_at" toml:"created_at"` + EncryptedDataKey string `yaml:"enc" json:"enc" toml:"enc"` } type agekey struct { - Recipient string `yaml:"recipient" json:"recipient"` - EncryptedDataKey string `yaml:"enc" json:"enc"` + Recipient string `yaml:"recipient" json:"recipient" toml:"recipient"` + EncryptedDataKey string `yaml:"enc" json:"enc" toml:"enc"` } type hckmskey struct { - KeyID string `yaml:"key_id" json:"key_id"` - CreatedAt string `yaml:"created_at" json:"created_at"` - EncryptedDataKey string `yaml:"enc" json:"enc"` + KeyID string `yaml:"key_id" json:"key_id" toml:"key_id"` + CreatedAt string `yaml:"created_at" json:"created_at" toml:"created_at"` + EncryptedDataKey string `yaml:"enc" json:"enc" toml:"enc"` } // MetadataFromInternal converts an internal SOPS metadata representation to a representation appropriate for storage diff --git a/stores/toml/store.go b/stores/toml/store.go new file mode 100644 index 000000000..73230532d --- /dev/null +++ b/stores/toml/store.go @@ -0,0 +1,300 @@ +package toml //import "github.com/getsops/sops/v3/stores/toml" + +import ( + "bytes" + "errors" + "fmt" + "sort" + + "github.com/getsops/sops/v3" + "github.com/getsops/sops/v3/config" + "github.com/getsops/sops/v3/stores" + + "github.com/pelletier/go-toml/v2" +) + +// Store handles storage of TOML data. +type Store struct { + config *config.TOMLStoreConfig +} + +func NewStore(c *config.TOMLStoreConfig) *Store { + return &Store{config: c} +} + +func (store *Store) Name() string { + return "toml" +} + +var errUnexpectedValue = errors.New("unexpected value") + +// mapToTreeBranch converts a map[string]any to a sops.TreeBranch. +func mapToTreeBranch(m map[string]any) (sops.TreeBranch, error) { + // Separate keys by type: simple values first, then complex types + // (tables/arrays). + var simpleKeys, complexKeys []string + for k, v := range m { + switch v.(type) { + case map[string]any: + complexKeys = append(complexKeys, k) + case []any: + // Check if it's an array of tables + if arr, ok := v.([]any); ok && len(arr) > 0 { + if _, isMap := arr[0].(map[string]any); isMap { + complexKeys = append(complexKeys, k) + } else { + simpleKeys = append(simpleKeys, k) + } + } else { + simpleKeys = append(simpleKeys, k) + } + default: + simpleKeys = append(simpleKeys, k) + } + } + + // Sort each group independently. + sortKeysNaturally(simpleKeys) + sortKeysNaturally(complexKeys) + + // Combine: simple values first, then complex types. + keys := append(simpleKeys, complexKeys...) + + var branch sops.TreeBranch + for _, k := range keys { + v := m[k] + value, err := anyToTreeItemValue(v) + if err != nil { + return nil, fmt.Errorf("mapToTreeBranch: %w - %v, %s", errUnexpectedValue, v, k) + } + branch = append(branch, sops.TreeItem{ + Key: k, + Value: value, + }) + } + return branch, nil +} + +// sortKeysNaturally sorts keys lexicographically for deterministic output. +func sortKeysNaturally(keys []string) { + sort.Strings(keys) +} + +// anyToTreeItemValue converts an any value from TOML unmarshaling +// to a sops TreeItem value. +func anyToTreeItemValue(v any) (any, error) { + switch val := v.(type) { + case map[string]any: + return mapToTreeBranch(val) + case []any: + // Check if it's an array of maps (array of tables in TOML). + if len(val) > 0 { + if _, ok := val[0].(map[string]any); ok { + // Yes, it's an array of tables. + var branches []any + for _, item := range val { + if m, ok := item.(map[string]any); ok { + branch, err := mapToTreeBranch(m) + if err != nil { + return nil, err + } + branches = append(branches, branch) + } else { + return nil, fmt.Errorf("anyToTreeItemValue: expected map in array, got %T", item) + } + } + return branches, nil + } + } + return val, nil + default: + return val, nil + } +} + +// treeBranchToMap converts a sops.TreeBranch to a map[string]any. +func treeBranchToMap(branch sops.TreeBranch) (map[string]any, error) { + m := make(map[string]any) + for _, item := range branch { + key, ok := item.Key.(string) + if !ok { + // Skip non-string keys (like comments). + continue + } + value, err := treeItemValueToInterface(item.Value) + if err != nil { + return nil, err + } + m[key] = value + } + return m, nil +} + +// treeItemValueToInterface converts a sops TreeItem value to an any +// suitable for TOML marshaling. +func treeItemValueToInterface(value any) (any, error) { + switch val := value.(type) { + case sops.TreeBranch: + return treeBranchToMap(val) + case []any: + // Check if it's an array of TreeBranches. + if len(val) > 0 { + if _, ok := val[0].(sops.TreeBranch); ok { + var result []any + for _, item := range val { + if branch, ok := item.(sops.TreeBranch); ok { + m, err := treeBranchToMap(branch) + if err != nil { + return nil, err + } + result = append(result, m) + } else { + return nil, fmt.Errorf("treeItemValueToInterface: expected TreeBranch in array, got %T", item) + } + } + return result, nil + } + } + return val, nil + default: + return val, nil + } +} + +// LoadEncryptedFile loads the contents of an encrypted toml file onto a +// sops.Tree runtime object. +func (s *Store) LoadEncryptedFile(in []byte) (sops.Tree, error) { + var data map[string]any + if err := toml.Unmarshal(in, &data); err != nil { + return sops.Tree{}, fmt.Errorf("error unmarshalling input toml: %w", err) + } + + // Because we don't know what fields the input file will have, we have to + // load the file in two steps. + // + // First, we load the file's metadata, the structure of which is known. + metadataHolder := stores.SopsFile{} + if err := toml.Unmarshal(in, &metadataHolder); err != nil { + return sops.Tree{}, fmt.Errorf("error unmarshalling input toml: %w", err) + } + + if metadataHolder.Metadata == nil { + return sops.Tree{}, sops.MetadataNotFound + } + + metadata, err := metadataHolder.Metadata.ToInternal() + if err != nil { + return sops.Tree{}, fmt.Errorf("error unmarshalling input toml: %w", err) + } + + // Second, we load the rest of the file's contents into a generic tree + // structure. + branch, err := mapToTreeBranch(data) + if err != nil { + return sops.Tree{}, fmt.Errorf("error transforming toml data: %w", err) + } + + return sops.Tree{ + Branches: sops.TreeBranches{branch}, + Metadata: metadata, + }, nil +} + +// LoadPlainFile loads the contents of a plaintext toml file onto a +// sops.Tree runtime object. +func (s *Store) LoadPlainFile(in []byte) (sops.TreeBranches, error) { + var data map[string]any + if err := toml.Unmarshal(in, &data); err != nil { + return nil, fmt.Errorf("error unmarshalling input toml: %w", err) + } + + branch, err := mapToTreeBranch(data) + if err != nil { + return nil, fmt.Errorf("error transforming toml data: %w", err) + } + + return sops.TreeBranches{branch}, nil +} + +var errTOMLUniqueDocument = errors.New("toml can only contain 1 document") + +// EmitEncryptedFile returns the encrypted bytes of the toml file corresponding to a +// sops.Tree runtime object. +func (s *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) { + if len(in.Branches) != 1 { + return nil, errTOMLUniqueDocument + } + + data, err := treeBranchToMap(in.Branches[0]) + if err != nil { + return nil, fmt.Errorf("error converting tree branch: %w", err) + } + + data["sops"] = stores.MetadataFromInternal(in.Metadata) + + return s.marshalTOML(data) +} + +// EmitPlainFile returns the plaintext bytes of the toml file corresponding to a +// sops.TreeBranches runtime object. +func (s *Store) EmitPlainFile(branches sops.TreeBranches) ([]byte, error) { + if len(branches) != 1 { + return nil, errTOMLUniqueDocument + } + + data, err := treeBranchToMap(branches[0]) + if err != nil { + return nil, fmt.Errorf("emit plain file: %w", err) + } + + return s.marshalTOML(data) +} + +// marshalTOML marshals data to TOML with custom formatting +func (s *Store) marshalTOML(data any) ([]byte, error) { + buf := bytes.Buffer{} + encoder := toml.NewEncoder(&buf) + encoder.SetIndentTables(true) + if err := encoder.Encode(data); err != nil { + return nil, fmt.Errorf("error marshalling toml: %w", err) + } + + // Replace single quotes with double quotes for string values. + result := bytes.ReplaceAll(buf.Bytes(), []byte("'"), []byte("\"")) + return result, nil +} + +// EmitValue returns bytes corresponding to a single encoded value +// in a generic any object. +func (s *Store) EmitValue(v any) ([]byte, error) { + switch val := v.(type) { + case sops.TreeBranch: + data, err := treeBranchToMap(val) + if err != nil { + return nil, fmt.Errorf("emit value: %w", err) + } + + return s.marshalTOML(data) + case string: + // For strings, return quoted value. + return []byte(fmt.Sprintf("%q", val)), nil + default: + // For simple values, format them appropriately. + return []byte(fmt.Sprintf("%v", val)), nil + } +} + +// EmitExample returns the bytes corresponding to an example complex tree. +func (s *Store) EmitExample() []byte { + bytes, err := s.EmitPlainFile(stores.ExampleComplexTree.Branches) + if err != nil { + panic(err) + } + + return bytes +} + +// HasSopsTopLevelKey checks whether a top-level "sops" key exists. +func (store *Store) HasSopsTopLevelKey(branch sops.TreeBranch) bool { + return stores.HasSopsTopLevelKey(branch) +} diff --git a/stores/toml/store_test.go b/stores/toml/store_test.go new file mode 100644 index 000000000..95cf41b20 --- /dev/null +++ b/stores/toml/store_test.go @@ -0,0 +1,726 @@ +package toml + +import ( + "reflect" + "testing" + + "github.com/getsops/sops/v3" + "github.com/pelletier/go-toml/v2" + "github.com/stretchr/testify/assert" +) + +func testPlain() []byte { + return []byte( + `# 0 comment +0 = 0 +1 = 1 # 1 comment + +[2] + 21 = [21.1, 21.2] # 21 comment + 22 = 22 + +[2.1] + 211 = 211 + +[[2.1.1]] + 2111 = 2111 + +[[2.1.1]] + 2112 = 2112 + +[[3]] + 31 = "thirty one" + +[[3]] + 32 = "thirty two" + +# 4 comment +[4] + # 41 comment + 41 = 41 +`) +} + +func testPlainNoComment() []byte { + return []byte( + `0 = 0 +1 = 1 + +[2] + 21 = [21.1, 21.2] + 22 = 22 + + [2.1] + 211 = 211 + + [[2.1.1]] + 2111 = 2111 + + [[2.1.1]] + 2112 = 2112 + +[[3]] + 31 = "thirty one" + +[[3]] + 32 = "thirty two" + +[4] + 41 = 41 +`) +} + +func testTreeBranches() sops.TreeBranches { + return sops.TreeBranches{ + sops.TreeBranch{ + // NOT IMPL on go-toml yet + // sops.TreeItem{ + // Key: sops.Comment{ + // Value: " 0 comment", + // }, + // Value: interface{}(nil), + // }, + sops.TreeItem{ + Key: "0", + Value: int64(0), + }, + sops.TreeItem{ + Key: "1", + Value: int64(1), + }, + // NOT IMPL on go-toml yet + // sops.TreeItem{ + // Key: sops.Comment{ + // Value: " 1 comment", + // }, + // Value: interface{}(nil), + // }, + sops.TreeItem{ + Key: "2", + Value: sops.TreeBranch{ + // NOT IMPL on go-toml yet + // sops.TreeItem{ + // Key: sops.Comment{ + // Value: " 21 comment", + // }, + // Value: interface{}(nil), + // }, + sops.TreeItem{ + Key: "21", + Value: []any{ + 21.1, + 21.2, + }, + }, + sops.TreeItem{ + Key: "22", + Value: int64(22), + }, + sops.TreeItem{ + Key: "1", + Value: sops.TreeBranch{ + sops.TreeItem{ + Key: "211", + Value: int64(211), + }, + sops.TreeItem{ + Key: "1", + Value: []any{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "2111", + Value: int64(2111), + }, + }, + sops.TreeBranch{ + sops.TreeItem{ + Key: "2112", + Value: int64(2112), + }, + }, + }, + }, + }, + }, + }, + }, + sops.TreeItem{ + Key: "3", + Value: []any{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "31", + Value: "thirty one", + }, + }, + sops.TreeBranch{ + sops.TreeItem{ + Key: "32", + Value: "thirty two", + }, + // NOT IMPL on go-toml yet + // sops.TreeItem{ + // Key: sops.Comment{ + // Value: " 4 comment", + // }, + // Value: any(nil), + // }, + }, + }, + }, + // NOT IMPL on go-toml yet + // sops.TreeItem{ + // Key: sops.Comment{ + // Value: " 41 comment", + // }, + // Value: any(nil), + // }, + sops.TreeItem{ + Key: "4", + Value: sops.TreeBranch{ + sops.TreeItem{ + Key: "41", + Value: int64(41), + }, + }, + }, + }, + } +} + +func TestLoadPlainFile(t *testing.T) { + t.Parallel() + + actualBranches, err := (&Store{}).LoadPlainFile(testPlain()) + if err != nil { + t.Errorf("expected no error, got: %v", err) + + return + } + + expectedBranches := testTreeBranches() + + if !reflect.DeepEqual(expectedBranches, actualBranches) { + t.Errorf("expected\n%#v\ngot\n%#v", expectedBranches, actualBranches) + + return + } +} + +func TestEmitPlainFile(t *testing.T) { + t.Parallel() + + branches := testTreeBranches() + + bytes, err := (&Store{}).EmitPlainFile(branches) + if err != nil { + t.Errorf("expected no error, got: %v", err) + + return + } + + if !reflect.DeepEqual(testPlainNoComment(), bytes) { + t.Errorf("expected\n\n-%s-\n\ngot\n\n-%s-", testPlainNoComment(), bytes) + + return + } +} + +func TestEmitValueString(t *testing.T) { + t.Parallel() + + bytes, err := (&Store{}).EmitValue("hello") + assert.Nil(t, err) + assert.Equal(t, []byte("\"hello\""), bytes) +} + +func TestUnmarshalMetadataFromNonSOPSFile(t *testing.T) { + t.Parallel() + + data := []byte(`hello = 2`) + _, err := (&Store{}).LoadEncryptedFile(data) + assert.Equal(t, sops.MetadataNotFound, err) +} + +func TestLoadPlainFileRoundTrip(t *testing.T) { + t.Parallel() + + // Load the plain file + branches, err := (&Store{}).LoadPlainFile(testPlain()) + assert.Nil(t, err) + + // Emit it back + bytes, err := (&Store{}).EmitPlainFile(branches) + assert.Nil(t, err) + + // Load again to verify round-trip works + branches2, err := (&Store{}).LoadPlainFile(bytes) + assert.Nil(t, err) + + // Should match the original loaded data + assert.Equal(t, branches, branches2) +} + +func TestEmitValueTreeBranch(t *testing.T) { + t.Parallel() + + branch := sops.TreeBranch{ + sops.TreeItem{ + Key: "key1", + Value: "value1", + }, + sops.TreeItem{ + Key: "key2", + Value: int64(42), + }, + } + + bytes, err := (&Store{}).EmitValue(branch) + assert.Nil(t, err) + + // Should be valid TOML + var result map[string]any + err = toml.Unmarshal(bytes, &result) + assert.Nil(t, err) + assert.Equal(t, "value1", result["key1"]) + assert.Equal(t, int64(42), result["key2"]) +} + +func TestEmitValueNumber(t *testing.T) { + t.Parallel() + + bytes, err := (&Store{}).EmitValue(42) + assert.Nil(t, err) + assert.Equal(t, []byte("42"), bytes) +} + +func TestEmitValueBool(t *testing.T) { + t.Parallel() + + bytes, err := (&Store{}).EmitValue(true) + assert.Nil(t, err) + assert.Equal(t, []byte("true"), bytes) +} + +func TestEmpty(t *testing.T) { + t.Parallel() + + // Empty TOML file - TOML treats empty input as an empty map (one branch with no items) + branches, err := (&Store{}).LoadPlainFile([]byte(``)) + assert.Nil(t, err) + assert.Equal(t, 1, len(branches)) + assert.Equal(t, 0, len(branches[0])) + + bytes, err := (&Store{}).EmitPlainFile(branches) + assert.Nil(t, err) + assert.Equal(t, ``, string(bytes)) +} + +func TestEmptyTable(t *testing.T) { + t.Parallel() + + data := []byte(`[empty] +`) + + branches, err := (&Store{}).LoadPlainFile(data) + assert.Nil(t, err) + assert.Equal(t, 1, len(branches)) + assert.Equal(t, 1, len(branches[0])) + + // Re-emit and verify + bytes, err := (&Store{}).EmitPlainFile(branches) + assert.Nil(t, err) + + // Should contain the empty table + var result map[string]any + err = toml.Unmarshal(bytes, &result) + assert.Nil(t, err) + assert.Contains(t, result, "empty") +} + +func TestHasSopsTopLevelKey(t *testing.T) { + t.Parallel() + + ok := (&Store{}).HasSopsTopLevelKey(sops.TreeBranch{ + sops.TreeItem{ + Key: "sops", + Value: "value", + }, + }) + assert.True(t, ok) + + ok = (&Store{}).HasSopsTopLevelKey(sops.TreeBranch{ + sops.TreeItem{ + Key: "sops_", + Value: "value", + }, + }) + assert.False(t, ok) + + ok = (&Store{}).HasSopsTopLevelKey(sops.TreeBranch{ + sops.TreeItem{ + Key: "other", + Value: "value", + }, + }) + assert.False(t, ok) +} + +func TestLoadEncryptedFile(t *testing.T) { + t.Parallel() + + // Create a sample encrypted TOML with metadata + data := []byte(`key1 = "value1" +key2 = 42 + +[sops] + version = "3.7.0" + mac = "ENC[AES256_GCM,data:abc123,iv:def456,tag:ghi789,type:str]" + lastmodified = "2023-01-01T00:00:00Z" + + [[sops.kms]] + arn = "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012" + created_at = "2023-01-01T00:00:00Z" + enc = "encrypted-data-key" +`) + + tree, err := (&Store{}).LoadEncryptedFile(data) + assert.Nil(t, err) + + // Verify metadata was loaded + assert.NotNil(t, tree.Metadata) + assert.Equal(t, "3.7.0", tree.Metadata.Version) + assert.Equal(t, "ENC[AES256_GCM,data:abc123,iv:def456,tag:ghi789,type:str]", tree.Metadata.MessageAuthenticationCode) + + // Verify data was loaded + assert.Equal(t, 1, len(tree.Branches)) + // The branch should contain key1, key2, and sops + assert.GreaterOrEqual(t, len(tree.Branches[0]), 2) +} + +func TestEmitEncryptedFile(t *testing.T) { + t.Parallel() + + // Create a simple tree with metadata + tree := sops.Tree{ + Branches: sops.TreeBranches{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "key1", + Value: "value1", + }, + sops.TreeItem{ + Key: "key2", + Value: int64(42), + }, + }, + }, + Metadata: sops.Metadata{ + Version: "3.7.0", + MessageAuthenticationCode: "test-mac", + }, + } + + bytes, err := (&Store{}).EmitEncryptedFile(tree) + assert.Nil(t, err) + + // Should be valid TOML + var result map[string]any + err = toml.Unmarshal(bytes, &result) + assert.Nil(t, err) + + // Should contain data + assert.Equal(t, "value1", result["key1"]) + assert.Equal(t, int64(42), result["key2"]) + + // Should contain sops metadata + assert.Contains(t, result, "sops") + sopsMap := result["sops"].(map[string]any) + assert.Equal(t, "3.7.0", sopsMap["version"]) + assert.Equal(t, "test-mac", sopsMap["mac"]) +} + +func TestNestedStructures(t *testing.T) { + t.Parallel() + + data := []byte(`[server] + host = "localhost" + port = 8080 + + [server.database] + name = "mydb" + user = "admin" + +[[users]] + name = "Alice" + role = "admin" + +[[users]] + name = "Bob" + role = "user" +`) + + branches, err := (&Store{}).LoadPlainFile(data) + assert.Nil(t, err) + assert.Equal(t, 1, len(branches)) + + // Re-emit and verify structure is preserved + bytes, err := (&Store{}).EmitPlainFile(branches) + assert.Nil(t, err) + + var result map[string]any + err = toml.Unmarshal(bytes, &result) + assert.Nil(t, err) + + // Verify nested table + assert.Contains(t, result, "server") + server := result["server"].(map[string]any) + assert.Equal(t, "localhost", server["host"]) + assert.Equal(t, int64(8080), server["port"]) + + db := server["database"].(map[string]any) + assert.Equal(t, "mydb", db["name"]) + assert.Equal(t, "admin", db["user"]) + + // Verify array of tables + assert.Contains(t, result, "users") + users := result["users"].([]any) + assert.Equal(t, 2, len(users)) + + user1 := users[0].(map[string]any) + assert.Equal(t, "Alice", user1["name"]) + assert.Equal(t, "admin", user1["role"]) + + user2 := users[1].(map[string]any) + assert.Equal(t, "Bob", user2["name"]) + assert.Equal(t, "user", user2["role"]) +} + +func TestErrorOnMultipleBranches(t *testing.T) { + t.Parallel() + + branches := sops.TreeBranches{ + sops.TreeBranch{ + sops.TreeItem{Key: "key1", Value: "value1"}, + }, + sops.TreeBranch{ + sops.TreeItem{Key: "key2", Value: "value2"}, + }, + } + + // TOML can only contain one document + _, err := (&Store{}).EmitPlainFile(branches) + assert.NotNil(t, err) + assert.Equal(t, errTOMLUniqueDocument, err) +} + +func TestArraysOfArrays(t *testing.T) { + t.Parallel() + + data := []byte(`arrays = [[["foo", "bar"], ["baz"]], [["qux"]]] +`) + + branches, err := (&Store{}).LoadPlainFile(data) + assert.Nil(t, err) + assert.Equal(t, 1, len(branches)) + + // Re-emit and verify structure is preserved + bytes, err := (&Store{}).EmitPlainFile(branches) + assert.Nil(t, err) + + var result map[string]any + err = toml.Unmarshal(bytes, &result) + assert.Nil(t, err) + + arrays := result["arrays"].([]any) + assert.Equal(t, 2, len(arrays)) + + // First nested array + arr0 := arrays[0].([]any) + assert.Equal(t, 2, len(arr0)) + arr00 := arr0[0].([]any) + assert.Equal(t, []any{"foo", "bar"}, arr00) + arr01 := arr0[1].([]any) + assert.Equal(t, []any{"baz"}, arr01) + + // Second nested array + arr1 := arrays[1].([]any) + assert.Equal(t, 1, len(arr1)) + arr10 := arr1[0].([]any) + assert.Equal(t, []any{"qux"}, arr10) +} + +func TestSpecialCharacters(t *testing.T) { + t.Parallel() + + // Test that TOML handles various special characters + data := []byte(`simple = "hello world" +with_space = "hello world" +number = 42 +`) + + // Load the TOML + branches, err := (&Store{}).LoadPlainFile(data) + assert.Nil(t, err) + assert.Equal(t, 1, len(branches)) + + // Emit and verify round-trip works + bytes, err := (&Store{}).EmitPlainFile(branches) + assert.Nil(t, err) + + // Re-load to verify round-trip + branches2, err := (&Store{}).LoadPlainFile(bytes) + assert.Nil(t, err) + + // Should be equal after round-trip + assert.Equal(t, branches, branches2) +} + +func TestEmitValueNonString(t *testing.T) { + t.Parallel() + + // TreeBranch should work + branch := sops.TreeBranch{ + sops.TreeItem{Key: "key", Value: "value"}, + } + _, err := (&Store{}).EmitValue(branch) + assert.Nil(t, err) + + // Other complex types should also work + _, err = (&Store{}).EmitValue(42) + assert.Nil(t, err) + + _, err = (&Store{}).EmitValue(true) + assert.Nil(t, err) +} + +func TestMixedArrayTypes(t *testing.T) { + t.Parallel() + + // TOML requires arrays to have consistent types + // Test that we can handle arrays with mixed simple types + data := []byte(`mixed_numbers = [1, 2.5, 3] +strings = ["hello", "world"] +`) + + branches, err := (&Store{}).LoadPlainFile(data) + assert.Nil(t, err) + + // Re-emit and verify + bytes, err := (&Store{}).EmitPlainFile(branches) + assert.Nil(t, err) + + var result map[string]any + err = toml.Unmarshal(bytes, &result) + assert.Nil(t, err) + + // Verify mixed_numbers array + nums := result["mixed_numbers"].([]any) + assert.Equal(t, 3, len(nums)) + + // Verify strings array + strs := result["strings"].([]any) + assert.Equal(t, 2, len(strs)) + assert.Equal(t, "hello", strs[0]) + assert.Equal(t, "world", strs[1]) +} + +func TestDateTimeValues(t *testing.T) { + t.Parallel() + + // TOML has native datetime support + data := []byte(`datetime = 1979-05-27T07:32:00Z +date = 1979-05-27 +`) + + branches, err := (&Store{}).LoadPlainFile(data) + assert.Nil(t, err) + assert.Equal(t, 1, len(branches)) + + // Re-emit and verify structure is preserved + bytes, err := (&Store{}).EmitPlainFile(branches) + assert.Nil(t, err) + + var result map[string]any + err = toml.Unmarshal(bytes, &result) + assert.Nil(t, err) + + // Should contain datetime fields + assert.Contains(t, result, "datetime") + assert.Contains(t, result, "date") +} + +func TestInlineTable(t *testing.T) { + t.Parallel() + + data := []byte(`name = { first = "Tom", last = "Preston-Werner" } +point = { x = 1, y = 2 } +`) + + branches, err := (&Store{}).LoadPlainFile(data) + assert.Nil(t, err) + assert.Equal(t, 1, len(branches)) + + // Re-emit and verify structure is preserved + bytes, err := (&Store{}).EmitPlainFile(branches) + assert.Nil(t, err) + + var result map[string]any + err = toml.Unmarshal(bytes, &result) + assert.Nil(t, err) + + // Verify inline tables are loaded correctly + name := result["name"].(map[string]any) + assert.Equal(t, "Tom", name["first"]) + assert.Equal(t, "Preston-Werner", name["last"]) + + point := result["point"].(map[string]any) + assert.Equal(t, int64(1), point["x"]) + assert.Equal(t, int64(2), point["y"]) +} + +func TestBooleanValues(t *testing.T) { + t.Parallel() + + data := []byte(`enabled = true +disabled = false +`) + + branches, err := (&Store{}).LoadPlainFile(data) + assert.Nil(t, err) + + // Verify boolean values are preserved + bytes, err := (&Store{}).EmitPlainFile(branches) + assert.Nil(t, err) + + var result map[string]any + err = toml.Unmarshal(bytes, &result) + assert.Nil(t, err) + + assert.Equal(t, true, result["enabled"]) + assert.Equal(t, false, result["disabled"]) +} + +func TestFloatValues(t *testing.T) { + t.Parallel() + + data := []byte(`pi = 3.14159 +negative = -0.01 +exponent = 5e+22 +`) + + branches, err := (&Store{}).LoadPlainFile(data) + assert.Nil(t, err) + + // Re-emit and verify + bytes, err := (&Store{}).EmitPlainFile(branches) + assert.Nil(t, err) + + var result map[string]any + err = toml.Unmarshal(bytes, &result) + assert.Nil(t, err) + + assert.Equal(t, 3.14159, result["pi"]) + assert.Equal(t, -0.01, result["negative"]) + assert.Equal(t, 5e+22, result["exponent"]) +}