Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ deb-pkg: install
mkdir -p tmppkg/usr/local/bin
cp $$GOPATH/bin/sops tmppkg/usr/local/bin/
fpm -C tmppkg -n sops --license MPL2.0 --vendor mozilla \
--description "Sops is an editor of encrypted files that supports YAML, JSON and BINARY formats and encrypts with AWS KMS and PGP." \
--description "Sops is an editor of encrypted files that supports YAML, JSON, TOML and BINARY formats and encrypts with AWS KMS and PGP." \
-m "Julien Vehent <jvehent+sops@mozilla.com>" \
--url https://go.mozilla.org/sops \
--architecture x86_64 \
Expand All @@ -65,7 +65,7 @@ rpm-pkg: install
mkdir -p tmppkg/usr/local/bin
cp $$GOPATH/bin/sops tmppkg/usr/local/bin/
fpm -C tmppkg -n sops --license MPL2.0 --vendor mozilla \
--description "Sops is an editor of encrypted files that supports YAML, JSON and BINARY formats and encrypts with AWS KMS and PGP." \
--description "Sops is an editor of encrypted files that supports YAML, JSON, TOML and BINARY formats and encrypts with AWS KMS and PGP." \
-m "Julien Vehent <jvehent+sops@mozilla.com>" \
--url https://go.mozilla.org/sops \
--architecture x86_64 \
Expand All @@ -80,7 +80,7 @@ else
mkdir -p tmppkg/usr/local/bin
cp $$GOPATH/bin/sops tmppkg/usr/local/bin/
fpm -C tmppkg -n sops --license MPL2.0 --vendor mozilla \
--description "Sops is an editor of encrypted files that supports YAML, JSON and BINARY formats and encrypts with AWS KMS and PGP." \
--description "Sops is an editor of encrypted files that supports YAML, JSON, TOML and BINARY formats and encrypts with AWS KMS and PGP." \
-m "Julien Vehent <jvehent+sops@mozilla.com>" \
--url https://go.mozilla.org/sops \
--architecture x86_64 \
Expand Down
10 changes: 5 additions & 5 deletions README.rst
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
SOPS: Secrets OPerationS
========================

**sops** is an editor of encrypted files that supports YAML, JSON, ENV, INI and BINARY
**sops** is an editor of encrypted files that supports YAML, JSON, TOML, ENV, INI and BINARY
formats and encrypts with AWS KMS, GCP KMS, Azure Key Vault and PGP.
(`demo <https://www.youtube.com/watch?v=YTEVyLXFiq0>`_)

Expand Down Expand Up @@ -980,11 +980,11 @@ Below is an example of publishing to Vault (using token auth with a local dev in
Important information on types
------------------------------

YAML and JSON type extensions
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
File type extensions
~~~~~~~~~~~~~~~~~~~~

``sops`` uses the file extension to decide which encryption method to use on the file
content. ``YAML``, ``JSON``, ``ENV``, and ``INI`` files are treated as trees of data, and key/values are
content. ``YAML``, ``JSON``, ``TOML``, ``ENV``, and ``INI`` files are treated as trees of data, and key/values are
extracted from the files to only encrypt the leaf values. The tree structure is also
used to check the integrity of the file.

Expand Down Expand Up @@ -1028,7 +1028,7 @@ This file will not work in ``sops``:
``sops`` uses the path to a value as additional data in the AEAD encryption, and thus
dynamic paths generated by anchors break the authentication step.

JSON and TEXT file types do not support anchors and thus have no such limitation.
Other file formats do not support anchors and thus have no such limitation.

YAML Streams
~~~~~~~~~~~~
Expand Down
5 changes: 5 additions & 0 deletions aes/cipher.go
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,8 @@ func (c Cipher) Decrypt(ciphertext string, key []byte, additionalData string) (p
plaintext = decryptedValue
case "int":
plaintext, err = strconv.Atoi(decryptedValue)
case "int64":
plaintext, err = strconv.ParseInt(decryptedValue, 10, 64)
case "float":
plaintext, err = strconv.ParseFloat(decryptedValue, 64)
case "bytes":
Expand Down Expand Up @@ -165,6 +167,9 @@ func (c Cipher) Encrypt(plaintext interface{}, key []byte, additionalData string
case int:
encryptedType = "int"
plainBytes = []byte(strconv.Itoa(value))
case int64:
encryptedType = "int64"
plainBytes = []byte(strconv.FormatInt(value, 10))
case float64:
encryptedType = "float"
// The Python version encodes floats without padding 0s after the decimal point.
Expand Down
8 changes: 8 additions & 0 deletions cmd/sops/common/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"go.mozilla.org/sops/stores/dotenv"
"go.mozilla.org/sops/stores/ini"
"go.mozilla.org/sops/stores/json"
"go.mozilla.org/sops/stores/toml"
"go.mozilla.org/sops/stores/yaml"
"go.mozilla.org/sops/version"
"golang.org/x/crypto/ssh/terminal"
Expand Down Expand Up @@ -129,6 +130,11 @@ func IsJSONFile(path string) bool {
return strings.HasSuffix(path, ".json")
}

// IsTOMLFile returns true if a given file path corresponds to a TOML file
func IsTOMLFile(path string) bool {
return strings.HasSuffix(path, ".toml")
}

// IsEnvFile returns true if a given file path corresponds to a .env file
func IsEnvFile(path string) bool {
return strings.HasSuffix(path, ".env")
Expand All @@ -146,6 +152,8 @@ func DefaultStoreForPath(path string) Store {
return &yaml.Store{}
} else if IsJSONFile(path) {
return &json.Store{}
} else if IsTOMLFile(path) {
return &toml.Store{}
} else if IsEnvFile(path) {
return &dotenv.Store{}
} else if IsIniFile(path) {
Expand Down
9 changes: 7 additions & 2 deletions cmd/sops/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import (
"go.mozilla.org/sops/stores/dotenv"
"go.mozilla.org/sops/stores/ini"
"go.mozilla.org/sops/stores/json"
"go.mozilla.org/sops/stores/toml"
yamlstores "go.mozilla.org/sops/stores/yaml"
"go.mozilla.org/sops/version"
"google.golang.org/grpc"
Expand Down Expand Up @@ -491,11 +492,11 @@ func main() {
},
cli.StringFlag{
Name: "input-type",
Usage: "currently json, yaml, dotenv and binary are supported. If not set, sops will use the file's extension to determine the type",
Usage: "currently json, yaml, toml, dotenv and binary are supported. If not set, sops will use the file's extension to determine the type",
},
cli.StringFlag{
Name: "output-type",
Usage: "currently json, yaml, dotenv and binary are supported. If not set, sops will use the input file's extension to determine the output format",
Usage: "currently json, yaml, toml, dotenv and binary are supported. If not set, sops will use the input file's extension to determine the output format",
},
cli.BoolFlag{
Name: "show-master-keys, s",
Expand Down Expand Up @@ -880,6 +881,8 @@ func inputStore(context *cli.Context, path string) common.Store {
return &yamlstores.Store{}
case "json":
return &json.Store{}
case "toml":
return &toml.Store{}
case "dotenv":
return &dotenv.Store{}
case "ini":
Expand All @@ -897,6 +900,8 @@ func outputStore(context *cli.Context, path string) common.Store {
return &yamlstores.Store{}
case "json":
return &json.Store{}
case "toml":
return &toml.Store{}
case "dotenv":
return &dotenv.Store{}
case "ini":
Expand Down
3 changes: 3 additions & 0 deletions decrypt/decrypt.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"go.mozilla.org/sops/aes"
sopsdotenv "go.mozilla.org/sops/stores/dotenv"
sopsjson "go.mozilla.org/sops/stores/json"
sopstoml "go.mozilla.org/sops/stores/toml"
sopsyaml "go.mozilla.org/sops/stores/yaml"
)

Expand All @@ -39,6 +40,8 @@ func Data(data []byte, format string) (cleartext []byte, err error) {
store = &sopsjson.Store{}
case "yaml":
store = &sopsyaml.Store{}
case "toml":
store = &sopstoml.Store{}
case "dotenv":
store = &sopsdotenv.Store{}
default:
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ require (
github.com/Azure/go-autorest/autorest/azure/auth v0.1.0
github.com/Azure/go-autorest/autorest/to v0.2.0 // indirect
github.com/Azure/go-autorest/autorest/validation v0.1.0 // indirect
github.com/BurntSushi/toml v0.3.1
github.com/aws/aws-sdk-go v1.21.6
github.com/blang/semver v3.5.1+incompatible
github.com/fatih/color v1.7.0
Expand Down
1 change: 1 addition & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1Gn
github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
github.com/Azure/go-autorest/tracing v0.1.0 h1:TRBxC5Pj/fIuh4Qob0ZpkggbfT8RC0SubHbpV3p4/Vc=
github.com/Azure/go-autorest/tracing v0.1.0/go.mod h1:ROEEAFwXycQw7Sn3DXNtEedEvdeRAgDr0izn4z5Ij88=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
Expand Down
4 changes: 4 additions & 0 deletions sops.go
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,8 @@ func (branch TreeBranch) walkValue(in interface{}, path []string, onLeaves func(
return onLeaves(string(in), path)
case int:
return onLeaves(in, path)
case int64:
return onLeaves(in, path)
case bool:
return onLeaves(in, path)
case float64:
Expand Down Expand Up @@ -702,6 +704,8 @@ func ToBytes(in interface{}) ([]byte, error) {
return []byte(in), nil
case int:
return []byte(strconv.Itoa(in)), nil
case int64:
return []byte(strconv.FormatInt(in, 10)), nil
case float64:
return []byte(strconv.FormatFloat(in, 'f', -1, 64)), nil
case bool:
Expand Down
68 changes: 34 additions & 34 deletions stores/stores.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,62 +27,62 @@ type SopsFile struct {
// in the SOPS file by checking for nil. This way we can show the user a
// helpful error message indicating that the metadata wasn't found, instead
// of showing a cryptic parsing error
Metadata *Metadata `yaml:"sops" json:"sops" ini:"sops"`
Metadata *Metadata `yaml:"sops" json:"sops" toml:"sops" ini:"sops"`
}

// Metadata is stored in SOPS encrypted files, and it contains the information necessary to decrypt the file.
// This struct is just used for serialization, and SOPS uses another struct internally, sops.Metadata. It exists
// in order to allow the binary format to stay backwards compatible over time, but at the same time allow the internal
// representation SOPS uses to change over time.
type Metadata struct {
ShamirThreshold int `yaml:"shamir_threshold,omitempty" json:"shamir_threshold,omitempty"`
KeyGroups []keygroup `yaml:"key_groups,omitempty" json:"key_groups,omitempty"`
KMSKeys []kmskey `yaml:"kms" json:"kms"`
GCPKMSKeys []gcpkmskey `yaml:"gcp_kms" json:"gcp_kms"`
AzureKeyVaultKeys []azkvkey `yaml:"azure_kv" json:"azure_kv"`
LastModified string `yaml:"lastmodified" json:"lastmodified"`
MessageAuthenticationCode string `yaml:"mac" json:"mac"`
PGPKeys []pgpkey `yaml:"pgp" json:"pgp"`
UnencryptedSuffix string `yaml:"unencrypted_suffix,omitempty" json:"unencrypted_suffix,omitempty"`
EncryptedSuffix string `yaml:"encrypted_suffix,omitempty" json:"encrypted_suffix,omitempty"`
EncryptedRegex string `yaml:"encrypted_regex,omitempty" json:"encrypted_regex,omitempty"`
Version string `yaml:"version" json:"version"`
ShamirThreshold int `yaml:"shamir_threshold,omitempty" json:"shamir_threshold,omitempty" toml:"shamir_threshold,omitempty"`
KeyGroups []keygroup `yaml:"key_groups,omitempty" json:"key_groups,omitempty" toml:"key_groups,omitempty"`
KMSKeys []kmskey `yaml:"kms" json:"kms" toml:"kms"`
GCPKMSKeys []gcpkmskey `yaml:"gcp_kms" json:"gcp_kms" toml:"gcp_kms"`
AzureKeyVaultKeys []azkvkey `yaml:"azure_kv" json:"azure_kv" toml:"azure_kv"`
LastModified string `yaml:"lastmodified" json:"lastmodified" toml:"lastmodified"`
MessageAuthenticationCode string `yaml:"mac" json:"mac" toml:"mac"`
PGPKeys []pgpkey `yaml:"pgp" json:"pgp" toml:"pgp"`
UnencryptedSuffix string `yaml:"unencrypted_suffix,omitempty" json:"unencrypted_suffix,omitempty" toml:"unencrypted_suffix,omitempty"`
EncryptedSuffix string `yaml:"encrypted_suffix,omitempty" json:"encrypted_suffix,omitempty" toml:"encrypted_suffix,omitempty"`
EncryptedRegex string `yaml:"encrypted_regex,omitempty" json:"encrypted_regex,omitempty" toml:"encrypted_regex,omitempty"`
Version string `yaml:"version" json:"version" toml:"version"`
}

type keygroup struct {
PGPKeys []pgpkey `yaml:"pgp,omitempty" json:"pgp,omitempty"`
KMSKeys []kmskey `yaml:"kms,omitempty" json:"kms,omitempty"`
GCPKMSKeys []gcpkmskey `yaml:"gcp_kms,omitempty" json:"gcp_kms,omitempty"`
AzureKeyVaultKeys []azkvkey `yaml:"azure_kv,omitempty" json:"azure_kv,omitempty"`
PGPKeys []pgpkey `yaml:"pgp,omitempty" json:"pgp,omitempty" toml:"pgp,omitempty"`
KMSKeys []kmskey `yaml:"kms,omitempty" json:"kms,omitempty" toml:"kms,omitempty"`
GCPKMSKeys []gcpkmskey `yaml:"gcp_kms,omitempty" json:"gcp_kms,omitempty" toml:"gcp_kms,omitempty"`
AzureKeyVaultKeys []azkvkey `yaml:"azure_kv,omitempty" json:"azure_kv,omitempty" toml:"azure_kv,omitempty"`
}

type pgpkey struct {
CreatedAt string `yaml:"created_at" json:"created_at"`
EncryptedDataKey string `yaml:"enc" json:"enc"`
Fingerprint string `yaml:"fp" json:"fp"`
CreatedAt string `yaml:"created_at" json:"created_at" toml:"created_at"`
EncryptedDataKey string `yaml:"enc" json:"enc" toml:"enc"`
Fingerprint string `yaml:"fp" json:"fp" toml:"fp"`
}

type kmskey struct {
Arn string `yaml:"arn" json:"arn"`
Role string `yaml:"role,omitempty" json:"role,omitempty"`
Context map[string]*string `yaml:"context,omitempty" json:"context,omitempty"`
CreatedAt string `yaml:"created_at" json:"created_at"`
EncryptedDataKey string `yaml:"enc" json:"enc"`
AwsProfile string `yaml:"aws_profile" json:"aws_profile"`
Arn string `yaml:"arn" json:"arn" toml:"arn"`
Role string `yaml:"role,omitempty" json:"role,omitempty" toml:"role,omitempty"`
Context map[string]*string `yaml:"context,omitempty" json:"context,omitempty" toml:"context,omitempty"`
CreatedAt string `yaml:"created_at" json:"created_at" toml:"created_at"`
EncryptedDataKey string `yaml:"enc" json:"enc" toml:"enc"`
AwsProfile string `yaml:"aws_profile" json:"aws_profile" toml:"aws_profile"`
}

type gcpkmskey struct {
ResourceID string `yaml:"resource_id" json:"resource_id"`
CreatedAt string `yaml:"created_at" json:"created_at"`
EncryptedDataKey string `yaml:"enc" json:"enc"`
ResourceID string `yaml:"resource_id" json:"resource_id" toml:"resource_id"`
CreatedAt string `yaml:"created_at" json:"created_at" toml:"created_at"`
EncryptedDataKey string `yaml:"enc" json:"enc" toml:"enc"`
}

type azkvkey struct {
VaultURL string `yaml:"vault_url" json:"vault_url"`
Name string `yaml:"name" json:"name"`
Version string `yaml:"version" json:"version"`
CreatedAt string `yaml:"created_at" json:"created_at"`
EncryptedDataKey string `yaml:"enc" json:"enc"`
VaultURL string `yaml:"vault_url" json:"vault_url" toml:"vault_url"`
Name string `yaml:"name" json:"name" toml:"name"`
Version string `yaml:"version" json:"version" toml:"version"`
CreatedAt string `yaml:"created_at" json:"created_at" toml:"created_at"`
EncryptedDataKey string `yaml:"enc" json:"enc" toml:"enc"`
}

// MetadataFromInternal converts an internal SOPS metadata representation to a representation appropriate for storage
Expand Down
79 changes: 79 additions & 0 deletions stores/toml/reader.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
package toml //import "go.mozilla.org/sops/stores/toml"

import (
"github.com/BurntSushi/toml"
"go.mozilla.org/sops"
)

// TOML reader that preserves original order via toml.Metadata.Keys(),
// see https://godoc.org/github.com/BurntSushi/toml#MetaData.Keys
// and converts the unoredered data (multi-level tree of
// map[string]interface{}) into sops.BranchTree.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
// map[string]interface{}) into sops.BranchTree.
// map[string]interface{}) into sops.TreeBranch.

type tomlReader []toml.Key

func (c tomlReader) keysInOrder() []string {
var keys []string
visited := map[string]struct{}{}
for _, key := range c {
_, ok := visited[key[0]]
if !ok {
keys = append(keys, key[0])
visited[key[0]] = struct{}{}
}
}
return keys
}

func (c tomlReader) table(key string) tomlReader {
var keys []toml.Key
for _, k := range c {
if len(k) > 1 && k[0] == key {
keys = append(keys, k[1:])
}
}
return tomlReader(keys)
}

func (c tomlReader) arrayItem(key string, at int) tomlReader {
var keys []toml.Key
arrayAt := -1
for _, k := range c {
if len(k) == 1 && k[0] == key {
arrayAt++
continue
}
if len(k) > 1 && k[0] == key && at == arrayAt {
keys = append(keys, k[1:])
}
}
return tomlReader(keys)
}

func (c tomlReader) readToTreeBranch(unordered map[string]interface{}) sops.TreeBranch {
var branch sops.TreeBranch
for _, key := range c.keysInOrder() {
value := unordered[key]
switch v := value.(type) {
case map[string]interface{}:
branch = append(branch, sops.TreeItem{
Key: key,
Value: c.table(key).readToTreeBranch(v),
})
case []map[string]interface{}:
var array []interface{}
for i, item := range v {
array = append(array, c.arrayItem(key, i).readToTreeBranch(item))
}
branch = append(branch, sops.TreeItem{
Key: key,
Value: array,
})
default:
branch = append(branch, sops.TreeItem{
Key: key,
Value: value,
})
}
}
return branch
}
Loading