From 5a0e3bdf4a149a3b0d4edbc9420087a571c29858 Mon Sep 17 00:00:00 2001 From: parasssh Date: Mon, 21 Sep 2020 09:11:43 -0700 Subject: [PATCH 1/6] Allow all encrpted/unencrpted input/output combinations in bulk --- dgraph/cmd/bulk/loader.go | 1 + dgraph/cmd/bulk/reduce.go | 7 ++++++- dgraph/cmd/bulk/run.go | 11 +++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/dgraph/cmd/bulk/loader.go b/dgraph/cmd/bulk/loader.go index 77a88fa5edf..dd848ebfa78 100644 --- a/dgraph/cmd/bulk/loader.go +++ b/dgraph/cmd/bulk/loader.go @@ -67,6 +67,7 @@ type options struct { NewUids bool ClientDir string Encrypted bool + EncryptedOut bool MapShards int ReduceShards int diff --git a/dgraph/cmd/bulk/reduce.go b/dgraph/cmd/bulk/reduce.go index 27b11ac4b84..5f05b57e698 100644 --- a/dgraph/cmd/bulk/reduce.go +++ b/dgraph/cmd/bulk/reduce.go @@ -142,12 +142,17 @@ func (r *reducer) createBadgerInternal(dir string, compression bool) *badger.DB } } + key := r.opt.EncryptionKey + if !r.opt.EncryptedOut { + key = nil + } + opt := badger.DefaultOptions(dir). WithSyncWrites(false). WithTableLoadingMode(bo.MemoryMap). WithValueThreshold(1 << 10 /* 1 KB */). WithLogger(nil). - WithEncryptionKey(r.opt.EncryptionKey). + WithEncryptionKey(key). WithBlockCacheSize(r.opt.BlockCacheSize). WithIndexCacheSize(r.opt.IndexCacheSize) diff --git a/dgraph/cmd/bulk/run.go b/dgraph/cmd/bulk/run.go index d208090e546..dc8c7d8508f 100644 --- a/dgraph/cmd/bulk/run.go +++ b/dgraph/cmd/bulk/run.go @@ -61,6 +61,8 @@ func init() { "Specify file format (rdf or json) instead of getting it from filename.") flag.Bool("encrypted", false, "Flag to indicate whether schema and data files are encrypted.") + flag.Bool("encrypted_out", false, + "Flag to indicate whether to encrypt the output.") flag.String("out", defaultOutDir, "Location to write the final dgraph data directories.") flag.Bool("replace_out", false, @@ -122,6 +124,7 @@ func run() { SchemaFile: Bulk.Conf.GetString("schema"), GqlSchemaFile: Bulk.Conf.GetString("graphql_schema"), Encrypted: Bulk.Conf.GetBool("encrypted"), + EncryptedOut: Bulk.Conf.GetBool("encrypted_out"), OutDir: Bulk.Conf.GetString("out"), ReplaceOutDir: Bulk.Conf.GetBool("replace_out"), TmpDir: Bulk.Conf.GetString("tmp"), @@ -166,10 +169,18 @@ func run() { fmt.Printf("unable to read key %v", err) return } + if len(opt.EncryptionKey) > 0 && !opt.Encrypted && !opt.EncryptedOut { + fmt.Printf("Must use --encrypted/--encrypted_out with --encryption_key_file option.\n") + os.Exit(1) + } if opt.Encrypted && len(opt.EncryptionKey) == 0 { fmt.Printf("Must use --encryption_key_file or vault option(s) with --encrypted option.\n") os.Exit(1) } + if opt.EncryptedOut && len(opt.EncryptionKey) == 0 { + fmt.Printf("Must use --encryption_key_file or vault option(s) with --encrypted_out option.\n") + os.Exit(1) + } if opt.SchemaFile == "" { fmt.Fprint(os.Stderr, "Schema file must be specified.\n") os.Exit(1) From 9153a03a13c0eeea53b5c52bb9a006d1ad561587 Mon Sep 17 00:00:00 2001 From: parasssh Date: Mon, 21 Sep 2020 14:10:11 -0700 Subject: [PATCH 2/6] addr. review --- dgraph/cmd/bulk/run.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dgraph/cmd/bulk/run.go b/dgraph/cmd/bulk/run.go index dc8c7d8508f..16a4ddb7069 100644 --- a/dgraph/cmd/bulk/run.go +++ b/dgraph/cmd/bulk/run.go @@ -170,9 +170,9 @@ func run() { return } if len(opt.EncryptionKey) > 0 && !opt.Encrypted && !opt.EncryptedOut { - fmt.Printf("Must use --encrypted/--encrypted_out with --encryption_key_file option.\n") - os.Exit(1) - } + opt.Encrypted = true + opt.EncryptedOut = true + } if opt.Encrypted && len(opt.EncryptionKey) == 0 { fmt.Printf("Must use --encryption_key_file or vault option(s) with --encrypted option.\n") os.Exit(1) From 1e238bb6c203c741b95946ab37f31841857f3d14 Mon Sep 17 00:00:00 2001 From: parasssh Date: Thu, 24 Sep 2020 15:37:06 -0700 Subject: [PATCH 3/6] new behavior --- dgraph/cmd/bulk/run.go | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/dgraph/cmd/bulk/run.go b/dgraph/cmd/bulk/run.go index 16a4ddb7069..63660ee30b3 100644 --- a/dgraph/cmd/bulk/run.go +++ b/dgraph/cmd/bulk/run.go @@ -60,9 +60,11 @@ func init() { flag.String("format", "", "Specify file format (rdf or json) instead of getting it from filename.") flag.Bool("encrypted", false, - "Flag to indicate whether schema and data files are encrypted.") + "Flag to indicate whether schema and data files are encrypted." + + "Must be specified with --encryption_key_file or vault options") flag.Bool("encrypted_out", false, - "Flag to indicate whether to encrypt the output.") + "Flag to indicate whether to encrypt the output." + + "Must be specified with --encryption_key_file or vault option.") flag.String("out", defaultOutDir, "Location to write the final dgraph data directories.") flag.Bool("replace_out", false, @@ -169,18 +171,27 @@ func run() { fmt.Printf("unable to read key %v", err) return } - if len(opt.EncryptionKey) > 0 && !opt.Encrypted && !opt.EncryptedOut { - opt.Encrypted = true - opt.EncryptedOut = true - } - if opt.Encrypted && len(opt.EncryptionKey) == 0 { - fmt.Printf("Must use --encryption_key_file or vault option(s) with --encrypted option.\n") - os.Exit(1) - } - if opt.EncryptedOut && len(opt.EncryptionKey) == 0 { - fmt.Printf("Must use --encryption_key_file or vault option(s) with --encrypted_out option.\n") - os.Exit(1) + + if len(opt.EncryptionKey) == 0 { + if opt.Encrypted || opt.EncryptedOut { + fmt.Fprint(os.Stderr, "Must use --encryption_key_file or vault option(s).\n") + os.Exit(1) + } + } else { + requiredFlags := Bulk.Cmd.Flags().Changed("encrypted") && + Bulk.Cmd.Flags().Changed("encrypted_out") + if !requiredFlags { + fmt.Fprint(os.Stderr, "Must specify --encrypted and --encrypted_out.\n") + os.Exit(1) + } + if !opt.Encrypted || !opt.EncryptedOut { + fmt.Fprint(os.Stderr, "Must set --encrypted and/or --encrypted_out to true.\n") + os.Exit(1) + } } + + fmt.Printf("Input is encrytped? %v, Output will be encrypted? %v\n", opt.Encrypted, opt.EncryptedOut) + if opt.SchemaFile == "" { fmt.Fprint(os.Stderr, "Schema file must be specified.\n") os.Exit(1) From 61c2b0f98de3ab08bb123b735023b82d6e60523f Mon Sep 17 00:00:00 2001 From: parasssh Date: Thu, 24 Sep 2020 16:09:49 -0700 Subject: [PATCH 4/6] review comments --- dgraph/cmd/bulk/run.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/dgraph/cmd/bulk/run.go b/dgraph/cmd/bulk/run.go index 63660ee30b3..396fd091ec6 100644 --- a/dgraph/cmd/bulk/run.go +++ b/dgraph/cmd/bulk/run.go @@ -60,11 +60,11 @@ func init() { flag.String("format", "", "Specify file format (rdf or json) instead of getting it from filename.") flag.Bool("encrypted", false, - "Flag to indicate whether schema and data files are encrypted." + - "Must be specified with --encryption_key_file or vault options") + "Flag to indicate whether schema and data files are encrypted. "+ + "Must be specified with --encryption_key_file or vault options.") flag.Bool("encrypted_out", false, - "Flag to indicate whether to encrypt the output." + - "Must be specified with --encryption_key_file or vault option.") + "Flag to indicate whether to encrypt the output. "+ + "Must be specified with --encryption_key_file or vault options.") flag.String("out", defaultOutDir, "Location to write the final dgraph data directories.") flag.Bool("replace_out", false, @@ -171,7 +171,6 @@ func run() { fmt.Printf("unable to read key %v", err) return } - if len(opt.EncryptionKey) == 0 { if opt.Encrypted || opt.EncryptedOut { fmt.Fprint(os.Stderr, "Must use --encryption_key_file or vault option(s).\n") @@ -185,12 +184,11 @@ func run() { os.Exit(1) } if !opt.Encrypted || !opt.EncryptedOut { - fmt.Fprint(os.Stderr, "Must set --encrypted and/or --encrypted_out to true.\n") + fmt.Fprint(os.Stderr, "Must set --encrypted and/or --encrypted_out to true when providing encryption key.\n") os.Exit(1) } } - - fmt.Printf("Input is encrytped? %v, Output will be encrypted? %v\n", opt.Encrypted, opt.EncryptedOut) + fmt.Printf("Input Encrypted: %v; Output Encrypted: %v\n", opt.Encrypted, opt.EncryptedOut) if opt.SchemaFile == "" { fmt.Fprint(os.Stderr, "Schema file must be specified.\n") From 494cb3707e1ac3e1a50e4af0f4988b068bd75cb5 Mon Sep 17 00:00:00 2001 From: parasssh Date: Thu, 24 Sep 2020 16:17:00 -0700 Subject: [PATCH 5/6] fixes and minor fmt --- dgraph/cmd/bulk/run.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dgraph/cmd/bulk/run.go b/dgraph/cmd/bulk/run.go index 396fd091ec6..948667ef99f 100644 --- a/dgraph/cmd/bulk/run.go +++ b/dgraph/cmd/bulk/run.go @@ -61,10 +61,10 @@ func init() { "Specify file format (rdf or json) instead of getting it from filename.") flag.Bool("encrypted", false, "Flag to indicate whether schema and data files are encrypted. "+ - "Must be specified with --encryption_key_file or vault options.") + "Must be specified with --encryption_key_file or vault option(s).") flag.Bool("encrypted_out", false, "Flag to indicate whether to encrypt the output. "+ - "Must be specified with --encryption_key_file or vault options.") + "Must be specified with --encryption_key_file or vault option(s).") flag.String("out", defaultOutDir, "Location to write the final dgraph data directories.") flag.Bool("replace_out", false, @@ -180,10 +180,10 @@ func run() { requiredFlags := Bulk.Cmd.Flags().Changed("encrypted") && Bulk.Cmd.Flags().Changed("encrypted_out") if !requiredFlags { - fmt.Fprint(os.Stderr, "Must specify --encrypted and --encrypted_out.\n") + fmt.Fprint(os.Stderr, "Must specify --encrypted and --encrypted_out when providing encryption key.\n") os.Exit(1) } - if !opt.Encrypted || !opt.EncryptedOut { + if !opt.Encrypted && !opt.EncryptedOut { fmt.Fprint(os.Stderr, "Must set --encrypted and/or --encrypted_out to true when providing encryption key.\n") os.Exit(1) } From d14ea532684feb6137122aa8d8bea5ea41e7184a Mon Sep 17 00:00:00 2001 From: parasssh Date: Thu, 24 Sep 2020 16:19:43 -0700 Subject: [PATCH 6/6] minor --- dgraph/cmd/bulk/run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dgraph/cmd/bulk/run.go b/dgraph/cmd/bulk/run.go index 948667ef99f..208a8024220 100644 --- a/dgraph/cmd/bulk/run.go +++ b/dgraph/cmd/bulk/run.go @@ -188,7 +188,7 @@ func run() { os.Exit(1) } } - fmt.Printf("Input Encrypted: %v; Output Encrypted: %v\n", opt.Encrypted, opt.EncryptedOut) + fmt.Printf("Encrypted input: %v; Encrypted output: %v\n", opt.Encrypted, opt.EncryptedOut) if opt.SchemaFile == "" { fmt.Fprint(os.Stderr, "Schema file must be specified.\n")