From 47e75d84fc65cfe72bec58407d21474005c33202 Mon Sep 17 00:00:00 2001 From: aman-bansal Date: Thu, 17 Dec 2020 00:41:53 +0530 Subject: [PATCH 1/3] fixing bulk loader when encryption + mtls is enabled --- dgraph/cmd/bulk/loader.go | 2 ++ dgraph/cmd/bulk/reduce.go | 1 + dgraph/cmd/bulk/run.go | 3 +++ 3 files changed, 6 insertions(+) diff --git a/dgraph/cmd/bulk/loader.go b/dgraph/cmd/bulk/loader.go index 5a38d8d80bc..d75af48e443 100644 --- a/dgraph/cmd/bulk/loader.go +++ b/dgraph/cmd/bulk/loader.go @@ -20,6 +20,7 @@ import ( "bytes" "compress/gzip" "context" + "crypto/tls" "fmt" "hash/adler32" "io" @@ -64,6 +65,7 @@ type options struct { Version bool StoreXids bool ZeroAddr string + ClientTLSConfig *tls.Config `json:"-"` HttpAddr string IgnoreErrors bool CustomTokenizers string diff --git a/dgraph/cmd/bulk/reduce.go b/dgraph/cmd/bulk/reduce.go index 3927f4d0cf0..18aeb8f752c 100644 --- a/dgraph/cmd/bulk/reduce.go +++ b/dgraph/cmd/bulk/reduce.go @@ -129,6 +129,7 @@ func (r *reducer) createBadgerInternal(dir string, compression bool) *badger.DB if r.opt.EncryptionKey != nil { // Need to set zero addr in WorkerConfig before checking the license. x.WorkerConfig.ZeroAddr = []string{r.opt.ZeroAddr} + x.WorkerConfig.TLSClientConfig = r.opt.ClientTLSConfig if !worker.EnterpriseEnabled() { // Crash since the enterprise license is not enabled.. diff --git a/dgraph/cmd/bulk/run.go b/dgraph/cmd/bulk/run.go index 3450fa3c5d5..c8923aa0003 100644 --- a/dgraph/cmd/bulk/run.go +++ b/dgraph/cmd/bulk/run.go @@ -123,6 +123,8 @@ func init() { func run() { ctype, clevel := x.ParseCompression(Bulk.Conf.GetString("badger.compression")) + tlsConf, err := x.LoadClientTLSConfigForInternalPort(Bulk.Conf) + x.Check(err) opt := options{ DataFiles: Bulk.Conf.GetString("files"), DataFormat: Bulk.Conf.GetString("format"), @@ -142,6 +144,7 @@ func run() { Version: Bulk.Conf.GetBool("version"), StoreXids: Bulk.Conf.GetBool("store_xids"), ZeroAddr: Bulk.Conf.GetString("zero"), + ClientTLSConfig: tlsConf, HttpAddr: Bulk.Conf.GetString("http"), IgnoreErrors: Bulk.Conf.GetBool("ignore_errors"), MapShards: Bulk.Conf.GetInt("map_shards"), From ff8470fff64734cc23844ea5dcd26521f1a0ce07 Mon Sep 17 00:00:00 2001 From: aman-bansal Date: Thu, 17 Dec 2020 00:52:00 +0530 Subject: [PATCH 2/3] making enterprise check early --- dgraph/cmd/bulk/loader.go | 2 -- dgraph/cmd/bulk/reduce.go | 14 -------------- dgraph/cmd/bulk/run.go | 13 ++++++++++++- 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/dgraph/cmd/bulk/loader.go b/dgraph/cmd/bulk/loader.go index d75af48e443..5a38d8d80bc 100644 --- a/dgraph/cmd/bulk/loader.go +++ b/dgraph/cmd/bulk/loader.go @@ -20,7 +20,6 @@ import ( "bytes" "compress/gzip" "context" - "crypto/tls" "fmt" "hash/adler32" "io" @@ -65,7 +64,6 @@ type options struct { Version bool StoreXids bool ZeroAddr string - ClientTLSConfig *tls.Config `json:"-"` HttpAddr string IgnoreErrors bool CustomTokenizers string diff --git a/dgraph/cmd/bulk/reduce.go b/dgraph/cmd/bulk/reduce.go index 18aeb8f752c..ab9bdf81e8f 100644 --- a/dgraph/cmd/bulk/reduce.go +++ b/dgraph/cmd/bulk/reduce.go @@ -42,7 +42,6 @@ import ( "github.com/dgraph-io/dgraph/codec" "github.com/dgraph-io/dgraph/posting" "github.com/dgraph-io/dgraph/protos/pb" - "github.com/dgraph-io/dgraph/worker" "github.com/dgraph-io/dgraph/x" "github.com/dgraph-io/ristretto/z" "github.com/dustin/go-humanize" @@ -126,19 +125,6 @@ func (r *reducer) run() error { } func (r *reducer) createBadgerInternal(dir string, compression bool) *badger.DB { - if r.opt.EncryptionKey != nil { - // Need to set zero addr in WorkerConfig before checking the license. - x.WorkerConfig.ZeroAddr = []string{r.opt.ZeroAddr} - x.WorkerConfig.TLSClientConfig = r.opt.ClientTLSConfig - - if !worker.EnterpriseEnabled() { - // Crash since the enterprise license is not enabled.. - log.Fatal("Enterprise License needed for the Encryption feature.") - } else { - log.Printf("Encryption feature enabled.") - } - } - key := r.opt.EncryptionKey if !r.opt.EncryptedOut { key = nil diff --git a/dgraph/cmd/bulk/run.go b/dgraph/cmd/bulk/run.go index c8923aa0003..4fa725451f3 100644 --- a/dgraph/cmd/bulk/run.go +++ b/dgraph/cmd/bulk/run.go @@ -29,6 +29,8 @@ import ( "strconv" "strings" + "github.com/dgraph-io/dgraph/worker" + "github.com/dgraph-io/dgraph/ee/enc" "github.com/dgraph-io/dgraph/tok" "github.com/dgraph-io/dgraph/x" @@ -144,7 +146,6 @@ func run() { Version: Bulk.Conf.GetBool("version"), StoreXids: Bulk.Conf.GetBool("store_xids"), ZeroAddr: Bulk.Conf.GetString("zero"), - ClientTLSConfig: tlsConf, HttpAddr: Bulk.Conf.GetString("http"), IgnoreErrors: Bulk.Conf.GetBool("ignore_errors"), MapShards: Bulk.Conf.GetInt("map_shards"), @@ -190,6 +191,16 @@ func run() { fmt.Fprint(os.Stderr, "Must set --encrypted and/or --encrypted_out to true when providing encryption key.\n") os.Exit(1) } + + // Need to set zero addr in WorkerConfig before checking the license. + x.WorkerConfig.ZeroAddr = []string{opt.ZeroAddr} + x.WorkerConfig.TLSClientConfig = tlsConf + if !worker.EnterpriseEnabled() { + // Crash since the enterprise license is not enabled.. + log.Fatal("Enterprise License needed for the Encryption feature.") + } else { + log.Printf("Encryption feature enabled.") + } } fmt.Printf("Encrypted input: %v; Encrypted output: %v\n", opt.Encrypted, opt.EncryptedOut) From 92c852a4adaace9fe05e8440aa7eff7ff4153253 Mon Sep 17 00:00:00 2001 From: aman-bansal Date: Thu, 17 Dec 2020 00:56:50 +0530 Subject: [PATCH 3/3] making tls var localised --- dgraph/cmd/bulk/run.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dgraph/cmd/bulk/run.go b/dgraph/cmd/bulk/run.go index 4fa725451f3..b2f93c7de9a 100644 --- a/dgraph/cmd/bulk/run.go +++ b/dgraph/cmd/bulk/run.go @@ -125,8 +125,6 @@ func init() { func run() { ctype, clevel := x.ParseCompression(Bulk.Conf.GetString("badger.compression")) - tlsConf, err := x.LoadClientTLSConfigForInternalPort(Bulk.Conf) - x.Check(err) opt := options{ DataFiles: Bulk.Conf.GetString("files"), DataFormat: Bulk.Conf.GetString("format"), @@ -192,6 +190,8 @@ func run() { os.Exit(1) } + tlsConf, err := x.LoadClientTLSConfigForInternalPort(Bulk.Conf) + x.Check(err) // Need to set zero addr in WorkerConfig before checking the license. x.WorkerConfig.ZeroAddr = []string{opt.ZeroAddr} x.WorkerConfig.TLSClientConfig = tlsConf