Skip to content

Commit 42e6632

Browse files
author
marc
committed
Encryption flags dictate storage version
1 parent 7cca4c1 commit 42e6632

File tree

5 files changed

+57
-52
lines changed

5 files changed

+57
-52
lines changed

‎pkg/base/store_spec.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,11 @@ type StoreSpec struct {
6262
// SizeInBytes is used for calculating free space and making rebalancing
6363
// decisions. Zero indicates that there is no maximum size. This value is not
6464
// actually used by the engine and thus not enforced.
65-
SizeInBytes int64
66-
SizePercent float64
67-
InMemory bool
68-
Attributes roachpb.Attributes
65+
SizeInBytes int64
66+
SizePercent float64
67+
InMemory bool
68+
Attributes roachpb.Attributes
69+
UseSwitchingEnv bool
6970
// EncryptionSpec is non-nil after MatchStoreAndEncryptionSpecs if there is an encryption
7071
// spec with matching path.
7172
// EncryptionSpec *StoreEncryptionSpec

‎pkg/base/store_spec_test.go‎

Lines changed: 32 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -34,49 +34,49 @@ func TestNewStoreSpec(t *testing.T) {
3434
expected StoreSpec
3535
}{
3636
// path
37-
{"path=/mnt/hda1", "", StoreSpec{"/mnt/hda1", 0, 0, false, roachpb.Attributes{}, nil}},
38-
{",path=/mnt/hda1", "", StoreSpec{"/mnt/hda1", 0, 0, false, roachpb.Attributes{}, nil}},
39-
{"path=/mnt/hda1,", "", StoreSpec{"/mnt/hda1", 0, 0, false, roachpb.Attributes{}, nil}},
40-
{",,,path=/mnt/hda1,,,", "", StoreSpec{"/mnt/hda1", 0, 0, false, roachpb.Attributes{}, nil}},
41-
{"/mnt/hda1", "", StoreSpec{"/mnt/hda1", 0, 0, false, roachpb.Attributes{}, nil}},
37+
{"path=/mnt/hda1", "", StoreSpec{"/mnt/hda1", 0, 0, false, roachpb.Attributes{}, false, nil}},
38+
{",path=/mnt/hda1", "", StoreSpec{"/mnt/hda1", 0, 0, false, roachpb.Attributes{}, false, nil}},
39+
{"path=/mnt/hda1,", "", StoreSpec{"/mnt/hda1", 0, 0, false, roachpb.Attributes{}, false, nil}},
40+
{",,,path=/mnt/hda1,,,", "", StoreSpec{"/mnt/hda1", 0, 0, false, roachpb.Attributes{}, false, nil}},
41+
{"/mnt/hda1", "", StoreSpec{"/mnt/hda1", 0, 0, false, roachpb.Attributes{}, false, nil}},
4242
{"path=", "no value specified for path", StoreSpec{}},
4343
{"path=/mnt/hda1,path=/mnt/hda2", "path field was used twice in store definition", StoreSpec{}},
4444
{"/mnt/hda1,path=/mnt/hda2", "path field was used twice in store definition", StoreSpec{}},
4545

4646
// attributes
47-
{"path=/mnt/hda1,attrs=ssd", "", StoreSpec{"/mnt/hda1", 0, 0, false, roachpb.Attributes{Attrs: []string{"ssd"}}, nil}},
48-
{"path=/mnt/hda1,attrs=ssd:hdd", "", StoreSpec{"/mnt/hda1", 0, 0, false, roachpb.Attributes{Attrs: []string{"hdd", "ssd"}}, nil}},
49-
{"path=/mnt/hda1,attrs=hdd:ssd", "", StoreSpec{"/mnt/hda1", 0, 0, false, roachpb.Attributes{Attrs: []string{"hdd", "ssd"}}, nil}},
50-
{"attrs=ssd:hdd,path=/mnt/hda1", "", StoreSpec{"/mnt/hda1", 0, 0, false, roachpb.Attributes{Attrs: []string{"hdd", "ssd"}}, nil}},
51-
{"attrs=hdd:ssd,path=/mnt/hda1,", "", StoreSpec{"/mnt/hda1", 0, 0, false, roachpb.Attributes{Attrs: []string{"hdd", "ssd"}}, nil}},
47+
{"path=/mnt/hda1,attrs=ssd", "", StoreSpec{"/mnt/hda1", 0, 0, false, roachpb.Attributes{Attrs: []string{"ssd"}}, false, nil}},
48+
{"path=/mnt/hda1,attrs=ssd:hdd", "", StoreSpec{"/mnt/hda1", 0, 0, false, roachpb.Attributes{Attrs: []string{"hdd", "ssd"}}, false, nil}},
49+
{"path=/mnt/hda1,attrs=hdd:ssd", "", StoreSpec{"/mnt/hda1", 0, 0, false, roachpb.Attributes{Attrs: []string{"hdd", "ssd"}}, false, nil}},
50+
{"attrs=ssd:hdd,path=/mnt/hda1", "", StoreSpec{"/mnt/hda1", 0, 0, false, roachpb.Attributes{Attrs: []string{"hdd", "ssd"}}, false, nil}},
51+
{"attrs=hdd:ssd,path=/mnt/hda1,", "", StoreSpec{"/mnt/hda1", 0, 0, false, roachpb.Attributes{Attrs: []string{"hdd", "ssd"}}, false, nil}},
5252
{"attrs=hdd:ssd", "no path specified", StoreSpec{}},
5353
{"path=/mnt/hda1,attrs=", "no value specified for attrs", StoreSpec{}},
5454
{"path=/mnt/hda1,attrs=hdd:hdd", "duplicate attribute given for store: hdd", StoreSpec{}},
5555
{"path=/mnt/hda1,attrs=hdd,attrs=ssd", "attrs field was used twice in store definition", StoreSpec{}},
5656

5757
// size
58-
{"path=/mnt/hda1,size=671088640", "", StoreSpec{"/mnt/hda1", 671088640, 0, false, roachpb.Attributes{}, nil}},
59-
{"path=/mnt/hda1,size=20GB", "", StoreSpec{"/mnt/hda1", 20000000000, 0, false, roachpb.Attributes{}, nil}},
60-
{"size=20GiB,path=/mnt/hda1", "", StoreSpec{"/mnt/hda1", 21474836480, 0, false, roachpb.Attributes{}, nil}},
61-
{"size=0.1TiB,path=/mnt/hda1", "", StoreSpec{"/mnt/hda1", 109951162777, 0, false, roachpb.Attributes{}, nil}},
62-
{"path=/mnt/hda1,size=.1TiB", "", StoreSpec{"/mnt/hda1", 109951162777, 0, false, roachpb.Attributes{}, nil}},
63-
{"path=/mnt/hda1,size=123TB", "", StoreSpec{"/mnt/hda1", 123000000000000, 0, false, roachpb.Attributes{}, nil}},
64-
{"path=/mnt/hda1,size=123TiB", "", StoreSpec{"/mnt/hda1", 135239930216448, 0, false, roachpb.Attributes{}, nil}},
58+
{"path=/mnt/hda1,size=671088640", "", StoreSpec{"/mnt/hda1", 671088640, 0, false, roachpb.Attributes{}, false, nil}},
59+
{"path=/mnt/hda1,size=20GB", "", StoreSpec{"/mnt/hda1", 20000000000, 0, false, roachpb.Attributes{}, false, nil}},
60+
{"size=20GiB,path=/mnt/hda1", "", StoreSpec{"/mnt/hda1", 21474836480, 0, false, roachpb.Attributes{}, false, nil}},
61+
{"size=0.1TiB,path=/mnt/hda1", "", StoreSpec{"/mnt/hda1", 109951162777, 0, false, roachpb.Attributes{}, false, nil}},
62+
{"path=/mnt/hda1,size=.1TiB", "", StoreSpec{"/mnt/hda1", 109951162777, 0, false, roachpb.Attributes{}, false, nil}},
63+
{"path=/mnt/hda1,size=123TB", "", StoreSpec{"/mnt/hda1", 123000000000000, 0, false, roachpb.Attributes{}, false, nil}},
64+
{"path=/mnt/hda1,size=123TiB", "", StoreSpec{"/mnt/hda1", 135239930216448, 0, false, roachpb.Attributes{}, false, nil}},
6565
// %
66-
{"path=/mnt/hda1,size=50.5%", "", StoreSpec{"/mnt/hda1", 0, 50.5, false, roachpb.Attributes{}, nil}},
67-
{"path=/mnt/hda1,size=100%", "", StoreSpec{"/mnt/hda1", 0, 100, false, roachpb.Attributes{}, nil}},
68-
{"path=/mnt/hda1,size=1%", "", StoreSpec{"/mnt/hda1", 0, 1, false, roachpb.Attributes{}, nil}},
66+
{"path=/mnt/hda1,size=50.5%", "", StoreSpec{"/mnt/hda1", 0, 50.5, false, roachpb.Attributes{}, false, nil}},
67+
{"path=/mnt/hda1,size=100%", "", StoreSpec{"/mnt/hda1", 0, 100, false, roachpb.Attributes{}, false, nil}},
68+
{"path=/mnt/hda1,size=1%", "", StoreSpec{"/mnt/hda1", 0, 1, false, roachpb.Attributes{}, false, nil}},
6969
{"path=/mnt/hda1,size=0.999999%", "store size (0.999999%) must be between 1% and 100%", StoreSpec{}},
7070
{"path=/mnt/hda1,size=100.0001%", "store size (100.0001%) must be between 1% and 100%", StoreSpec{}},
7171
// 0.xxx
72-
{"path=/mnt/hda1,size=0.99", "", StoreSpec{"/mnt/hda1", 0, 99, false, roachpb.Attributes{}, nil}},
73-
{"path=/mnt/hda1,size=0.5000000", "", StoreSpec{"/mnt/hda1", 0, 50, false, roachpb.Attributes{}, nil}},
74-
{"path=/mnt/hda1,size=0.01", "", StoreSpec{"/mnt/hda1", 0, 1, false, roachpb.Attributes{}, nil}},
72+
{"path=/mnt/hda1,size=0.99", "", StoreSpec{"/mnt/hda1", 0, 99, false, roachpb.Attributes{}, false, nil}},
73+
{"path=/mnt/hda1,size=0.5000000", "", StoreSpec{"/mnt/hda1", 0, 50, false, roachpb.Attributes{}, false, nil}},
74+
{"path=/mnt/hda1,size=0.01", "", StoreSpec{"/mnt/hda1", 0, 1, false, roachpb.Attributes{}, false, nil}},
7575
{"path=/mnt/hda1,size=0.009999", "store size (0.009999) must be between 1% and 100%", StoreSpec{}},
7676
// .xxx
77-
{"path=/mnt/hda1,size=.999", "", StoreSpec{"/mnt/hda1", 0, 99.9, false, roachpb.Attributes{}, nil}},
78-
{"path=/mnt/hda1,size=.5000000", "", StoreSpec{"/mnt/hda1", 0, 50, false, roachpb.Attributes{}, nil}},
79-
{"path=/mnt/hda1,size=.01", "", StoreSpec{"/mnt/hda1", 0, 1, false, roachpb.Attributes{}, nil}},
77+
{"path=/mnt/hda1,size=.999", "", StoreSpec{"/mnt/hda1", 0, 99.9, false, roachpb.Attributes{}, false, nil}},
78+
{"path=/mnt/hda1,size=.5000000", "", StoreSpec{"/mnt/hda1", 0, 50, false, roachpb.Attributes{}, false, nil}},
79+
{"path=/mnt/hda1,size=.01", "", StoreSpec{"/mnt/hda1", 0, 1, false, roachpb.Attributes{}, false, nil}},
8080
{"path=/mnt/hda1,size=.009999", "store size (.009999) must be between 1% and 100%", StoreSpec{}},
8181
// errors
8282
{"path=/mnt/hda1,size=0", "store size (0) must be larger than 640 MiB", StoreSpec{}},
@@ -86,10 +86,10 @@ func TestNewStoreSpec(t *testing.T) {
8686
{"size=123TB", "no path specified", StoreSpec{}},
8787

8888
// type
89-
{"type=mem,size=20GiB", "", StoreSpec{"", 21474836480, 0, true, roachpb.Attributes{}, nil}},
90-
{"size=20GiB,type=mem", "", StoreSpec{"", 21474836480, 0, true, roachpb.Attributes{}, nil}},
91-
{"size=20.5GiB,type=mem", "", StoreSpec{"", 22011707392, 0, true, roachpb.Attributes{}, nil}},
92-
{"size=20GiB,type=mem,attrs=mem", "", StoreSpec{"", 21474836480, 0, true, roachpb.Attributes{Attrs: []string{"mem"}}, nil}},
89+
{"type=mem,size=20GiB", "", StoreSpec{"", 21474836480, 0, true, roachpb.Attributes{}, false, nil}},
90+
{"size=20GiB,type=mem", "", StoreSpec{"", 21474836480, 0, true, roachpb.Attributes{}, false, nil}},
91+
{"size=20.5GiB,type=mem", "", StoreSpec{"", 22011707392, 0, true, roachpb.Attributes{}, false, nil}},
92+
{"size=20GiB,type=mem,attrs=mem", "", StoreSpec{"", 21474836480, 0, true, roachpb.Attributes{Attrs: []string{"mem"}}, false, nil}},
9393
{"type=mem,size=20", "store size (20) must be larger than 640 MiB", StoreSpec{}},
9494
{"type=mem,size=", "no value specified for size", StoreSpec{}},
9595
{"type=mem,attrs=ssd", "size must be specified for an in memory store", StoreSpec{}},
@@ -98,8 +98,8 @@ func TestNewStoreSpec(t *testing.T) {
9898
{"path=/mnt/hda1,type=mem,size=20GiB", "path specified for in memory store", StoreSpec{}},
9999

100100
// all together
101-
{"path=/mnt/hda1,attrs=hdd:ssd,size=20GiB", "", StoreSpec{"/mnt/hda1", 21474836480, 0, false, roachpb.Attributes{Attrs: []string{"hdd", "ssd"}}, nil}},
102-
{"type=mem,attrs=hdd:ssd,size=20GiB", "", StoreSpec{"", 21474836480, 0, true, roachpb.Attributes{Attrs: []string{"hdd", "ssd"}}, nil}},
101+
{"path=/mnt/hda1,attrs=hdd:ssd,size=20GiB", "", StoreSpec{"/mnt/hda1", 21474836480, 0, false, roachpb.Attributes{Attrs: []string{"hdd", "ssd"}}, false, nil}},
102+
{"type=mem,attrs=hdd:ssd,size=20GiB", "", StoreSpec{"", 21474836480, 0, true, roachpb.Attributes{Attrs: []string{"hdd", "ssd"}}, false, nil}},
103103

104104
// other error cases
105105
{"", "no value specified", StoreSpec{}},

‎pkg/ccl/baseccl/encryption_spec.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,8 @@ func MatchStoreAndEncryptionSpecs(
182182
storeSpecs.Specs[i].ExtraFields["old-key"] = es.OldKeyPath
183183
intSeconds := int(es.RotationPeriod.Seconds())
184184
storeSpecs.Specs[i].ExtraFields["rotation-duration"] = strconv.Itoa(intSeconds)
185+
// Tell the store we absolutely need the switching env.
186+
storeSpecs.Specs[i].UseSwitchingEnv = true
185187
found = true
186188
break
187189
}

‎pkg/server/config.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -499,6 +499,7 @@ func (cfg *Config) CreateEngines(ctx context.Context) (Engines, error) {
499499
MaxOpenFiles: openFileLimitPerStore,
500500
WarnLargeBatchThreshold: 500 * time.Millisecond,
501501
Settings: cfg.Settings,
502+
UseSwitchingEnv: spec.UseSwitchingEnv,
502503
}
503504

504505
eng, err := engine.NewRocksDB(rocksDBConfig, cache)

‎pkg/storage/engine/rocksdb.go‎

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -68,10 +68,6 @@ var minWALSyncInterval = settings.RegisterDurationSetting(
6868
0*time.Millisecond,
6969
)
7070

71-
// TODO(mberhault): enterprise encryption flags will dictate the need for the
72-
// switching env format. Until then, use an env variable.
73-
var useSwitchingEnv = envutil.EnvOrDefaultBool("COCKROACH_USE_SWITCHING_ENV", false)
74-
7571
//export rocksDBLog
7672
func rocksDBLog(s *C.char, n C.int) {
7773
// Note that rocksdb logging is only enabled if log.V(3) is true
@@ -311,6 +307,9 @@ type RocksDBConfig struct {
311307
WarnLargeBatchThreshold time.Duration
312308
// Settings instance for cluster-wide knobs.
313309
Settings *cluster.Settings
310+
// UseSwitchingEnv is true if the switching env is needed (eg: encryption-at-rest).
311+
// This may force the store version to versionSwitchingEnv if currently lower.
312+
UseSwitchingEnv bool
314313
}
315314

316315
// RocksDB is a wrapper around a RocksDB database instance.
@@ -421,16 +420,18 @@ func (r *RocksDB) open() error {
421420
versionCurrent, existingVersion, versionMinimum)
422421
}
423422

424-
if existingVersion != versionCurrent {
425-
if useSwitchingEnv {
426-
newVersion = versionCurrent
427-
} else {
428-
// If the switching env is not needed, use the older version. This allows downgrade to binaries unaware
429-
// of versionSwitchingEnv.
430-
// TODO(mberhault): once enough releases supporting versionSwitchingEnv have passed, we can upgrade
431-
// to it without worry.
432-
newVersion = versionBeta20160331
433-
}
423+
newVersion = existingVersion
424+
if newVersion == versionNoFile {
425+
// We currently set the default store version one before the switching env
426+
// to allow downgrades to older binaries as long as encryption is not in use.
427+
// TODO(mberhault): once enough releases supporting versionSwitchingEnv have passed, we can upgrade
428+
// to it without worry.
429+
newVersion = versionBeta20160331
430+
}
431+
432+
// Using the switching environment forces the latest version. We can't downgrade!
433+
if r.cfg.UseSwitchingEnv {
434+
newVersion = versionCurrent
434435
}
435436
} else {
436437
if log.V(2) {
@@ -456,14 +457,14 @@ func (r *RocksDB) open() error {
456457
logging_enabled: C.bool(log.V(3)),
457458
num_cpu: C.int(runtime.NumCPU()),
458459
max_open_files: C.int(maxOpenFiles),
459-
use_switching_env: C.bool(useSwitchingEnv),
460+
use_switching_env: C.bool(newVersion == versionCurrent),
460461
})
461462
if err := statusToError(status); err != nil {
462463
return errors.Wrap(err, "could not open rocksdb instance")
463464
}
464465

465466
// Update or add the version file if needed and if on-disk.
466-
if len(r.cfg.Dir) != 0 && existingVersion != newVersion {
467+
if len(r.cfg.Dir) != 0 && existingVersion < newVersion {
467468
if err := writeVersionFile(r.cfg.Dir, newVersion); err != nil {
468469
return err
469470
}

0 commit comments

Comments
 (0)