diff --git a/v1/bundle/bundle.go b/v1/bundle/bundle.go index be320f6a73..0a8eaf55c6 100644 --- a/v1/bundle/bundle.go +++ b/v1/bundle/bundle.go @@ -67,8 +67,9 @@ type Bundle struct { // Raw contains raw bytes representing the bundle's content type Raw struct { - Path string - Value []byte + Path string + Value []byte + module *ModuleFile } // Patch contains an array of objects wherein each object represents the patch operation to be @@ -633,15 +634,6 @@ func (r *Reader) Read() (Bundle, error) { fullPath := r.fullPath(path) bs := buf.Bytes() - if r.lazyLoadingMode { - p := fullPath - if r.name != "" { - p = modulePathWithPrefix(r.name, fullPath) - } - - raw = append(raw, Raw{Path: p, Value: bs}) - } - // Modules are parsed after we've had a chance to read the manifest mf := ModuleFile{ URL: f.URL(), @@ -650,6 +642,15 @@ func (r *Reader) Read() (Bundle, error) { Raw: bs, } modules = append(modules, mf) + + if r.lazyLoadingMode { + p := fullPath + if r.name != "" { + p = modulePathWithPrefix(r.name, fullPath) + } + + raw = append(raw, Raw{Path: p, Value: bs, module: &mf}) + } } else if filepath.Base(path) == WasmFile { bundle.WasmModules = append(bundle.WasmModules, WasmModuleFile{ URL: f.URL(), @@ -1220,6 +1221,19 @@ func (b *Bundle) RegoVersionForFile(path string, def ast.RegoVersion) (ast.RegoV return def, fmt.Errorf("unknown bundle rego-version %d for file '%s'", *version, path) } +func (m *Manifest) RegoVersionForFile(path string) (ast.RegoVersion, error) { + v, err := m.numericRegoVersionForFile(path) + if err != nil { + return ast.RegoUndefined, err + } + + if v == nil { + return ast.RegoUndefined, nil + } + + return ast.RegoVersionFromInt(*v), nil +} + func (m *Manifest) numericRegoVersionForFile(path string) (*int, error) { var version *int diff --git a/v1/bundle/file.go b/v1/bundle/file.go index 2f1cdb8ead..12e159254c 100644 --- a/v1/bundle/file.go +++ b/v1/bundle/file.go @@ -438,15 +438,11 @@ func (it *iterator) Next() (*storage.Update, error) { for _, item := range it.raw { f := file{name: item.Path} - fpath := strings.TrimLeft(normalizePath(filepath.Dir(f.name)), "/.") - if strings.HasSuffix(f.name, RegoExt) { - fpath = strings.Trim(normalizePath(f.name), "/") + p, err := getFileStoragePath(f.name) + if err != nil { + return nil, err } - p, ok := storage.ParsePathEscaped("/" + fpath) - if !ok { - return nil, fmt.Errorf("storage path invalid: %v", f.name) - } f.path = p f.raw = item.Value @@ -506,3 +502,16 @@ func getdepth(path string, isDir bool) int { basePath := strings.Trim(filepath.Dir(filepath.ToSlash(path)), "/") return len(strings.Split(basePath, "/")) } + +func getFileStoragePath(path string) (storage.Path, error) { + fpath := strings.TrimLeft(normalizePath(filepath.Dir(path)), "/.") + if strings.HasSuffix(path, RegoExt) { + fpath = strings.Trim(normalizePath(path), "/") + } + + p, ok := storage.ParsePathEscaped("/" + fpath) + if !ok { + return nil, fmt.Errorf("storage path invalid: %v", path) + } + return p, nil +} diff --git a/v1/bundle/store.go b/v1/bundle/store.go index b9c8c3388d..8278ff6dc8 100644 --- a/v1/bundle/store.go +++ b/v1/bundle/store.go @@ -23,6 +23,8 @@ import ( // BundlesBasePath is the storage path used for storing bundle metadata var BundlesBasePath = storage.MustParsePath("/system/bundles") +var ModulesInfoBasePath = storage.MustParsePath("/system/modules") + // Note: As needed these helpers could be memoized. // ManifestStoragePath is the storage path used for the given named bundle manifest. @@ -59,6 +61,14 @@ func metadataPath(name string) storage.Path { return append(BundlesBasePath, name, "manifest", "metadata") } +func moduleRegoVersionPath(id string) storage.Path { + return append(ModulesInfoBasePath, strings.Trim(id, "/"), "rego_version") +} + +func moduleInfoPath(id string) storage.Path { + return append(ModulesInfoBasePath, strings.Trim(id, "/")) +} + func read(ctx context.Context, store storage.Store, txn storage.Transaction, path storage.Path) (interface{}, error) { value, err := store.Read(ctx, txn, path) if err != nil { @@ -166,6 +176,16 @@ func eraseWasmModulesFromStore(ctx context.Context, store storage.Store, txn sto return suppressNotFound(err) } +func eraseModuleRegoVersionsFromStore(ctx context.Context, store storage.Store, txn storage.Transaction, modules []string) error { + for _, module := range modules { + err := store.Write(ctx, txn, storage.RemoveOp, moduleInfoPath(module), nil) + if err := suppressNotFound(err); err != nil { + return err + } + } + return nil +} + // ReadWasmMetadataFromStore will read Wasm module resolver metadata from the store. func ReadWasmMetadataFromStore(ctx context.Context, store storage.Store, txn storage.Transaction, name string) ([]WasmResolver, error) { path := wasmEntrypointsPath(name) @@ -626,7 +646,7 @@ func eraseBundles(ctx context.Context, store storage.Store, txn storage.Transact return nil, err } - remaining, err := erasePolicies(ctx, store, txn, parserOpts, roots) + remaining, removed, err := erasePolicies(ctx, store, txn, parserOpts, roots) if err != nil { return nil, err } @@ -649,6 +669,11 @@ func eraseBundles(ctx context.Context, store storage.Store, txn storage.Transact } } + err = eraseModuleRegoVersionsFromStore(ctx, store, txn, removed) + if err != nil { + return nil, err + } + return remaining, nil } @@ -668,44 +693,103 @@ func eraseData(ctx context.Context, store storage.Store, txn storage.Transaction return nil } -func erasePolicies(ctx context.Context, store storage.Store, txn storage.Transaction, parserOpts ast.ParserOptions, roots map[string]struct{}) (map[string]*ast.Module, error) { +type moduleInfo struct { + RegoVersion ast.RegoVersion `json:"rego_version"` +} + +func readModuleInfoFromStore(ctx context.Context, store storage.Store, txn storage.Transaction) (map[string]moduleInfo, error) { + value, err := read(ctx, store, txn, ModulesInfoBasePath) + if suppressNotFound(err) != nil { + return nil, err + } + + if value == nil { + return nil, nil + } + + if m, ok := value.(map[string]any); ok { + versions := make(map[string]moduleInfo, len(m)) + + for k, v := range m { + if m0, ok := v.(map[string]any); ok { + if ver, ok := m0["rego_version"]; ok { + if vs, ok := ver.(json.Number); ok { + i, err := vs.Int64() + if err != nil { + return nil, fmt.Errorf("corrupt rego version") + } + versions[k] = moduleInfo{RegoVersion: ast.RegoVersionFromInt(int(i))} + } + } + } + } + return versions, nil + } + + return nil, fmt.Errorf("corrupt rego version") +} + +func erasePolicies(ctx context.Context, store storage.Store, txn storage.Transaction, parserOpts ast.ParserOptions, roots map[string]struct{}) (map[string]*ast.Module, []string, error) { ids, err := store.ListPolicies(ctx, txn) if err != nil { - return nil, err + return nil, nil, err + } + + modulesInfo, err := readModuleInfoFromStore(ctx, store, txn) + if err != nil { + return nil, nil, fmt.Errorf("failed to read module info from store: %w", err) + } + + getRegoVersion := func(modId string) (ast.RegoVersion, bool) { + info, ok := modulesInfo[modId] + if !ok { + return ast.RegoUndefined, false + } + return info.RegoVersion, true } remaining := map[string]*ast.Module{} + var removed []string for _, id := range ids { bs, err := store.GetPolicy(ctx, txn, id) if err != nil { - return nil, err + return nil, nil, err } - module, err := ast.ParseModuleWithOpts(id, string(bs), parserOpts) + + parserOptsCpy := parserOpts + if regoVersion, ok := getRegoVersion(id); ok { + parserOptsCpy.RegoVersion = regoVersion + } + + module, err := ast.ParseModuleWithOpts(id, string(bs), parserOptsCpy) if err != nil { - return nil, err + return nil, nil, err } path, err := module.Package.Path.Ptr() if err != nil { - return nil, err + return nil, nil, err } deleted := false for root := range roots { if RootPathsContain([]string{root}, path) { if err := store.DeletePolicy(ctx, txn, id); err != nil { - return nil, err + return nil, nil, err } deleted = true break } } - if !deleted { + + if deleted { + removed = append(removed, id) + } else { remaining[id] = module } } - return remaining, nil + return remaining, removed, nil } func writeManifestToStore(opts *ActivateOpts, name string, manifest Manifest) error { @@ -758,6 +842,12 @@ func writeDataAndModules(ctx context.Context, store storage.Store, txn storage.T if err := store.UpsertPolicy(ctx, txn, path, mf.Raw); err != nil { return err } + + if regoVersion, err := b.RegoVersionForFile(mf.Path, ast.RegoUndefined); err == nil && regoVersion != ast.RegoUndefined { + if err := write(ctx, store, txn, moduleRegoVersionPath(path), regoVersion.Int()); err != nil { + return fmt.Errorf("failed to write rego version for '%s' in bundle '%s': %w", mf.Path, name, err) + } + } } } else { params.BasePaths = *b.Manifest.Roots @@ -766,6 +856,25 @@ func writeDataAndModules(ctx context.Context, store storage.Store, txn storage.T if err != nil { return fmt.Errorf("store truncate failed for bundle '%s': %v", name, err) } + + for _, f := range b.Raw { + if strings.HasSuffix(f.Path, RegoExt) { + p, err := getFileStoragePath(f.Path) + if err != nil { + return fmt.Errorf("failed get storage path for module '%s' in bundle '%s': %w", f.Path, name, err) + } + + if m := f.module; m != nil { + // 'f.module.Path' contains the module's path as it relates to the bundle root, and can be used for looking up the rego-version. + // 'f.Path' can differ, based on how the bundle reader was initialized. + if regoVersion, err := b.RegoVersionForFile(f.module.Path, ast.RegoUndefined); err == nil && regoVersion != ast.RegoUndefined { + if err := write(ctx, store, txn, moduleRegoVersionPath(p.String()), regoVersion.Int()); err != nil { + return fmt.Errorf("failed to write rego version for '%s' in bundle '%s': %w", f.Path, name, err) + } + } + } + } + } } } diff --git a/v1/bundle/store_test.go b/v1/bundle/store_test.go index 8a2a94472a..19e4548495 100644 --- a/v1/bundle/store_test.go +++ b/v1/bundle/store_test.go @@ -301,15 +301,1057 @@ func TestBundleLazyModeNoPolicyOrData(t *testing.T) { } } +func TestBundleLifecycle_ModuleRegoVersions(t *testing.T) { + type files [][2]string + type bundles map[string]files + type deactivation struct { + bundles map[string]struct{} + expData string + } + type activation struct { + bundles bundles + lazy bool + readWithBundleName bool + expData string + } + + tests := []struct { + note string + updates []interface{} + }{ + // single v0 bundle + { + note: "v0 bundle, lazy, read with bundle name", + updates: []interface{}{ + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"], "rego_version": 0}`}, + {"a/policy.rego", `package a + p[42] { true }`}, + }, + }, + lazy: true, + readWithBundleName: true, + // Lazy mode, bundle reader decides if module name should be prefixed with bundle name; reader initialized with bundle name, so prefix is expected. + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"rego_version":0,"revision":"","roots":["a"]}}}, + "modules":{"bundle1/a/policy.rego":{"rego_version":0}} + } + }`, + }, + deactivation{ + bundles: map[string]struct{}{"bundle1": {}}, + expData: `{"system":{"bundles":{},"modules":{}}}`, + }, + }, + }, + { + note: "v0 bundle, not lazy, read with bundle name", + updates: []interface{}{ + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"], "rego_version": 0}`}, + {"a/policy.rego", `package a + p[42] { true }`}, + }, + }, + lazy: false, + readWithBundleName: true, + // Not lazy mode, bundle store decides that module name should be prefixed with bundle name. + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"rego_version":0,"revision":"","roots":["a"]}}}, + "modules":{"bundle1/a/policy.rego":{"rego_version":0}} + } + }`, + }, + deactivation{ + bundles: map[string]struct{}{"bundle1": {}}, + expData: `{"system":{"bundles":{},"modules":{}}}`, + }, + }, + }, + { + note: "v0 bundle, lazy, read with NO bundle name", + updates: []interface{}{ + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"], "rego_version": 0}`}, + {"a/policy.rego", `package a + p[42] { true }`}, + }, + }, + lazy: true, + readWithBundleName: false, + // Lazy mode, bundle reader decides if module name should be prefixed with bundle name; reader not initialized with bundle name, so prefix not expected. + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"rego_version":0,"revision":"","roots":["a"]}}}, + "modules":{"a/policy.rego":{"rego_version":0}} + } + }`, + }, + deactivation{ + bundles: map[string]struct{}{"bundle1": {}}, + expData: `{"system":{"bundles":{},"modules":{}}}`, + }, + }, + }, + { + note: "v0 bundle, not lazy, read with NO bundle name", + updates: []interface{}{ + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"], "rego_version": 0}`}, + {"a/policy.rego", `package a + p[42] { true }`}, + }, + }, + lazy: false, + readWithBundleName: false, + // Not lazy mode, bundle store decides that module name should be prefixed with bundle name. + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"rego_version":0,"revision":"","roots":["a"]}}}, + "modules":{"bundle1/a/policy.rego":{"rego_version":0}} + } + }`, + }, + deactivation{ + bundles: map[string]struct{}{"bundle1": {}}, + expData: `{"system":{"bundles":{},"modules":{}}}`, + }, + }, + }, + + // single v1 bundle + { + note: "v1 bundle, lazy, read with bundle name", + updates: []interface{}{ + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"], "rego_version": 1}`}, + {"a/policy.rego", `package a + p contains 42 if { true }`}, + }, + }, + lazy: true, + readWithBundleName: true, + // Lazy mode, bundle reader decides if module name should be prefixed with bundle name; reader initialized with bundle name, so prefix is expected. + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"rego_version":1,"revision":"","roots":["a"]}}}, + "modules":{"bundle1/a/policy.rego":{"rego_version":1}} + } + }`, + }, + deactivation{ + bundles: map[string]struct{}{"bundle1": {}}, + expData: `{"system":{"bundles":{},"modules":{}}}`, + }, + }, + }, + { + note: "v1 bundle, not lazy, read with bundle name", + updates: []interface{}{ + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"], "rego_version": 1}`}, + {"a/policy.rego", `package a + p contains 42 if { true }`}, + }, + }, + lazy: false, + readWithBundleName: true, + // Not lazy mode, bundle store decides that module name should be prefixed with bundle name. + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"rego_version":1,"revision":"","roots":["a"]}}}, + "modules":{"bundle1/a/policy.rego":{"rego_version":1}} + } + }`, + }, + deactivation{ + bundles: map[string]struct{}{"bundle1": {}}, + expData: `{"system":{"bundles":{},"modules":{}}}`, + }, + }, + }, + { + note: "v1 bundle, lazy, read with NO bundle name", + updates: []interface{}{ + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"], "rego_version": 1}`}, + {"a/policy.rego", `package a + p contains 42 if { true }`}, + }, + }, + lazy: true, + readWithBundleName: false, + // Lazy mode, bundle reader decides if module name should be prefixed with bundle name; reader not initialized with bundle name, so prefix not expected. + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"rego_version":1,"revision":"","roots":["a"]}}}, + "modules":{"a/policy.rego":{"rego_version":1}} + } + }`, + }, + deactivation{ + bundles: map[string]struct{}{"bundle1": {}}, + expData: `{"system":{"bundles":{},"modules":{}}}`, + }, + }, + }, + { + note: "v1 bundle, not lazy, read with NO bundle name", + updates: []interface{}{ + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"], "rego_version": 1}`}, + {"a/policy.rego", `package a + p contains 42 if { true }`}, + }, + }, + lazy: false, + readWithBundleName: false, + // Not lazy mode, bundle store decides that module name should be prefixed with bundle name. + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"rego_version":1,"revision":"","roots":["a"]}}}, + "modules":{"bundle1/a/policy.rego":{"rego_version":1}} + } + }`, + }, + deactivation{ + bundles: map[string]struct{}{"bundle1": {}}, + expData: `{"system":{"bundles":{},"modules":{}}}`, + }, + }, + }, + + { + note: "custom bundle without rego-version, lazy", + updates: []interface{}{ + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"]}`}, + {"a/policy.rego", `package a + p contains 42 if { true }`}, + }, + }, + lazy: true, + readWithBundleName: true, + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"revision":"","roots":["a"]}}}, + "modules":{"bundle1/a/policy.rego":{"rego_version":1}} + } + }`, + }, + deactivation{ + bundles: map[string]struct{}{"bundle1": {}}, + expData: `{"system":{"bundles":{},"modules":{}}}`, + }, + }, + }, + + { + note: "v0, lazy replaced by non-lazy", + updates: []interface{}{ + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"], "rego_version": 0}`}, + {"a/policy.rego", `package a + p[42] { true }`}, + }, + }, + lazy: true, + readWithBundleName: true, + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"rego_version":0,"revision":"","roots":["a"]}}}, + "modules":{"bundle1/a/policy.rego":{"rego_version":0}} + } + }`, + }, + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"], "rego_version": 1}`}, + {"a/policy.rego", `package a + p contains 42 if { true }`}, + }, + }, + lazy: false, + readWithBundleName: true, + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"rego_version":1,"revision":"","roots":["a"]}}}, + "modules":{"bundle1/a/policy.rego":{"rego_version":1}} + } + }`, + }, + deactivation{ + bundles: map[string]struct{}{"bundle1": {}}, + expData: `{"system":{"bundles":{},"modules":{}}}`, + }, + }, + }, + + { + note: "v0 bundle replaced by v1 bundle, lazy", + updates: []interface{}{ + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"], "rego_version": 0}`}, + {"a/policy.rego", `package a + p[42] { true }`}, + }, + }, + lazy: true, + readWithBundleName: true, + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"rego_version":0,"revision":"","roots":["a"]}}}, + "modules":{"bundle1/a/policy.rego":{"rego_version":0}} + } + }`, + }, + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"], "rego_version": 1}`}, + {"a/policy.rego", `package a + p contains 42 if { true }`}, + }, + }, + lazy: true, + readWithBundleName: true, + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"rego_version":1,"revision":"","roots":["a"]}}}, + "modules":{"bundle1/a/policy.rego":{"rego_version":1}} + } + }`, + }, + deactivation{ + bundles: map[string]struct{}{"bundle1": {}}, + expData: `{"system":{"bundles":{},"modules":{}}}`, + }, + }, + }, + { + note: "v0 bundle replaced by v1 bundle, not lazy", + updates: []interface{}{ + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"], "rego_version": 0}`}, + {"a/policy.rego", `package a + p[42] { true }`}, + }, + }, + lazy: false, + readWithBundleName: true, + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"rego_version":0,"revision":"","roots":["a"]}}}, + "modules":{"bundle1/a/policy.rego":{"rego_version":0}} + } + }`, + }, + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"], "rego_version": 1}`}, + {"a/policy.rego", `package a + p contains 42 if { true }`}, + }, + }, + lazy: false, + readWithBundleName: true, + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"rego_version":1,"revision":"","roots":["a"]}}}, + "modules":{"bundle1/a/policy.rego":{"rego_version":1}} + } + }`, + }, + deactivation{ + bundles: map[string]struct{}{"bundle1": {}}, + expData: `{"system":{"bundles":{},"modules":{}}}`, + }, + }, + }, + { + note: "v0 bundle replaced by custom bundle, not lazy", + updates: []interface{}{ + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"], "rego_version": 0}`}, + {"a/policy.rego", `package a + p[42] { true }`}, + }, + }, + lazy: false, + readWithBundleName: true, + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"rego_version":0,"revision":"","roots":["a"]}}}, + "modules":{"bundle1/a/policy.rego":{"rego_version":0}} + } + }`, + }, + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"]}`}, // no rego-version + {"a/policy.rego", `package a + p contains 42 if { true }`}, + }, + }, + lazy: false, + readWithBundleName: true, + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"revision":"","roots":["a"]}}}, + "modules":{"bundle1/a/policy.rego":{"rego_version":1}} + } + }`, + }, + deactivation{ + bundles: map[string]struct{}{"bundle1": {}}, + expData: `{"system":{"bundles":{},"modules":{}}}`, + }, + }, + }, + + { + note: "v1 bundle replaced by v0 bundle, lazy", + updates: []interface{}{ + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"], "rego_version": 1}`}, + {"a/policy.rego", `package a + p contains 42 if { true }`}, + }, + }, + lazy: true, + readWithBundleName: true, + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"rego_version":1,"revision":"","roots":["a"]}}}, + "modules":{"bundle1/a/policy.rego":{"rego_version":1}} + } + }`, + }, + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"], "rego_version": 0}`}, + {"a/policy.rego", `package a + p[42] { true }`}, + }, + }, + lazy: true, + readWithBundleName: true, + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"rego_version":0,"revision":"","roots":["a"]}}}, + "modules":{"bundle1/a/policy.rego":{"rego_version":0}} + } + }`, + }, + deactivation{ + bundles: map[string]struct{}{"bundle1": {}}, + expData: `{"system":{"bundles":{},"modules":{}}}`, + }, + }, + }, + { + note: "v1 bundle replaced by v0 bundle, not lazy", + updates: []interface{}{ + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"], "rego_version": 1}`}, + {"a/policy.rego", `package a + p contains 42 if { true }`}, + }, + }, + lazy: false, + readWithBundleName: true, + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"rego_version":1,"revision":"","roots":["a"]}}}, + "modules":{"bundle1/a/policy.rego":{"rego_version":1}} + } + }`, + }, + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"], "rego_version": 0}`}, + {"a/policy.rego", `package a + p[42] { true }`}, + }, + }, + lazy: false, + readWithBundleName: true, + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"rego_version":0,"revision":"","roots":["a"]}}}, + "modules":{"bundle1/a/policy.rego":{"rego_version":0}} + } + }`, + }, + deactivation{ + bundles: map[string]struct{}{"bundle1": {}}, + expData: `{"system":{"bundles":{},"modules":{}}}`, + }, + }, + }, + { + note: "custom bundle replaced by v0 bundle, lazy", + updates: []interface{}{ + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"]}`}, // no rego-version + {"a/policy.rego", `package a + p contains 42 if { true }`}, + }, + }, + lazy: true, + readWithBundleName: true, + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"revision":"","roots":["a"]}}}, + "modules":{"bundle1/a/policy.rego":{"rego_version":1}} + } + }`, + }, + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"], "rego_version": 0}`}, + {"a/policy.rego", `package a + p[42] { true }`}, + }, + }, + lazy: true, + readWithBundleName: true, + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"rego_version":0,"revision":"","roots":["a"]}}}, + "modules":{"bundle1/a/policy.rego":{"rego_version":0}} + } + }`, + }, + deactivation{ + bundles: map[string]struct{}{"bundle1": {}}, + expData: `{"system":{"bundles":{},"modules":{}}}`, + }, + }, + }, + + { + note: "multiple v0 bundles, all dropped", + updates: []interface{}{ + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"], "rego_version": 0}`}, + {"a/policy.rego", `package a + p[42] { true }`}, + }, + "bundle2": { + {"/.manifest", `{"roots": ["b"], "rego_version": 0}`}, + {"b/policy.rego", `package b + p[42] { true }`}, + }, + }, + lazy: true, + readWithBundleName: true, + expData: `{ + "system":{ + "bundles":{ + "bundle1":{"etag":"bar","manifest":{"rego_version":0,"revision":"","roots":["a"]}}, + "bundle2":{"etag":"bar","manifest":{"rego_version":0,"revision":"","roots":["b"]}} + }, + "modules":{"bundle1/a/policy.rego":{"rego_version":0},"bundle2/b/policy.rego":{"rego_version":0}} + } + }`, + }, + deactivation{ + bundles: map[string]struct{}{"bundle1": {}, "bundle2": {}}, + expData: `{"system":{"bundles":{},"modules":{}}}`, + }, + }, + }, + + { + note: "multiple v0 bundles, one dropped", + updates: []interface{}{ + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"], "rego_version": 0}`}, + {"a/policy.rego", `package a + p[42] { true }`}, + }, + "bundle2": { + {"/.manifest", `{"roots": ["b"], "rego_version": 0}`}, + {"b/policy.rego", `package b + p[42] { true }`}, + }, + }, + lazy: true, + readWithBundleName: true, + expData: `{ + "system":{ + "bundles":{ + "bundle1":{"etag":"bar","manifest":{"rego_version":0,"revision":"","roots":["a"]}}, + "bundle2":{"etag":"bar","manifest":{"rego_version":0,"revision":"","roots":["b"]}} + }, + "modules":{"bundle1/a/policy.rego":{"rego_version":0},"bundle2/b/policy.rego":{"rego_version":0}} + } + }`, + }, + deactivation{ + bundles: map[string]struct{}{"bundle1": {}}, + expData: `{ + "system":{ + "bundles":{ + "bundle2":{"etag":"bar","manifest":{"rego_version":0,"revision":"","roots":["b"]}} + }, + "modules":{"bundle2/b/policy.rego":{"rego_version":0}} + } + }`, + }, + }, + }, + + { + note: "v0 bundle with v1 bundle added", + updates: []interface{}{ + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a"], "rego_version": 0}`}, + {"a/policy.rego", `package a + p[42] { true }`}, + }, + }, + lazy: true, + readWithBundleName: true, + expData: `{ + "system":{ + "bundles":{"bundle1":{"etag":"bar","manifest":{"rego_version":0,"revision":"","roots":["a"]}}}, + "modules":{"bundle1/a/policy.rego":{"rego_version":0}} + } + }`, + }, + activation{ + bundles: bundles{ + "bundle2": { + {"/.manifest", `{"roots": ["b"], "rego_version": 1}`}, + {"b/policy.rego", `package b + p contains 42 if { true }`}, + }, + }, + lazy: true, + readWithBundleName: true, + expData: `{ + "system":{ + "bundles":{ + "bundle1":{"etag":"bar","manifest":{"rego_version":0,"revision":"","roots":["a"]}}, + "bundle2":{"etag":"bar","manifest":{"rego_version":1,"revision":"","roots":["b"]}} + }, + "modules":{"bundle1/a/policy.rego":{"rego_version":0},"bundle2/b/policy.rego":{"rego_version":1}} + } + }`, + }, + deactivation{ + bundles: map[string]struct{}{"bundle1": {}, "bundle2": {}}, + expData: `{"system":{"bundles":{},"modules":{}}}`, + }, + }, + }, + + { + note: "mixed-version bundles, lazy", + updates: []interface{}{ + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a", "b"], "rego_version": 0, "file_rego_versions": {"/b/policy.rego": 1}}`}, + {"a/policy.rego", `package a + p[42] { true }`}, + {"b/policy.rego", `package b + p contains 42 if { true }`}, + }, + "bundle2": { + {"/.manifest", `{"roots": ["c", "d"], "rego_version": 1, "file_rego_versions": {"/d/policy.rego": 0}}`}, + {"c/policy.rego", `package c + p contains 42 if { true }`}, + {"d/policy.rego", `package d + p[42] { true }`}, + }, + }, + lazy: true, + readWithBundleName: true, + expData: `{ + "system":{ + "bundles":{ + "bundle1":{"etag":"bar","manifest":{"file_rego_versions":{"/b/policy.rego":1},"rego_version":0,"revision":"","roots":["a","b"]}}, + "bundle2":{"etag":"bar","manifest":{"file_rego_versions":{"/d/policy.rego":0},"rego_version":1,"revision":"","roots":["c","d"]}} + }, + "modules":{ + "bundle1/a/policy.rego":{"rego_version":0}, + "bundle1/b/policy.rego":{"rego_version":1}, + "bundle2/c/policy.rego":{"rego_version":1}, + "bundle2/d/policy.rego":{"rego_version":0} + } + } + }`, + }, + // replacing bundles + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a", "b"], "rego_version": 0, "file_rego_versions": {"/b/policy2.rego": 1}}`}, + {"a/policy2.rego", `package a + q[42] { true }`}, + {"b/policy2.rego", `package b + q contains 42 if { true }`}, + }, + "bundle2": { + {"/.manifest", `{"roots": ["c", "d"], "rego_version": 1, "file_rego_versions": {"/d/policy2.rego": 0}}`}, + {"c/policy2.rego", `package c + q contains 42 if { true }`}, + {"d/policy2.rego", `package d + q[42] { true }`}, + }, + }, + lazy: true, + readWithBundleName: true, + expData: `{ + "system":{ + "bundles":{ + "bundle1":{"etag":"bar","manifest":{"file_rego_versions":{"/b/policy2.rego":1},"rego_version":0,"revision":"","roots":["a","b"]}}, + "bundle2":{"etag":"bar","manifest":{"file_rego_versions":{"/d/policy2.rego":0},"rego_version":1,"revision":"","roots":["c","d"]}} + }, + "modules":{ + "bundle1/a/policy2.rego":{"rego_version":0}, + "bundle1/b/policy2.rego":{"rego_version":1}, + "bundle2/c/policy2.rego":{"rego_version":1}, + "bundle2/d/policy2.rego":{"rego_version":0} + } + } + }`, + }, + deactivation{ + bundles: map[string]struct{}{"bundle1": {}, "bundle2": {}}, + expData: `{"system":{"bundles":{},"modules":{}}}`, + }, + }, + }, + { + note: "mixed-version bundles, lazy, read with NO bundle name", + updates: []interface{}{ + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a", "b"], "rego_version": 0, "file_rego_versions": {"/b/policy.rego": 1}}`}, + {"a/policy.rego", `package a + p[42] { true }`}, + {"b/policy.rego", `package b + p contains 42 if { true }`}, + }, + "bundle2": { + {"/.manifest", `{"roots": ["c", "d"], "rego_version": 1, "file_rego_versions": {"/d/policy.rego": 0}}`}, + {"c/policy.rego", `package c + p contains 42 if { true }`}, + {"d/policy.rego", `package d + p[42] { true }`}, + }, + }, + lazy: true, + readWithBundleName: false, + expData: `{ + "system":{ + "bundles":{ + "bundle1":{"etag":"bar","manifest":{"file_rego_versions":{"/b/policy.rego":1},"rego_version":0,"revision":"","roots":["a","b"]}}, + "bundle2":{"etag":"bar","manifest":{"file_rego_versions":{"/d/policy.rego":0},"rego_version":1,"revision":"","roots":["c","d"]}} + }, + "modules":{ + "a/policy.rego":{"rego_version":0}, + "b/policy.rego":{"rego_version":1}, + "c/policy.rego":{"rego_version":1}, + "d/policy.rego":{"rego_version":0} + } + } + }`, + }, + // replacing bundles + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a", "b"], "rego_version": 0, "file_rego_versions": {"/b/policy2.rego": 1}}`}, + {"a/policy2.rego", `package a + q[42] { true }`}, + {"b/policy2.rego", `package b + q contains 42 if { true }`}, + }, + "bundle2": { + {"/.manifest", `{"roots": ["c", "d"], "rego_version": 1, "file_rego_versions": {"/d/policy2.rego": 0}}`}, + {"c/policy2.rego", `package c + q contains 42 if { true }`}, + {"d/policy2.rego", `package d + q[42] { true }`}, + }, + }, + lazy: true, + readWithBundleName: false, + expData: `{ + "system":{ + "bundles":{ + "bundle1":{"etag":"bar","manifest":{"file_rego_versions":{"/b/policy2.rego":1},"rego_version":0,"revision":"","roots":["a","b"]}}, + "bundle2":{"etag":"bar","manifest":{"file_rego_versions":{"/d/policy2.rego":0},"rego_version":1,"revision":"","roots":["c","d"]}} + }, + "modules":{ + "a/policy2.rego":{"rego_version":0}, + "b/policy2.rego":{"rego_version":1}, + "c/policy2.rego":{"rego_version":1}, + "d/policy2.rego":{"rego_version":0} + } + } + }`, + }, + deactivation{ + bundles: map[string]struct{}{"bundle1": {}, "bundle2": {}}, + expData: `{"system":{"bundles":{},"modules":{}}}`, + }, + }, + }, + { + note: "mixed-version bundles, not lazy", + updates: []interface{}{ + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a", "b"], "rego_version": 0, "file_rego_versions": {"/b/policy.rego": 1}}`}, + {"a/policy.rego", `package a + p[42] { true }`}, + {"b/policy.rego", `package b + p contains 42 if { true }`}, + }, + "bundle2": { + {"/.manifest", `{"roots": ["c", "d"], "rego_version": 1, "file_rego_versions": {"/d/policy.rego": 0}}`}, + {"c/policy.rego", `package c + p contains 42 if { true }`}, + {"d/policy.rego", `package d + p[42] { true }`}, + }, + }, + lazy: false, + readWithBundleName: true, + expData: `{ + "system":{ + "bundles":{ + "bundle1":{"etag":"bar","manifest":{"file_rego_versions":{"/b/policy.rego":1},"rego_version":0,"revision":"","roots":["a","b"]}}, + "bundle2":{"etag":"bar","manifest":{"file_rego_versions":{"/d/policy.rego":0},"rego_version":1,"revision":"","roots":["c","d"]}} + }, + "modules":{ + "bundle1/a/policy.rego":{"rego_version":0}, + "bundle1/b/policy.rego":{"rego_version":1}, + "bundle2/c/policy.rego":{"rego_version":1}, + "bundle2/d/policy.rego":{"rego_version":0} + } + } + }`, + }, + // replacing bundles + activation{ + bundles: bundles{ + "bundle1": { + {"/.manifest", `{"roots": ["a", "b"], "rego_version": 0, "file_rego_versions": {"/b/policy2.rego": 1}}`}, + {"a/policy2.rego", `package a + q[42] { true }`}, + {"b/policy2.rego", `package b + q contains 42 if { true }`}, + }, + "bundle2": { + {"/.manifest", `{"roots": ["c", "d"], "rego_version": 1, "file_rego_versions": {"/d/policy2.rego": 0}}`}, + {"c/policy2.rego", `package c + q contains 42 if { true }`}, + {"d/policy2.rego", `package d + q[42] { true }`}, + }, + }, + lazy: false, + readWithBundleName: true, + expData: `{ + "system":{ + "bundles":{ + "bundle1":{"etag":"bar","manifest":{"file_rego_versions":{"/b/policy2.rego":1},"rego_version":0,"revision":"","roots":["a","b"]}}, + "bundle2":{"etag":"bar","manifest":{"file_rego_versions":{"/d/policy2.rego":0},"rego_version":1,"revision":"","roots":["c","d"]}} + }, + "modules":{ + "bundle1/a/policy2.rego":{"rego_version":0}, + "bundle1/b/policy2.rego":{"rego_version":1}, + "bundle2/c/policy2.rego":{"rego_version":1}, + "bundle2/d/policy2.rego":{"rego_version":0} + } + } + }`, + }, + deactivation{ + bundles: map[string]struct{}{"bundle1": {}, "bundle2": {}}, + expData: `{"system":{"bundles":{},"modules":{}}}`, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.note, func(t *testing.T) { + ctx := context.Background() + mockStore := mock.New() + + compiler := ast.NewCompiler() + m := metrics.New() + + for _, update := range tc.updates { + if act, ok := update.(activation); ok { + bundles := map[string]*Bundle{} + for bundleName, files := range act.bundles { + buf := archive.MustWriteTarGz(files) + loader := NewTarballLoaderWithBaseURL(buf, "") + br := NewCustomReader(loader).WithBundleEtag("bar").WithLazyLoadingMode(act.lazy) + if act.readWithBundleName { + br = br.WithBundleName(bundleName) + } + + bundle, err := br.Read() + if err != nil { + t.Fatal(err) + } + + bundles[bundleName] = &bundle + } + + txn := storage.NewTransactionOrDie(ctx, mockStore, storage.WriteParams) + + err := Activate(&ActivateOpts{ + Ctx: ctx, + Store: mockStore, + Txn: txn, + Compiler: compiler, + Metrics: m, + Bundles: bundles, + }) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + err = mockStore.Commit(ctx, txn) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + // Start read transaction + txn = storage.NewTransactionOrDie(ctx, mockStore) + + actual, err := mockStore.Read(ctx, txn, storage.MustParsePath("/")) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + expectedRaw := act.expData + expected := loadExpectedSortedResult(expectedRaw) + if !reflect.DeepEqual(expected, actual) { + t.Errorf("expected:\n\n%s\n\ngot:\n\n%s", expectedRaw, string(util.MustMarshalJSON(actual))) + } + + // Stop the "read" transaction + mockStore.Abort(ctx, txn) + } else if deact, ok := update.(deactivation); ok { + txn := storage.NewTransactionOrDie(ctx, mockStore, storage.WriteParams) + + err := Deactivate(&DeactivateOpts{ + Ctx: ctx, + Store: mockStore, + Txn: txn, + BundleNames: deact.bundles, + }) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + err = mockStore.Commit(ctx, txn) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + // Start read transaction + txn = storage.NewTransactionOrDie(ctx, mockStore) + + actual, err := mockStore.Read(ctx, txn, storage.MustParsePath("/")) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + expectedRaw := deact.expData // `{"system": {"bundles": {}, "modules": {}}}` + expected := loadExpectedSortedResult(expectedRaw) + if !reflect.DeepEqual(expected, actual) { + t.Errorf("expected:\n\n%s\n\ngot:\n\n%s", expectedRaw, string(util.MustMarshalJSON(actual))) + } + + // Stop the "read" transaction + mockStore.Abort(ctx, txn) + } + } + }) + } +} + func TestBundleLazyModeLifecycleRaw(t *testing.T) { files := [][2]string{ {"/a/b/c/data.json", "[1,2,3]"}, {"/a/b/d/data.json", "true"}, {"/a/b/y/data.yaml", `foo: 1`}, - {"/example/example.rego", `package example`}, + {"/example/example.rego", `package example + p contains 42 if { true } + `}, + {"/example/example_v0.rego", `package example + q[42] { true } + `}, {"/authz/allow/policy.wasm", `wasm-module`}, {"/data.json", `{"x": {"y": true}, "a": {"b": {"z": true}}}`}, - {"/.manifest", `{"revision": "foo", "roots": ["a", "example", "x", "authz"],"wasm":[{"entrypoint": "authz/allow", "module": "/authz/allow/policy.wasm"}]}`}, + {"/.manifest", `{ + "revision": "foo", + "roots": ["a", "example", "x", "authz"], + "wasm":[{"entrypoint": "authz/allow", "module": "/authz/allow/policy.wasm"}], + "rego_version": 1, + "file_rego_versions": {"/example/example_v0.rego": 0} + }`}, } buf := archive.MustWriteTarGz(files) @@ -409,20 +1451,32 @@ func TestBundleLazyModeLifecycleRaw(t *testing.T) { "entrypoint": "authz/allow", "module": "/authz/allow/policy.wasm" } - ] + ], + "rego_version": 1, + "file_rego_versions": { + "/example/example_v0.rego": 0 + } }, "etag": "bar", "wasm": { "/authz/allow/policy.wasm": "d2FzbS1tb2R1bGU=" } } + }, + "modules":{ + "example/example.rego":{ + "rego_version":1 + }, + "example/example_v0.rego":{ + "rego_version":0 + } } } } ` expected := loadExpectedSortedResult(expectedRaw) if !reflect.DeepEqual(expected, actual) { - t.Errorf("expected %v, got %v", expectedRaw, string(util.MustMarshalJSON(actual))) + t.Errorf("expected %s, got %s", expectedRaw, string(util.MustMarshalJSON(actual))) } // Ensure that the extra module was included @@ -465,7 +1519,7 @@ func TestBundleLazyModeLifecycleRaw(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %s", err) } - expectedRaw = `{"system": {"bundles": {}}}` + expectedRaw = `{"system": {"bundles": {}, "modules": {}}}` expected = loadExpectedSortedResult(expectedRaw) if !reflect.DeepEqual(expected, actual) { t.Errorf("expected %v, got %v", expectedRaw, string(util.MustMarshalJSON(actual))) @@ -535,11 +1589,14 @@ func TestBundleLazyModeLifecycle(t *testing.T) { "mod1": ast.MustParseModule("package x\np = true"), } - mod1 := "package a\np = true" - mod2 := "package b\np = true" + // v1 bundle + + mod1 := `package a + p contains 42 if { true } + ` b1Files := [][2]string{ - {"/.manifest", `{"roots": ["a"]}`}, + {"/.manifest", `{"roots": ["a"], "rego_version": 1}`}, {"a/policy.rego", mod1}, {"/data.json", `{"a": {"b": "foo"}}`}, } @@ -553,8 +1610,14 @@ func TestBundleLazyModeLifecycle(t *testing.T) { t.Fatal(err) } + // v0 bundle + + mod2 := `package b + p[42] { true } + ` + b2Files := [][2]string{ - {"/.manifest", `{"roots": ["b", "c"]}`}, + {"/.manifest", `{"roots": ["b", "c"], "rego_version": 0}`}, {"b/policy.rego", mod2}, {"/data.json", `{}`}, } @@ -631,17 +1694,27 @@ func TestBundleLazyModeLifecycle(t *testing.T) { "bundle1": { "manifest": { "revision": "", - "roots": ["a"] + "roots": ["a"], + "rego_version": 1 }, "etag": "foo" }, "bundle2": { "manifest": { "revision": "", - "roots": ["b", "c"] + "roots": ["b", "c"], + "rego_version": 0 }, "etag": "" } + }, + "modules":{ + "bundle1/a/policy.rego":{ + "rego_version":1 + }, + "bundle2/b/policy.rego":{ + "rego_version":0 + } } } } @@ -691,7 +1764,7 @@ func TestBundleLazyModeLifecycle(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %s", err) } - expectedRaw = `{"system": {"bundles": {}}}` + expectedRaw = `{"system": {"bundles": {}, "modules": {}}}` expected = loadExpectedSortedResult(expectedRaw) if !reflect.DeepEqual(expected, actual) { t.Errorf("expected %v, got %v", expectedRaw, string(util.MustMarshalJSON(actual))) @@ -801,6 +1874,11 @@ func TestBundleLazyModeLifecycleRawNoBundleRoots(t *testing.T) { }, "etag": "foo" } + }, + "modules":{ + "example/example.rego":{ + "rego_version":1 + } } } } @@ -993,6 +2071,11 @@ func TestBundleLazyModeLifecycleRawNoBundleRootsDiskStorage(t *testing.T) { }, "etag": "foo" } + }, + "modules":{ + "example/example.rego":{ + "rego_version":1 + } } } } @@ -1165,7 +2248,12 @@ func TestBundleLazyModeLifecycleNoBundleRoots(t *testing.T) { }, "etag": "" } - } + }, + "modules":{ + "bundle1/a/policy.rego":{ + "rego_version":1 + } + } } }` @@ -1375,7 +2463,12 @@ func TestBundleLazyModeLifecycleNoBundleRootsDiskStorage(t *testing.T) { }, "etag": "" } - } + }, + "modules":{ + "bundle1/a/policy.rego":{ + "rego_version":1 + } + } } }` @@ -1608,7 +2701,12 @@ func TestBundleLazyModeLifecycleMixBundleTypeActivationDiskStorage(t *testing.T) }, "etag": "" } - } + }, + "modules":{ + "bundle1/a/policy.rego":{ + "rego_version":1 + } + } } }` @@ -1740,7 +2838,12 @@ func TestBundleLazyModeLifecycleOldBundleEraseDiskStorage(t *testing.T) { }, "etag": "" } - } + }, + "modules":{ + "bundle1/a/policy.rego":{ + "rego_version":1 + } + } } }` @@ -1952,7 +3055,12 @@ func TestBundleLazyModeLifecycleRestoreBackupDB(t *testing.T) { }, "etag": "" } - } + }, + "modules":{ + "bundle1/a/policy.rego":{ + "rego_version":1 + } + } } }` @@ -2031,7 +3139,12 @@ func TestBundleLazyModeLifecycleRestoreBackupDB(t *testing.T) { }, "etag": "" } - } + }, + "modules":{ + "bundle1/a/policy.rego":{ + "rego_version":1 + } + } } }` @@ -2317,6 +3430,14 @@ func TestDeltaBundleLazyModeLifecycleDiskStorage(t *testing.T) { }, "etag": "" } + }, + "modules":{ + "bundle1/a/policy.rego":{ + "rego_version":1 + }, + "bundle2/b/policy.rego":{ + "rego_version":1 + } } } }` @@ -3178,6 +4299,14 @@ func TestDeltaBundleLazyModeLifecycle(t *testing.T) { }, "etag": "" } + }, + "modules":{ + "bundle1/policy.rego":{ + "rego_version":1 + }, + "bundle2/policy.rego":{ + "rego_version":1 + } } } }` @@ -3468,6 +4597,14 @@ func TestDeltaBundleLazyModeWithDefaultRules(t *testing.T) { }, "etag": "" } + }, + "modules":{ + "bundle1/policy.rego":{ + "rego_version":1 + }, + "bundle2/policy.rego":{ + "rego_version":1 + } } } }` @@ -3615,6 +4752,14 @@ func TestBundleLifecycle(t *testing.T) { }, "etag": "" } + }, + "modules": { + "bundle1/a/policy.rego": { + "rego_version": 1 + }, + "bundle2/b/policy.rego": { + "rego_version": 1 + } } } } @@ -3661,7 +4806,7 @@ func TestBundleLifecycle(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %s", err) } - expectedRaw = `{"system": {"bundles": {}}}` + expectedRaw = `{"system": {"bundles": {}, "modules": {}}}` assertEqual(t, tc.readAst, expectedRaw, actual) mockStore.AssertValid(t) @@ -3929,6 +5074,14 @@ func TestDeltaBundleLifecycle(t *testing.T) { }, "etag": "" } + }, + "modules":{ + "bundle1/a/policy.rego":{ + "rego_version":1 + }, + "bundle2/b/policy.rego":{ + "rego_version":1 + } } } }` @@ -4367,7 +5520,7 @@ func TestErasePolicies(t *testing.T) { for _, root := range tc.roots { roots[root] = struct{}{} } - remaining, err := erasePolicies(ctx, mockStore, txn, ast.ParserOptions{}, roots) + remaining, _, err := erasePolicies(ctx, mockStore, txn, ast.ParserOptions{}, roots) if !tc.expectErr && err != nil { t.Fatalf("unepected error: %s", err) } else if tc.expectErr && err == nil { diff --git a/v1/plugins/bundle/plugin_test.go b/v1/plugins/bundle/plugin_test.go index 263dfed9c5..c90e7785f8 100644 --- a/v1/plugins/bundle/plugin_test.go +++ b/v1/plugins/bundle/plugin_test.go @@ -106,7 +106,13 @@ func TestPluginOneShot(t *testing.T) { } data, err := manager.Store.Read(ctx, txn, storage.Path{}) - expData := util.MustUnmarshalJSON([]byte(`{"foo": {"bar": 1, "baz": "qux"}, "system": {"bundles": {"test-bundle": {"etag": "foo", "manifest": {"revision": "quickbrownfaux", "roots": [""]}}}}}`)) + expData := util.MustUnmarshalJSON([]byte(`{ + "foo": {"bar": 1, "baz": "qux"}, + "system": { + "bundles": {"test-bundle": {"etag": "foo", "manifest": {"revision": "quickbrownfaux", "roots": [""]}}}, + "modules": {"test-bundle/foo/bar": {"rego_version": 1}} + } + }`)) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(data, expData) { @@ -961,7 +967,13 @@ func TestPluginStartLazyLoadInMem(t *testing.T) { t.Fatal(err) } - expected := `{"p": "x1", "q": "x2", "system": {"bundles": {"test-1": {"etag": "", "manifest": {"revision": "", "roots": ["p", "authz"]}}, "test-2": {"etag": "", "manifest": {"revision": "", "roots": ["q"]}}}}}` + expected := `{ + "p": "x1", "q": "x2", + "system": { + "bundles": {"test-1": {"etag": "", "manifest": {"revision": "", "roots": ["p", "authz"]}}, "test-2": {"etag": "", "manifest": {"revision": "", "roots": ["q"]}}}, + "modules": {"test-1/bar/policy.rego": {"rego_version": 1}} + } + }` if rm.readAst { expData := ast.MustParseTerm(expected) if ast.Compare(data, expData) != 0 { @@ -1028,11 +1040,11 @@ func TestPluginOneShotDiskStorageMetrics(t *testing.T) { t.Errorf("%s: expected %v, got %v", name, exp, act) } name = "disk_written_keys" - if exp, act := 6, met.Counter(name).Value(); act.(uint64) != uint64(exp) { + if exp, act := 7, met.Counter(name).Value(); act.(uint64) != uint64(exp) { t.Errorf("%s: expected %v, got %v", name, exp, act) } name = "disk_read_keys" - if exp, act := 13, met.Counter(name).Value(); act.(uint64) != uint64(exp) { + if exp, act := 14, met.Counter(name).Value(); act.(uint64) != uint64(exp) { t.Errorf("%s: expected %v, got %v", name, exp, act) } name = "disk_read_bytes" @@ -1074,7 +1086,13 @@ func TestPluginOneShotDiskStorageMetrics(t *testing.T) { } data, err := manager.Store.Read(ctx, txn, storage.Path{}) - expData := util.MustUnmarshalJSON([]byte(`{"foo": {"bar": 1, "baz": "qux"}, "system": {"bundles": {"test-bundle": {"etag": "", "manifest": {"revision": "quickbrownfaux", "roots": [""]}}}}}`)) + expData := util.MustUnmarshalJSON([]byte(`{ + "foo": {"bar": 1, "baz": "qux"}, + "system": { + "bundles": {"test-bundle": {"etag": "", "manifest": {"revision": "quickbrownfaux", "roots": [""]}}}, + "modules": {"test-bundle/foo/bar": {"rego_version": 1}} + } + }`)) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(data, expData) { @@ -1175,7 +1193,13 @@ func TestPluginOneShotDeltaBundle(t *testing.T) { if err != nil { t.Fatal(err) } - expData := util.MustUnmarshalJSON([]byte(`{"a": {"baz": "bux", "foo": ["hello", "world"]}, "system": {"bundles": {"test-bundle": {"etag": "foo", "manifest": {"revision": "delta", "roots": ["a"]}}}}}`)) + expData := util.MustUnmarshalJSON([]byte(`{ + "a": {"baz": "bux", "foo": ["hello", "world"]}, + "system": { + "bundles": {"test-bundle": {"etag": "foo", "manifest": {"revision": "delta", "roots": ["a"]}}}, + "modules": {"test-bundle/a/policy.rego": {"rego_version": 1}} + } + }`)) if !reflect.DeepEqual(data, expData) { t.Fatalf("Bad data content. Exp:\n%#v\n\nGot:\n\n%#v", expData, data) } @@ -1274,7 +1298,13 @@ func TestPluginOneShotDeltaBundleWithAstStore(t *testing.T) { if err != nil { t.Fatal(err) } - expData := ast.MustParseTerm(`{"a": {"baz": "bux", "foo": ["hello", "world"]}, "system": {"bundles": {"test-bundle": {"etag": "foo", "manifest": {"revision": "delta", "roots": ["a"]}}}}}`) + expData := ast.MustParseTerm(`{ + "a": {"baz": "bux", "foo": ["hello", "world"]}, + "system": { + "bundles": {"test-bundle": {"etag": "foo", "manifest": {"revision": "delta", "roots": ["a"]}}}, + "modules": {"test-bundle/a/policy.rego": {"rego_version": 1}} + } + }`) if ast.Compare(data, expData) != 0 { t.Fatalf("Bad data content. Exp:\n%#v\n\nGot:\n\n%#v", expData, data) } @@ -1455,7 +1485,13 @@ func TestPluginOneShotBundlePersistence(t *testing.T) { } data, err := manager.Store.Read(ctx, txn, storage.Path{}) - expData := util.MustUnmarshalJSON([]byte(`{"foo": {"bar": 1, "baz": "qux"}, "system": {"bundles": {"test-bundle": {"etag": "foo", "manifest": {"revision": "quickbrownfaux", "roots": [""]}}}}}`)) + expData := util.MustUnmarshalJSON([]byte(`{ + "foo": {"bar": 1, "baz": "qux"}, + "system": { + "bundles": {"test-bundle": {"etag": "foo", "manifest": {"revision": "quickbrownfaux", "roots": [""]}}}, + "modules": {"test-bundle/foo/bar.rego": {"rego_version": 1}} + } + }`)) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(data, expData) { @@ -1639,7 +1675,13 @@ corge contains 1 if { } data, err := manager.Store.Read(ctx, txn, storage.Path{}) - expData := util.MustUnmarshalJSON([]byte(`{"foo": {"bar": 1, "baz": "qux"}, "system": {"bundles": {"test-bundle": {"etag": "foo", "manifest": {"revision": "quickbrownfaux", "roots": [""]}}}}}`)) + expData := util.MustUnmarshalJSON([]byte(`{ + "foo": {"bar": 1, "baz": "qux"}, + "system": { + "bundles": {"test-bundle": {"etag": "foo", "manifest": {"revision": "quickbrownfaux", "roots": [""]}}}, + "modules": {"test-bundle/foo/bar.rego": {"rego_version": 1}} + } + }`)) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(data, expData) { @@ -1926,14 +1968,29 @@ corge contains 1 if { } data, err := manager.Store.Read(ctx, txn, storage.Path{}) - var regoVersion string + + var manifestRegoVersion string + if tc.bundleRegoVersion != nil { + manifestRegoVersion = fmt.Sprintf(`, "rego_version": %d`, bundleRegoVersion(*tc.bundleRegoVersion)) + } else { + manifestRegoVersion = "" + } + + var moduleRegoVersion int if tc.bundleRegoVersion != nil { - regoVersion = fmt.Sprintf(`, "rego_version": %d`, bundleRegoVersion(*tc.bundleRegoVersion)) + moduleRegoVersion = tc.bundleRegoVersion.Int() } else { - regoVersion = "" + moduleRegoVersion = ast.DefaultRegoVersion.Int() } - expData := util.MustUnmarshalJSON([]byte(fmt.Sprintf(`{"foo": {"bar": 1, "baz": "qux"}, "system": {"bundles": {"test-bundle": {"etag": "foo", "manifest": {"revision": "quickbrownfaux"%s, "roots": [""]}}}}}`, - regoVersion))) + + expData := util.MustUnmarshalJSON([]byte(fmt.Sprintf(`{ + "foo": {"bar": 1, "baz": "qux"}, + "system": { + "bundles": {"test-bundle": {"etag": "foo", "manifest": {"revision": "quickbrownfaux"%s, "roots": [""]}}}, + "modules": {"test-bundle/foo/bar.rego": {"rego_version": %d}} + } + }`, + manifestRegoVersion, moduleRegoVersion))) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(data, expData) { @@ -2115,7 +2172,13 @@ func TestLoadAndActivateBundlesFromDisk(t *testing.T) { } data, err := manager.Store.Read(ctx, txn, storage.Path{}) - expData := util.MustUnmarshalJSON([]byte(`{"foo": {"bar": 1, "baz": "qux"}, "system": {"bundles": {"test-bundle": {"etag": "", "manifest": {"revision": "quickbrownfaux", "roots": [""]}}}}}`)) + expData := util.MustUnmarshalJSON([]byte(`{ + "foo": {"bar": 1, "baz": "qux"}, + "system": { + "bundles": {"test-bundle": {"etag": "", "manifest": {"revision": "quickbrownfaux", "roots": [""]}}}, + "modules": {"test-bundle/foo/bar.rego": {"rego_version": 1}} + } + }`)) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(data, expData) { @@ -2195,7 +2258,13 @@ func TestLoadAndActivateBundlesFromDiskReservedChars(t *testing.T) { } data, err := manager.Store.Read(ctx, txn, storage.Path{}) - expData := util.MustUnmarshalJSON([]byte(`{"foo": {"bar": 1, "baz": "qux"}, "system": {"bundles": {"test?bundle=opa": {"etag": "", "manifest": {"revision": "quickbrownfaux", "roots": [""]}}}}}`)) + expData := util.MustUnmarshalJSON([]byte(`{ + "foo": {"bar": 1, "baz": "qux"}, + "system": { + "bundles": {"test?bundle=opa": {"etag": "", "manifest": {"revision": "quickbrownfaux", "roots": [""]}}}, + "modules": {"test/foo/bar.rego": {"rego_version": 1}} + } + }`)) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(data, expData) { @@ -2417,6 +2486,7 @@ corge contains 2 if { } else { txn := storage.NewTransactionOrDie(ctx, manager.Store) fatal := func(args ...any) { + t.Helper() manager.Store.Abort(ctx, txn) t.Fatal(args...) } @@ -2438,7 +2508,13 @@ corge contains 2 if { } data, err := manager.Store.Read(ctx, txn, storage.Path{}) - expData := util.MustUnmarshalJSON([]byte(`{"foo": {"bar": 1, "baz": "qux"}, "system": {"bundles": {"test-bundle": {"etag": "", "manifest": {"revision": "quickbrownfaux", "roots": [""]}}}}}`)) + expData := util.MustUnmarshalJSON([]byte(`{ + "foo": {"bar": 1, "baz": "qux"}, + "system": { + "bundles": {"test-bundle": {"etag": "", "manifest": {"revision": "quickbrownfaux", "roots": [""]}}}, + "modules": {"test-bundle/foo/bar.rego": {"rego_version": 1}} + } + }`)) if err != nil { fatal(err) } else if !reflect.DeepEqual(data, expData) { @@ -2650,12 +2726,27 @@ corge contains 1 if { } data, err := manager.Store.Read(ctx, txn, storage.Path{}) - regoVersionStr := "" + + manifestRegoVersionStr := "" + if tc.bundleRegoVersion != nil { + manifestRegoVersionStr = fmt.Sprintf(`, "rego_version": %d`, bundleRegoVersion(*tc.bundleRegoVersion)) + } + + var moduleRegoVersion int if tc.bundleRegoVersion != nil { - regoVersionStr = fmt.Sprintf(`, "rego_version": %d`, bundleRegoVersion(*tc.bundleRegoVersion)) + moduleRegoVersion = tc.bundleRegoVersion.Int() + } else { + moduleRegoVersion = ast.DefaultRegoVersion.Int() } - expData := util.MustUnmarshalJSON([]byte(fmt.Sprintf(`{"foo": {"bar": 1, "baz": "qux"}, "system": {"bundles": {"test-bundle": {"etag": "", "manifest": {"revision": "quickbrownfaux"%s, "roots": [""]}}}}}`, - regoVersionStr))) + + expData := util.MustUnmarshalJSON([]byte(fmt.Sprintf(`{ + "foo": {"bar": 1, "baz": "qux"}, + "system": { + "bundles": {"test-bundle": {"etag": "", "manifest": {"revision": "quickbrownfaux"%s, "roots": [""]}}}, + "modules": {"test-bundle/foo/bar.rego": {"rego_version": %d}} + } + }`, + manifestRegoVersionStr, moduleRegoVersion))) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(data, expData) { @@ -6435,7 +6526,13 @@ func TestPluginManualTriggerMultipleDiskStorage(t *testing.T) { } data, err := manager.Store.Read(ctx, txn, storage.Path{}) - expData := util.MustUnmarshalJSON([]byte(`{"p": "x1", "q": "x2", "system": {"bundles": {"test-1": {"etag": "", "manifest": {"revision": "", "roots": ["p", "authz"]}}, "test-2": {"etag": "", "manifest": {"revision": "", "roots": ["q"]}}}}}`)) + expData := util.MustUnmarshalJSON([]byte(`{ + "p": "x1", "q": "x2", + "system": { + "bundles": {"test-1": {"etag": "", "manifest": {"revision": "", "roots": ["p", "authz"]}}, "test-2": {"etag": "", "manifest": {"revision": "", "roots": ["q"]}}}, + "modules": {"test-1/bar/policy.rego": {"rego_version": 1}} + } + }`)) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(data, expData) {