From d6b57ef80325546e8228fc4fce154d2391723388 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Mon, 23 Sep 2024 13:13:50 +0200 Subject: [PATCH 01/17] Pass module_path via vm.Scope to enable the use of relative path --- internal/alloycli/cmd_run.go | 2 +- internal/runtime/alloy.go | 30 ++++++++++++++----- .../controller/component_references.go | 17 ++++------- .../controller/custom_component_registry.go | 10 +++++++ .../runtime/internal/controller/loader.go | 8 ++++- .../internal/controller/node_config_import.go | 21 ++++++++++++- .../internal/controller/value_cache.go | 9 +++++- .../internal/importsource/import_file.go | 5 ++++ .../internal/importsource/import_git.go | 9 ++++-- .../internal/importsource/import_http.go | 8 ++++- .../internal/importsource/import_source.go | 2 ++ .../internal/importsource/import_string.go | 7 +++++ internal/util/filepath.go | 13 ++++++++ syntax/vm/vm.go | 10 +++++-- 14 files changed, 124 insertions(+), 27 deletions(-) create mode 100644 internal/util/filepath.go diff --git a/internal/alloycli/cmd_run.go b/internal/alloycli/cmd_run.go index 9cbbbb27a7..f62aaec1fe 100644 --- a/internal/alloycli/cmd_run.go +++ b/internal/alloycli/cmd_run.go @@ -340,7 +340,7 @@ func (fr *alloyRun) Run(configPath string) error { if err != nil { return nil, fmt.Errorf("reading config path %q: %w", configPath, err) } - if err := f.LoadSource(alloySource, nil); err != nil { + if err := f.LoadSource(alloySource, nil, configPath); err != nil { return alloySource, fmt.Errorf("error during the initial load: %w", err) } diff --git a/internal/runtime/alloy.go b/internal/runtime/alloy.go index 4479ae13c3..1d226e41d8 100644 --- a/internal/runtime/alloy.go +++ b/internal/runtime/alloy.go @@ -61,6 +61,8 @@ import ( "github.com/grafana/alloy/internal/runtime/logging/level" "github.com/grafana/alloy/internal/runtime/tracing" "github.com/grafana/alloy/internal/service" + "github.com/grafana/alloy/internal/util" + "github.com/grafana/alloy/syntax/vm" ) // Options holds static options for an Alloy controller. @@ -296,22 +298,36 @@ func (f *Runtime) Run(ctx context.Context) { // The controller will only start running components after Load is called once // without any configuration errors. // LoadSource uses default loader configuration. -func (f *Runtime) LoadSource(source *Source, args map[string]any) error { - return f.loadSource(source, args, nil) +func (f *Runtime) LoadSource(source *Source, args map[string]any, configPath string) error { + return f.applyLoaderConfig(controller.ApplyOptions{ + Args: args, + ComponentBlocks: source.components, + ConfigBlocks: source.configBlocks, + DeclareBlocks: source.declareBlocks, + ArgScope: &vm.Scope{ + Parent: nil, + Variables: map[string]interface{}{ + "module_path": util.ExtractDirPath(configPath), + }, + }, + }) } // Same as above but with a customComponentRegistry that provides custom component definitions. func (f *Runtime) loadSource(source *Source, args map[string]any, customComponentRegistry *controller.CustomComponentRegistry) error { - f.loadMut.Lock() - defer f.loadMut.Unlock() - - applyOptions := controller.ApplyOptions{ + return f.applyLoaderConfig(controller.ApplyOptions{ Args: args, ComponentBlocks: source.components, ConfigBlocks: source.configBlocks, DeclareBlocks: source.declareBlocks, CustomComponentRegistry: customComponentRegistry, - } + ArgScope: customComponentRegistry.Scope(), + }) +} + +func (f *Runtime) applyLoaderConfig(applyOptions controller.ApplyOptions) error { + f.loadMut.Lock() + defer f.loadMut.Unlock() diags := f.loader.Apply(applyOptions) if !f.loadedOnce.Load() && diags.HasErrors() { diff --git a/internal/runtime/internal/controller/component_references.go b/internal/runtime/internal/controller/component_references.go index 8aee9a4915..cc5205dfdc 100644 --- a/internal/runtime/internal/controller/component_references.go +++ b/internal/runtime/internal/controller/component_references.go @@ -29,7 +29,7 @@ type Reference struct { // ComponentReferences returns the list of references a component is making to // other components. -func ComponentReferences(cn dag.Node, g *dag.Graph, l log.Logger) ([]Reference, diag.Diagnostics) { +func ComponentReferences(cn dag.Node, g *dag.Graph, l log.Logger, scope *vm.Scope) ([]Reference, diag.Diagnostics) { var ( traversals []Traversal @@ -48,25 +48,20 @@ func ComponentReferences(cn dag.Node, g *dag.Graph, l log.Logger) ([]Reference, ref, resolveDiags := resolveTraversal(t, g) componentRefMatch := !resolveDiags.HasErrors() - // We use an empty scope to determine if a reference refers to something in - // the stdlib, since vm.Scope.Lookup will search the scope tree + the - // stdlib. - // - // Any call to an stdlib function is ignored. - var emptyScope vm.Scope - _, stdlibMatch := emptyScope.Lookup(t[0].Name) + // we look for a match in the provided scope and the stdlib + _, scopeMatch := scope.Lookup(t[0].Name) - if !componentRefMatch && !stdlibMatch { + if !componentRefMatch && !scopeMatch { diags = append(diags, resolveDiags...) continue } if componentRefMatch { - if stdlibMatch { + if scope.IsStdlibIdentifiers(t[0].Name) { level.Warn(l).Log("msg", "a component is shadowing an existing stdlib name", "component", strings.Join(ref.Target.Block().Name, "."), "stdlib name", t[0].Name) } refs = append(refs, ref) - } else if stdlibMatch && emptyScope.IsDeprecated(t[0].Name) { + } else if scope.IsStdlibDeprecated(t[0].Name) { level.Warn(l).Log("msg", "this stdlib function is deprecated; please refer to the documentation for updated usage and alternatives", "function", t[0].Name) } } diff --git a/internal/runtime/internal/controller/custom_component_registry.go b/internal/runtime/internal/controller/custom_component_registry.go index 63f6e83557..1ec4053d28 100644 --- a/internal/runtime/internal/controller/custom_component_registry.go +++ b/internal/runtime/internal/controller/custom_component_registry.go @@ -5,6 +5,7 @@ import ( "sync" "github.com/grafana/alloy/syntax/ast" + "github.com/grafana/alloy/syntax/vm" ) // CustomComponentRegistry holds custom component definitions that are available in the context. @@ -14,6 +15,7 @@ type CustomComponentRegistry struct { parent *CustomComponentRegistry // nil if root config mut sync.RWMutex + scope *vm.Scope imports map[string]*CustomComponentRegistry // importNamespace: importScope declares map[string]ast.Body // customComponentName: template } @@ -42,6 +44,12 @@ func (s *CustomComponentRegistry) getImport(name string) (*CustomComponentRegist return im, ok } +func (s *CustomComponentRegistry) Scope() *vm.Scope { + s.mut.RLock() + defer s.mut.RUnlock() + return s.scope +} + // registerDeclare stores a local declare block. func (s *CustomComponentRegistry) registerDeclare(declare *ast.BlockStmt) { s.mut.Lock() @@ -71,6 +79,7 @@ func (s *CustomComponentRegistry) updateImportContent(importNode *ImportConfigNo } importScope := NewCustomComponentRegistry(nil) importScope.declares = importNode.ImportedDeclares() + importScope.scope = importNode.Scope() importScope.updateImportContentChildren(importNode) s.imports[importNode.label] = importScope } @@ -81,6 +90,7 @@ func (s *CustomComponentRegistry) updateImportContentChildren(importNode *Import for _, child := range importNode.ImportConfigNodesChildren() { childScope := NewCustomComponentRegistry(nil) childScope.declares = child.ImportedDeclares() + childScope.scope = child.Scope() childScope.updateImportContentChildren(child) s.imports[child.label] = childScope } diff --git a/internal/runtime/internal/controller/loader.go b/internal/runtime/internal/controller/loader.go index d10a8dd0a7..659ffdf35c 100644 --- a/internal/runtime/internal/controller/loader.go +++ b/internal/runtime/internal/controller/loader.go @@ -18,6 +18,7 @@ import ( "github.com/grafana/alloy/internal/service" "github.com/grafana/alloy/syntax/ast" "github.com/grafana/alloy/syntax/diag" + "github.com/grafana/alloy/syntax/vm" "github.com/grafana/dskit/backoff" "github.com/hashicorp/go-multierror" "go.opentelemetry.io/otel/attribute" @@ -124,6 +125,9 @@ type ApplyOptions struct { // The definition of a custom component instantiated inside of the loaded config // should be passed via this field if it's not declared or imported in the config. CustomComponentRegistry *CustomComponentRegistry + + // ArgScope contains additional variables that can be used in the current module. + ArgScope *vm.Scope } // Apply loads a new set of components into the Loader. Apply will drop any @@ -145,6 +149,8 @@ func (l *Loader) Apply(options ApplyOptions) diag.Diagnostics { l.cm.controllerEvaluation.Set(1) defer l.cm.controllerEvaluation.Set(0) + l.cache.SetScope(options.ArgScope) + for key, value := range options.Args { l.cache.CacheModuleArgument(key, value) } @@ -608,7 +614,7 @@ func (l *Loader) wireGraphEdges(g *dag.Graph) diag.Diagnostics { } // Finally, wire component references. - refs, nodeDiags := ComponentReferences(n, g, l.log) + refs, nodeDiags := ComponentReferences(n, g, l.log, l.cache.scope) for _, ref := range refs { g.AddEdge(dag.Edge{From: n, To: ref.Target}) } diff --git a/internal/runtime/internal/controller/node_config_import.go b/internal/runtime/internal/controller/node_config_import.go index f01795405d..785f65b059 100644 --- a/internal/runtime/internal/controller/node_config_import.go +++ b/internal/runtime/internal/controller/node_config_import.go @@ -289,6 +289,11 @@ func (cn *ImportConfigNode) processImportBlock(stmt *ast.BlockStmt, fullName str childGlobals.OnBlockNodeUpdate = cn.onChildrenContentUpdate // Children data paths are nested inside their parents to avoid collisions. childGlobals.DataPath = filepath.Join(childGlobals.DataPath, cn.globalID) + + if importsource.GetSourceType(cn.block.GetBlockName()) == importsource.HTTP && sourceType == importsource.File { + return fmt.Errorf("importing a module via import.http (nodeID: %s) that contains an import.file block is not supported", cn.nodeID) + } + cn.importConfigNodesChildren[stmt.Label] = NewImportConfigNode(stmt, childGlobals, sourceType) return nil } @@ -297,7 +302,12 @@ func (cn *ImportConfigNode) processImportBlock(stmt *ast.BlockStmt, fullName str func (cn *ImportConfigNode) evaluateChildren() error { for _, child := range cn.importConfigNodesChildren { err := child.Evaluate(&vm.Scope{ - Parent: nil, + Parent: &vm.Scope{ + Parent: nil, + Variables: map[string]interface{}{ + "module_path": cn.source.ModulePath(), + }, + }, Variables: make(map[string]interface{}), }) if err != nil { @@ -424,6 +434,15 @@ func (cn *ImportConfigNode) ImportedDeclares() map[string]ast.Body { return cn.importedDeclares } +// Scope returns the scope associated with the import source. +func (cn *ImportConfigNode) Scope() *vm.Scope { + return &vm.Scope{ + Variables: map[string]interface{}{ + "module_path": cn.source.ModulePath(), + }, + } +} + // ImportConfigNodesChildren returns the ImportConfigNodesChildren of this ImportConfigNode. func (cn *ImportConfigNode) ImportConfigNodesChildren() map[string]*ImportConfigNode { cn.mut.Lock() diff --git a/internal/runtime/internal/controller/value_cache.go b/internal/runtime/internal/controller/value_cache.go index fa2761ba81..3815a98a44 100644 --- a/internal/runtime/internal/controller/value_cache.go +++ b/internal/runtime/internal/controller/value_cache.go @@ -21,6 +21,7 @@ type valueCache struct { moduleArguments map[string]any // key -> module arguments value moduleExports map[string]any // name -> value for the value of module exports moduleChangedIndex int // Everytime a change occurs this is incremented + scope *vm.Scope // scope provides additional context for the nodes in the module } // newValueCache creates a new ValueCache. @@ -34,6 +35,12 @@ func newValueCache() *valueCache { } } +func (vc *valueCache) SetScope(scope *vm.Scope) { + vc.mut.Lock() + defer vc.mut.Unlock() + vc.scope = scope +} + // CacheArguments will cache the provided arguments by the given id. args may // be nil to store an empty object. func (vc *valueCache) CacheArguments(id ComponentID, args component.Arguments) { @@ -165,7 +172,7 @@ func (vc *valueCache) BuildContext() *vm.Scope { defer vc.mut.RUnlock() scope := &vm.Scope{ - Parent: nil, + Parent: vc.scope, Variables: make(map[string]interface{}), } diff --git a/internal/runtime/internal/importsource/import_file.go b/internal/runtime/internal/importsource/import_file.go index 811047bb48..0eef81eae1 100644 --- a/internal/runtime/internal/importsource/import_file.go +++ b/internal/runtime/internal/importsource/import_file.go @@ -16,6 +16,7 @@ import ( "github.com/grafana/alloy/internal/component" filedetector "github.com/grafana/alloy/internal/filedetector" "github.com/grafana/alloy/internal/runtime/logging/level" + "github.com/grafana/alloy/internal/util" "github.com/grafana/alloy/syntax/vm" ) @@ -254,3 +255,7 @@ func collectFilesFromDir(path string) ([]string, error) { func (im *ImportFile) SetEval(eval *vm.Evaluator) { im.eval = eval } + +func (im *ImportFile) ModulePath() string { + return util.ExtractDirPath(im.args.Filename) +} diff --git a/internal/runtime/internal/importsource/import_git.go b/internal/runtime/internal/importsource/import_git.go index f0b77965fe..02ee1ed675 100644 --- a/internal/runtime/internal/importsource/import_git.go +++ b/internal/runtime/internal/importsource/import_git.go @@ -29,6 +29,7 @@ type ImportGit struct { repo *vcs.GitRepo repoOpts vcs.GitRepoOptions args GitArguments + repoPath string onContentChange func(map[string]string) argsChanged chan struct{} @@ -197,7 +198,7 @@ func (im *ImportGit) Update(args component.Arguments) (err error) { // TODO(rfratto): store in a repo-specific directory so changing repositories // doesn't risk break the module loader if there's a SHA collision between // the two different repositories. - repoPath := filepath.Join(im.opts.DataPath, "repo") + im.repoPath = filepath.Join(im.opts.DataPath, "repo") repoOpts := vcs.GitRepoOptions{ Repository: newArgs.Repository, @@ -208,7 +209,7 @@ func (im *ImportGit) Update(args component.Arguments) (err error) { // Create or update the repo field. // Failure to update repository makes the module loader temporarily use cached contents on disk if im.repo == nil || !reflect.DeepEqual(repoOpts, im.repoOpts) { - r, err := vcs.NewGitRepo(context.Background(), repoPath, repoOpts) + r, err := vcs.NewGitRepo(context.Background(), im.repoPath, repoOpts) if err != nil { if errors.As(err, &vcs.UpdateFailedError{}) { level.Error(im.log).Log("msg", "failed to update repository", "err", err) @@ -303,3 +304,7 @@ func (im *ImportGit) CurrentHealth() component.Health { func (im *ImportGit) SetEval(eval *vm.Evaluator) { im.eval = eval } + +func (im *ImportGit) ModulePath() string { + return im.repoPath +} diff --git a/internal/runtime/internal/importsource/import_http.go b/internal/runtime/internal/importsource/import_http.go index 23bb896a1d..f8cbfd469b 100644 --- a/internal/runtime/internal/importsource/import_http.go +++ b/internal/runtime/internal/importsource/import_http.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "path" "reflect" "time" @@ -16,7 +17,7 @@ import ( // ImportHTTP imports a module from a HTTP server via the remote.http component. type ImportHTTP struct { managedRemoteHTTP *remote_http.Component - arguments component.Arguments + arguments HTTPArguments managedOpts component.Options eval *vm.Evaluator } @@ -106,3 +107,8 @@ func (im *ImportHTTP) CurrentHealth() component.Health { func (im *ImportHTTP) SetEval(eval *vm.Evaluator) { im.eval = eval } + +func (im *ImportHTTP) ModulePath() string { + dir, _ := path.Split(im.arguments.URL) + return dir +} diff --git a/internal/runtime/internal/importsource/import_source.go b/internal/runtime/internal/importsource/import_source.go index 79686d6735..5e2322f375 100644 --- a/internal/runtime/internal/importsource/import_source.go +++ b/internal/runtime/internal/importsource/import_source.go @@ -34,6 +34,8 @@ type ImportSource interface { CurrentHealth() component.Health // Update evaluator SetEval(eval *vm.Evaluator) + // ModulePath is the path where the module is stored locally. + ModulePath() string } // NewImportSource creates a new ImportSource depending on the type. diff --git a/internal/runtime/internal/importsource/import_string.go b/internal/runtime/internal/importsource/import_string.go index 91057f9994..7c638f46a0 100644 --- a/internal/runtime/internal/importsource/import_string.go +++ b/internal/runtime/internal/importsource/import_string.go @@ -15,6 +15,7 @@ type ImportString struct { arguments component.Arguments eval *vm.Evaluator onContentChange func(map[string]string) + modulePath string } var _ ImportSource = (*ImportString)(nil) @@ -41,6 +42,8 @@ func (im *ImportString) Evaluate(scope *vm.Scope) error { } im.arguments = arguments + im.modulePath, _ = scope.Variables["module_path"].(string) + // notifies that the content has changed im.onContentChange(map[string]string{"import_string": arguments.Content.Value}) @@ -63,3 +66,7 @@ func (im *ImportString) CurrentHealth() component.Health { func (im *ImportString) SetEval(eval *vm.Evaluator) { im.eval = eval } + +func (im *ImportString) ModulePath() string { + return im.modulePath +} diff --git a/internal/util/filepath.go b/internal/util/filepath.go new file mode 100644 index 0000000000..7a0dde4891 --- /dev/null +++ b/internal/util/filepath.go @@ -0,0 +1,13 @@ +package util + +import "path/filepath" + +// ExtractDirPath removes the file part of a path if it exists. +func ExtractDirPath(p string) string { + // If the base of the path has an extension, it's likely a file. + if filepath.Ext(filepath.Base(p)) != "" { + return filepath.Dir(p) + } + // Otherwise, return the path as is. + return p +} diff --git a/syntax/vm/vm.go b/syntax/vm/vm.go index 2df052b92b..9dfed95ac6 100644 --- a/syntax/vm/vm.go +++ b/syntax/vm/vm.go @@ -485,8 +485,14 @@ func (s *Scope) Lookup(name string) (interface{}, bool) { return nil, false } -// IsDeprecated returns true if the identifier exists and is deprecated. -func (s *Scope) IsDeprecated(name string) bool { +// IsStdlibIdentifiers returns true if the identifier exists. +func (s *Scope) IsStdlibIdentifiers(name string) bool { + _, exist := stdlib.Identifiers[name] + return exist +} + +// IsStdlibDeprecated returns true if the identifier exists and is deprecated. +func (s *Scope) IsStdlibDeprecated(name string) bool { _, exist := stdlib.DeprecatedIdentifiers[name] return exist } From 9dec04df5d373e83e5e9886cf62df034d458fc39 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Mon, 23 Sep 2024 13:15:07 +0200 Subject: [PATCH 02/17] update LoadSource refs --- internal/converter/internal/test_common/testing.go | 2 +- internal/runtime/alloy_services.go | 3 ++- internal/runtime/alloy_services_test.go | 14 +++++++------- internal/runtime/alloy_updates_test.go | 10 +++++----- internal/runtime/declare_test.go | 8 ++++---- internal/runtime/import_git_test.go | 10 +++++----- internal/runtime/module.go | 2 +- internal/runtime/module_eval_test.go | 10 +++++----- internal/runtime/module_fail_test.go | 2 +- internal/runtime/module_test.go | 6 +++--- internal/runtime/source_test.go | 4 ++-- internal/service/remotecfg/remotecfg_test.go | 2 +- 12 files changed, 37 insertions(+), 36 deletions(-) diff --git a/internal/converter/internal/test_common/testing.go b/internal/converter/internal/test_common/testing.go index 98b5377a45..a315f0362c 100644 --- a/internal/converter/internal/test_common/testing.go +++ b/internal/converter/internal/test_common/testing.go @@ -217,7 +217,7 @@ func attemptLoadingAlloyConfig(t *testing.T, bb []byte) { }, EnableCommunityComps: true, }) - err = f.LoadSource(cfg, nil) + err = f.LoadSource(cfg, nil, "") // Many components will fail to build as e.g. the cert files are missing, so we ignore these errors. // This is not ideal, but we still validate for other potential issues. diff --git a/internal/runtime/alloy_services.go b/internal/runtime/alloy_services.go index ae22785eef..bcded46bda 100644 --- a/internal/runtime/alloy_services.go +++ b/internal/runtime/alloy_services.go @@ -98,7 +98,8 @@ func (sc ServiceController) LoadSource(b []byte, args map[string]any) error { if err != nil { return err } - return sc.f.LoadSource(source, args) + // The config is loaded via a service, the config path is left empty. + return sc.f.LoadSource(source, args, "") } func (sc ServiceController) Ready() bool { return sc.f.Ready() } diff --git a/internal/runtime/alloy_services_test.go b/internal/runtime/alloy_services_test.go index 0e35261e09..14bc228afa 100644 --- a/internal/runtime/alloy_services_test.go +++ b/internal/runtime/alloy_services_test.go @@ -38,7 +38,7 @@ func TestServices(t *testing.T) { opts.Services = append(opts.Services, svc) ctrl := New(opts) - require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil)) + require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil, "")) // Start the controller. This should cause our service to run. go ctrl.Run(ctx) @@ -90,7 +90,7 @@ func TestServices_Configurable(t *testing.T) { ctrl := New(opts) - require.NoError(t, ctrl.LoadSource(f, nil)) + require.NoError(t, ctrl.LoadSource(f, nil, "")) // Start the controller. This should cause our service to run. go ctrl.Run(ctx) @@ -137,7 +137,7 @@ func TestServices_Configurable_Optional(t *testing.T) { ctrl := New(opts) - require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil)) + require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil, "")) // Start the controller. This should cause our service to run. go ctrl.Run(ctx) @@ -171,7 +171,7 @@ func TestAlloy_GetServiceConsumers(t *testing.T) { ctrl := New(opts) defer cleanUpController(ctrl) - require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil)) + require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil, "")) expectConsumers := []service.Consumer{{ Type: service.ConsumerTypeService, @@ -253,7 +253,7 @@ func TestComponents_Using_Services(t *testing.T) { ComponentRegistry: registry, ModuleRegistry: newModuleRegistry(), }) - require.NoError(t, ctrl.LoadSource(f, nil)) + require.NoError(t, ctrl.LoadSource(f, nil, "")) go ctrl.Run(ctx) require.NoError(t, componentBuilt.Wait(5*time.Second), "Component should have been built") @@ -332,7 +332,7 @@ func TestComponents_Using_Services_In_Modules(t *testing.T) { ComponentRegistry: registry, ModuleRegistry: newModuleRegistry(), }) - require.NoError(t, ctrl.LoadSource(f, nil)) + require.NoError(t, ctrl.LoadSource(f, nil, "")) go ctrl.Run(ctx) require.NoError(t, componentBuilt.Wait(5*time.Second), "Component should have been built") @@ -360,7 +360,7 @@ func TestNewControllerNoLeak(t *testing.T) { opts.Services = append(opts.Services, svc) ctrl := New(opts) - require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil)) + require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil, "")) // Start the controller. This should cause our service to run. go ctrl.Run(ctx) diff --git a/internal/runtime/alloy_updates_test.go b/internal/runtime/alloy_updates_test.go index 3bc4a631d3..cff70d898c 100644 --- a/internal/runtime/alloy_updates_test.go +++ b/internal/runtime/alloy_updates_test.go @@ -42,7 +42,7 @@ func TestController_Updates(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -122,7 +122,7 @@ func TestController_Updates_WithQueueFull(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -195,7 +195,7 @@ func TestController_Updates_WithLag(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -269,7 +269,7 @@ func TestController_Updates_WithOtherLaggingPipeline(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -338,7 +338,7 @@ func TestController_Updates_WithLaggingComponent(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) diff --git a/internal/runtime/declare_test.go b/internal/runtime/declare_test.go index d3c727a9eb..f8c58e5982 100644 --- a/internal/runtime/declare_test.go +++ b/internal/runtime/declare_test.go @@ -336,7 +336,7 @@ func TestDeclare(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -461,7 +461,7 @@ func TestDeclareError(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") if err == nil { t.Errorf("Expected error to match regex %q, but got: nil", tc.expectedError) } else if !tc.expectedError.MatchString(err.Error()) { @@ -545,7 +545,7 @@ func TestDeclareUpdateConfig(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -569,7 +569,7 @@ func TestDeclareUpdateConfig(t *testing.T) { require.NotNil(t, f) // Reload the controller with the new config. - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) require.Eventually(t, func() bool { diff --git a/internal/runtime/import_git_test.go b/internal/runtime/import_git_test.go index 6f1a922c6b..393e75faa1 100644 --- a/internal/runtime/import_git_test.go +++ b/internal/runtime/import_git_test.go @@ -56,7 +56,7 @@ testImport.add "cc" { defer verifyNoGoroutineLeaks(t) ctrl, f := setup(t, main) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -123,7 +123,7 @@ testImport.add "cc" { defer verifyNoGoroutineLeaks(t) ctrl, f := setup(t, main) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -206,7 +206,7 @@ testImport.add "cc" { defer verifyNoGoroutineLeaks(t) ctrl, f := setup(t, main) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -270,7 +270,7 @@ testImport.add "cc" { defer verifyNoGoroutineLeaks(t) ctrl, f := setup(t, main) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -355,7 +355,7 @@ testImport.add "cc" { defer verifyNoGoroutineLeaks(t) ctrl, f := setup(t, main) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") expectedErr := vcs.InvalidRevisionError{ Revision: "nonexistent", } diff --git a/internal/runtime/module.go b/internal/runtime/module.go index 2f2955d63b..3bce44e99d 100644 --- a/internal/runtime/module.go +++ b/internal/runtime/module.go @@ -160,7 +160,7 @@ func (c *module) LoadConfig(config []byte, args map[string]any) error { if err != nil { return err } - return c.f.LoadSource(ff, args) + return c.f.LoadSource(ff, args, "") } // LoadBody loads a pre-parsed Alloy config. diff --git a/internal/runtime/module_eval_test.go b/internal/runtime/module_eval_test.go index a75b18d174..3075f23aae 100644 --- a/internal/runtime/module_eval_test.go +++ b/internal/runtime/module_eval_test.go @@ -62,7 +62,7 @@ func TestUpdates_EmptyModule(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -123,7 +123,7 @@ func TestUpdates_ThroughModule(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -185,7 +185,7 @@ func TestUpdates_TwoModules_SameCompNames(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -252,7 +252,7 @@ func TestUpdates_ReloadConfig(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -307,7 +307,7 @@ func TestUpdates_ReloadConfig(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) require.Eventually(t, func() bool { diff --git a/internal/runtime/module_fail_test.go b/internal/runtime/module_fail_test.go index 8b792b26b0..042c674612 100644 --- a/internal/runtime/module_fail_test.go +++ b/internal/runtime/module_fail_test.go @@ -16,7 +16,7 @@ func TestIDRemovalIfFailedToLoad(t *testing.T) { fullContent := "test.fail.module \"t1\" { content = \"\" }" fl, err := ParseSource("test", []byte(fullContent)) require.NoError(t, err) - err = f.LoadSource(fl, nil) + err = f.LoadSource(fl, nil, "") require.NoError(t, err) ctx := context.Background() ctx, cnc := context.WithTimeout(ctx, 600*time.Second) diff --git a/internal/runtime/module_test.go b/internal/runtime/module_test.go index a869ca6189..a93edc95c6 100644 --- a/internal/runtime/module_test.go +++ b/internal/runtime/module_test.go @@ -156,7 +156,7 @@ func TestArgsNotInModules(t *testing.T) { defer cleanUpController(f) fl, err := ParseSource("test", []byte("argument \"arg\"{}")) require.NoError(t, err) - err = f.LoadSource(fl, nil) + err = f.LoadSource(fl, nil, "") require.ErrorContains(t, err, "argument blocks only allowed inside a module") } @@ -166,7 +166,7 @@ func TestExportsNotInModules(t *testing.T) { defer cleanUpController(f) fl, err := ParseSource("test", []byte("export \"arg\"{ value = 1}")) require.NoError(t, err) - err = f.LoadSource(fl, nil) + err = f.LoadSource(fl, nil, "") require.ErrorContains(t, err, "export blocks only allowed inside a module") } @@ -177,7 +177,7 @@ func TestExportsWhenNotUsed(t *testing.T) { fullContent := "test.module \"t1\" { content = \"" + content + "\" }" fl, err := ParseSource("test", []byte(fullContent)) require.NoError(t, err) - err = f.LoadSource(fl, nil) + err = f.LoadSource(fl, nil, "") require.NoError(t, err) ctx := context.Background() ctx, cnc := context.WithTimeout(ctx, 1*time.Second) diff --git a/internal/runtime/source_test.go b/internal/runtime/source_test.go index 0128b342fc..8d1e9a92bd 100644 --- a/internal/runtime/source_test.go +++ b/internal/runtime/source_test.go @@ -89,7 +89,7 @@ func TestParseSources_DuplicateComponent(t *testing.T) { require.NoError(t, err) ctrl := New(testOptions(t)) defer cleanUpController(ctrl) - err = ctrl.LoadSource(s, nil) + err = ctrl.LoadSource(s, nil, "") diagErrs, ok := err.(diag.Diagnostics) require.True(t, ok) require.Len(t, diagErrs, 2) @@ -120,7 +120,7 @@ func TestParseSources_UniqueComponent(t *testing.T) { require.NoError(t, err) ctrl := New(testOptions(t)) defer cleanUpController(ctrl) - err = ctrl.LoadSource(s, nil) + err = ctrl.LoadSource(s, nil, "") require.NoError(t, err) } diff --git a/internal/service/remotecfg/remotecfg_test.go b/internal/service/remotecfg/remotecfg_test.go index 171046c08c..93890dfc57 100644 --- a/internal/service/remotecfg/remotecfg_test.go +++ b/internal/service/remotecfg/remotecfg_test.go @@ -295,6 +295,6 @@ func (sc serviceController) LoadSource(b []byte, args map[string]any) error { if err != nil { return err } - return sc.f.LoadSource(source, args) + return sc.f.LoadSource(source, args, "") } func (sc serviceController) Ready() bool { return sc.f.Ready() } From bda5585acf15c7f6fb2f5cf64ad1201f6efd4fa6 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Mon, 23 Sep 2024 13:15:23 +0200 Subject: [PATCH 03/17] add tests --- internal/runtime/alloy_test.go | 37 +++++++++- internal/runtime/import_test.go | 69 +++++++++++++----- .../testdata/import_file/import_file_18.txtar | 50 +++++++++++++ .../testdata/import_file/import_file_19.txtar | 49 +++++++++++++ .../testdata/import_file/import_file_20.txtar | 50 +++++++++++++ .../import_file_folder_7.txtar | 58 +++++++++++++++ .../testdata/import_git/import_git_4.txtar | 21 ++++++ .../testdata/import_git/import_git_5.txtar | 21 ++++++ internal/runtime/testdata/repo.git.tar | Bin 68096 -> 87040 bytes 9 files changed, 337 insertions(+), 18 deletions(-) create mode 100644 internal/runtime/testdata/import_file/import_file_18.txtar create mode 100644 internal/runtime/testdata/import_file/import_file_19.txtar create mode 100644 internal/runtime/testdata/import_file/import_file_20.txtar create mode 100644 internal/runtime/testdata/import_file_folder/import_file_folder_7.txtar create mode 100644 internal/runtime/testdata/import_git/import_git_4.txtar create mode 100644 internal/runtime/testdata/import_git/import_git_5.txtar diff --git a/internal/runtime/alloy_test.go b/internal/runtime/alloy_test.go index a9c7efcf01..fb93e5853a 100644 --- a/internal/runtime/alloy_test.go +++ b/internal/runtime/alloy_test.go @@ -43,7 +43,7 @@ func TestController_LoadSource_Evaluation(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) require.Len(t, ctrl.loader.Components(), 4) @@ -54,6 +54,41 @@ func TestController_LoadSource_Evaluation(t *testing.T) { require.Equal(t, "hello, world!", out.(testcomponents.PassthroughExports).Output) } +var modulePathTestFile = ` + testcomponents.tick "ticker" { + frequency = "1s" + } + testcomponents.passthrough "static" { + input = module_path + } + testcomponents.passthrough "ticker" { + input = testcomponents.tick.ticker.tick_time + } + testcomponents.passthrough "forwarded" { + input = testcomponents.passthrough.ticker.output + } +` + +func TestController_LoadSource_WithModulePath_Evaluation(t *testing.T) { + defer verifyNoGoroutineLeaks(t) + ctrl := New(testOptions(t)) + defer cleanUpController(ctrl) + + f, err := ParseSource(t.Name(), []byte(modulePathTestFile)) + require.NoError(t, err) + require.NotNil(t, f) + + err = ctrl.LoadSource(f, nil, "path/to/config/main.alloy") + require.NoError(t, err) + require.Len(t, ctrl.loader.Components(), 4) + + // Check the inputs and outputs of things that should be immediately resolved + // without having to run the components. + in, out := getFields(t, ctrl.loader.Graph(), "testcomponents.passthrough.static") + require.Equal(t, "path/to/config", in.(testcomponents.PassthroughConfig).Input) + require.Equal(t, "path/to/config", out.(testcomponents.PassthroughExports).Output) +} + func getFields(t *testing.T, g *dag.Graph, nodeID string) (component.Arguments, component.Exports) { t.Helper() diff --git a/internal/runtime/import_test.go b/internal/runtime/import_test.go index dab9400d31..f6a1ab499b 100644 --- a/internal/runtime/import_test.go +++ b/internal/runtime/import_test.go @@ -27,13 +27,15 @@ const mainFile = "main.alloy" // The tests are using the .txtar files stored in the testdata folder. type testImportFile struct { - description string // description at the top of the txtar file - main string // root config that the controller should load - module string // module imported by the root config - nestedModule string // nested module that can be imported by the module - reloadConfig string // root config that the controller should apply on reload - otherNestedModule string // another nested module - update *updateFile // update can be used to update the content of a file at runtime + description string // description at the top of the txtar file + main string // root config that the controller should load + module string // module imported by the root config + nestedModule string // nested module that can be imported by the module + reloadConfig string // root config that the controller should apply on reload + otherNestedModule string // another nested module + nestedPathModule string // a module in a subdirectory + deeplyNestedPathModule string // a module in a sub-subdirectory + update *updateFile // update can be used to update the content of a file at runtime } type updateFile struct { @@ -70,6 +72,10 @@ func buildTestImportFile(t *testing.T, filename string) testImportFile { tc.reloadConfig = string(alloyConfig.Data) case "other_nested_module.alloy": tc.otherNestedModule = string(alloyConfig.Data) + case "nested_test/module.alloy": + tc.nestedPathModule = string(alloyConfig.Data) + case "nested_test/utils/module.alloy": + tc.deeplyNestedPathModule = string(alloyConfig.Data) } } return tc @@ -91,6 +97,18 @@ func TestImportFile(t *testing.T) { require.NoError(t, os.WriteFile("other_nested_module.alloy", []byte(tc.otherNestedModule), 0664)) } + if tc.nestedPathModule != "" || tc.deeplyNestedPathModule != "" { + require.NoError(t, os.Mkdir("nested_test", 0700)) + defer os.RemoveAll("nested_test") + if tc.nestedPathModule != "" { + require.NoError(t, os.WriteFile("nested_test/module.alloy", []byte(tc.nestedPathModule), 0664)) + } + if tc.deeplyNestedPathModule != "" { + require.NoError(t, os.Mkdir("nested_test/utils", 0700)) + require.NoError(t, os.WriteFile("nested_test/utils/module.alloy", []byte(tc.deeplyNestedPathModule), 0664)) + } + } + if tc.update != nil { testConfig(t, tc.main, tc.reloadConfig, func() { require.NoError(t, os.WriteFile(tc.update.name, []byte(tc.update.updateConfig), 0664)) @@ -117,6 +135,8 @@ func TestImportGit(t *testing.T) { // Extract repo.git.tar so tests can make use of it. // Make repo.git.tar with: // tar -C repo.git -cvf repo.git.tar . + // NOTE: when modifying the files in the repo, make sure to commit the files else + // the changes will not be taken into account. require.NoError(t, util.Untar("./testdata/repo.git.tar", "./testdata/repo.git")) require.NoError(t, util.Untar("./testdata/repo2.git.tar", "./testdata/repo2.git")) t.Cleanup(func() { @@ -146,13 +166,14 @@ func TestImportHTTP(t *testing.T) { } type testImportFileFolder struct { - description string // description at the top of the txtar file - main string // root config that the controller should load - module1 string // module imported by the root config - module2 string // another module imported by the root config - removed string // module will be removed in the dir on update - added string // module which will be added in the dir on update - update *updateFile // update can be used to update the content of a file at runtime + description string // description at the top of the txtar file + main string // root config that the controller should load + module1 string // module imported by the root config + module2 string // another module imported by the root config + utilsModule2 string // another module in a nested subdirectory + removed string // module will be removed in the dir on update + added string // module which will be added in the dir on update + update *updateFile // update can be used to update the content of a file at runtime } func buildTestImportFileFolder(t *testing.T, filename string) testImportFileFolder { @@ -168,6 +189,8 @@ func buildTestImportFileFolder(t *testing.T, filename string) testImportFileFold tc.module1 = string(alloyConfig.Data) case "module2.alloy": tc.module2 = string(alloyConfig.Data) + case "utils/module2.alloy": + tc.utilsModule2 = string(alloyConfig.Data) case "added.alloy": tc.added = string(alloyConfig.Data) case "removed.alloy": @@ -184,6 +207,12 @@ func buildTestImportFileFolder(t *testing.T, filename string) testImportFileFold name: "module2.alloy", updateConfig: string(alloyConfig.Data), } + case "utils/update_module2.alloy": + require.Nil(t, tc.update) + tc.update = &updateFile{ + name: "utils/module2.alloy", + updateConfig: string(alloyConfig.Data), + } } } return tc @@ -210,6 +239,12 @@ func TestImportFileFolder(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(dir, "removed.alloy"), []byte(tc.removed), 0700)) } + if tc.utilsModule2 != "" { + nestedDir := filepath.Join(dir, "utils") + require.NoError(t, os.Mkdir(nestedDir, 0700)) + require.NoError(t, os.WriteFile(filepath.Join(nestedDir, "module2.alloy"), []byte(tc.utilsModule2), 0700)) + } + // TODO: ideally we would like to check the health of the node but that's not yet possible for import nodes. // We should expect that adding or removing files in the dir is gracefully handled and the node should be // healthy once it polls the content of the dir again. @@ -265,7 +300,7 @@ func testConfig(t *testing.T, config string, reloadConfig string, update func()) defer verifyNoGoroutineLeaks(t) ctrl, f := setup(t, config) - err := ctrl.LoadSource(f, nil) + err := ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -303,7 +338,7 @@ func testConfig(t *testing.T, config string, reloadConfig string, update func()) require.NotNil(t, f) // Reload the controller with the new config. - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) // Export should be -10 after update @@ -317,7 +352,7 @@ func testConfig(t *testing.T, config string, reloadConfig string, update func()) func testConfigError(t *testing.T, config string, expectedError string) { defer verifyNoGoroutineLeaks(t) ctrl, f := setup(t, config) - err := ctrl.LoadSource(f, nil) + err := ctrl.LoadSource(f, nil, "") require.ErrorContains(t, err, expectedError) ctx, cancel := context.WithCancel(context.Background()) var wg sync.WaitGroup diff --git a/internal/runtime/testdata/import_file/import_file_18.txtar b/internal/runtime/testdata/import_file/import_file_18.txtar new file mode 100644 index 0000000000..36584f2f22 --- /dev/null +++ b/internal/runtime/testdata/import_file/import_file_18.txtar @@ -0,0 +1,50 @@ +Import nested passthrough module with relative import path. + +-- main.alloy -- +testcomponents.count "inc" { + frequency = "10ms" + max = 10 +} + +import.file "testImport" { + filename = "nested_test/module.alloy" +} + +testImport.a "cc" { + input = testcomponents.count.inc.count +} + +testcomponents.summation "sum" { + input = testImport.a.cc.output +} + +-- nested_test/module.alloy -- +import.file "testImport" { + filename = file.path_join(module_path, "utils/module.alloy") +} + +declare "a" { + argument "input" {} + + testImport.a "cc" { + input = argument.input.value + } + + export "output" { + value = testImport.a.cc.output + } +} + +-- nested_test/utils/module.alloy -- +declare "a" { + argument "input" {} + + testcomponents.passthrough "pt" { + input = argument.input.value + lag = "1ms" + } + + export "output" { + value = testcomponents.passthrough.pt.output + } +} diff --git a/internal/runtime/testdata/import_file/import_file_19.txtar b/internal/runtime/testdata/import_file/import_file_19.txtar new file mode 100644 index 0000000000..728855b84a --- /dev/null +++ b/internal/runtime/testdata/import_file/import_file_19.txtar @@ -0,0 +1,49 @@ +Import string with import file with relative import path. + +-- main.alloy -- +testcomponents.count "inc" { + frequency = "10ms" + max = 10 +} + +import.string "testImport" { + content = ` + import.file "testImport" { + filename = file.path_join(module_path, "nested_test/module.alloy") + } + + declare "a" { + argument "input" {} + + testImport.a "cc" { + input = argument.input.value + } + + export "output" { + value = testImport.a.cc.output + } + } + ` +} + +testImport.a "cc" { + input = testcomponents.count.inc.count +} + +testcomponents.summation "sum" { + input = testImport.a.cc.output +} + +-- nested_test/module.alloy -- +declare "a" { + argument "input" {} + + testcomponents.passthrough "pt" { + input = argument.input.value + lag = "1ms" + } + + export "output" { + value = testcomponents.passthrough.pt.output + } +} diff --git a/internal/runtime/testdata/import_file/import_file_20.txtar b/internal/runtime/testdata/import_file/import_file_20.txtar new file mode 100644 index 0000000000..7fd7f3b697 --- /dev/null +++ b/internal/runtime/testdata/import_file/import_file_20.txtar @@ -0,0 +1,50 @@ +Import nested passthrough module with relative import path in a declare. + +-- main.alloy -- +testcomponents.count "inc" { + frequency = "10ms" + max = 10 +} + +import.file "testImport" { + filename = "nested_test/module.alloy" +} + +testImport.a "cc" { + input = testcomponents.count.inc.count +} + +testcomponents.summation "sum" { + input = testImport.a.cc.output +} + +-- nested_test/module.alloy -- +declare "a" { + argument "input" {} + + import.file "testImport" { + filename = file.path_join(module_path, "utils/module.alloy") + } + + testImport.a "cc" { + input = argument.input.value + } + + export "output" { + value = testImport.a.cc.output + } +} + +-- nested_test/utils/module.alloy -- +declare "a" { + argument "input" {} + + testcomponents.passthrough "pt" { + input = argument.input.value + lag = "1ms" + } + + export "output" { + value = testcomponents.passthrough.pt.output + } +} diff --git a/internal/runtime/testdata/import_file_folder/import_file_folder_7.txtar b/internal/runtime/testdata/import_file_folder/import_file_folder_7.txtar new file mode 100644 index 0000000000..8c2ae7866b --- /dev/null +++ b/internal/runtime/testdata/import_file_folder/import_file_folder_7.txtar @@ -0,0 +1,58 @@ +Import nested folder with relative path. + +-- main.alloy -- +testcomponents.count "inc" { + frequency = "10ms" + max = 10 +} + +import.file "testImport" { + filename = "tmpTest" +} + +testImport.a "cc" { + input = testcomponents.count.inc.count +} + +testcomponents.summation "sum" { + input = testImport.a.cc.output +} + +-- module1.alloy -- +import.file "testImport" { + filename = file.path_join(module_path, "utils") +} +declare "a" { + argument "input" {} + + testImport.b "cc" { + input = argument.input.value + } + + export "output" { + value = testImport.b.cc.output + } +} + +-- utils/module2.alloy -- +declare "b" { + argument "input" {} + + testcomponents.passthrough "pt" { + input = argument.input.value + lag = "1ms" + } + + export "output" { + value = testcomponents.passthrough.pt.output + } +} + +-- utils/update_module2.alloy -- +declare "b" { + argument "input" {} + + export "output" { + value = -argument.input.value + } +} diff --git a/internal/runtime/testdata/import_git/import_git_4.txtar b/internal/runtime/testdata/import_git/import_git_4.txtar new file mode 100644 index 0000000000..f4f8feef9c --- /dev/null +++ b/internal/runtime/testdata/import_git/import_git_4.txtar @@ -0,0 +1,21 @@ +Import a module that contains an import.file with a relative import path. + +-- main.alloy -- +testcomponents.count "inc" { + frequency = "10ms" + max = 10 +} + +import.git "testImport" { + // Requires repo.git.tar to be extracted + repository = "./testdata/repo.git" + path = "module_import_file.alloy" +} + +testImport.a "cc" { + input = testcomponents.count.inc.count +} + +testcomponents.summation "sum" { + input = testImport.a.cc.output +} diff --git a/internal/runtime/testdata/import_git/import_git_5.txtar b/internal/runtime/testdata/import_git/import_git_5.txtar new file mode 100644 index 0000000000..c5cefee502 --- /dev/null +++ b/internal/runtime/testdata/import_git/import_git_5.txtar @@ -0,0 +1,21 @@ +Import a module that contains an import.file with a relative import path inside of a declare. + +-- main.alloy -- +testcomponents.count "inc" { + frequency = "10ms" + max = 10 +} + +import.git "testImport" { + // Requires repo.git.tar to be extracted + repository = "./testdata/repo.git" + path = "module_import_file_in_declare.alloy" +} + +testImport.a "cc" { + input = testcomponents.count.inc.count +} + +testcomponents.summation "sum" { + input = testImport.a.cc.output +} diff --git a/internal/runtime/testdata/repo.git.tar b/internal/runtime/testdata/repo.git.tar index 53147ced4e40f98d4d5e038079e42fd0f0ec06ed..156b4abb222a43dac175a2765547a756173357d6 100644 GIT binary patch delta 6268 zcmeHLX;c(f+U{Z#0oeou6lj#Nh(hhv9XA9Ql*JDfzlZ^>s_sHIL1`Qp5XB_{L`gS_ zTLK1&N@NHEiUJ~T5k%ZT2^hi8L8384R1{Qnsu$>LGMVw5nRDjP$DhmT_r6O#_j#9l zH)ZQ=I;xwMVNReh2ICOK2O$K;5FQ_s)#`~#0U#aV$}k*;hh*9sG8lvroW}>1db)tN zd2F00F-9C7855tR{{Nx(^ z@e*-@WS+vrc>V?hKKm#CeI*$y=1)`n>k&^%R)j~zMXq#Jd6fS!i9bG`Oh}NdSRI!b zu|fvu*l3M+lPxuvXr{2Cw(3F2vJDV##joPE@=*EX;}RwDiIPEoa-VC+m_)fluAT=? zdnk7*l%<9brTQ93Vig(Ly9D5DUpMp;`Qu;RcJ9=S4HB&K7?)k?%eo8vijA?@Em|QA9u% zDHX3wG*QGbj73<*)zKP!gis7o<6}9br{FjSqQsD%jiw$3;w%qUHpgnpa73OPs#xrX z*@O|bO-D^V4CLh0gjB$mJ$+1lo(e`_5T?$bPYi$*J*FqPz-gb!Uk6u!$1~9$H)E}Vl#Kf)R@mPC#F3#q{ z!Bu5k;V3OJX!oCzu1y(O1p{UYPuP^w}}I34b)qA?%;yk1L&dvIh8yFO-6V z_LRF>I^vzB!VP zi;azxe99+9BEB*|`G1L&tl*O?`O0pfXqC#}R*PduN#q(aU*0DTKGG-57m6_vdpH6H z#~2v471Oi;$6=a4DOyA^ID`ra5{D54l1-`uQ!oS-u(QOl@IIn+7-;(Wb2ohOpCp74 zCnG>MAWsLF0EIBbc2P~h7*+1d92bR1>$xcdvAL-(=vRkwQ_(;ONZyoGP2!rop#SlR z34&mATSevnKVtbfqwEg<%_F|0dBmuEo>d*uZDVaMp;%@HuID5(%K~}PQJwT%V?btd zCrMMzsqU;?i^4c2aQUpqnot*c#TJ;eMZqT=#MDEFd<#nbgDspuH0j9ZrJ9aEn*adB z(N@HO5%Lsu*;~rc7$E-~w$^cOJP$?yDPGyrZ&N|BgX$Lm}U<4%8 zE&PFI`$J$@-Tpd_wcSLRCnaX5BL)+SFihTkgeV4)AbVk?MM8)apokEpXaZCs#w4Ud)d|yP zgTJMgE*ZFDwtMf!H|t)dg>L*JvBvQC(Q})_vOfF#b7@d^`2KHz@)K}uY+Ko^;>g;m zyByx$TMq4Wt@806S?JqB65CFf+CbW@=0$keD3)&C3Rnow zj~ZN6tkKLf1!)ivR?m{1`ap_qXqUxLhCa9Eb!m_HsM_*tbOXP6c(ckit->PDRNGu`zJy>{NT z(Z2b_bdl?vsEE2v3#B)4`=S?Z?~)o9_a}ZIdv`&x!-B_Gb2c3DHVi0fMsn<|jz8lE zy~yV|ww&!QnebuVh5C1orWI!<^WMe=>{h0kxrrMp@)U|uMhszMh=EXwByjB6~aC<%dbQczxDJ|bQIVTBR&UW|Bj>BwQ{E+?znvN-r!s~J|; z)_Fvd|Dfjg(N-Ve>66n6tNA4r6;AoCHRv?$D6zP;^<-pP*zMGH_A_3cuBG6hwcEVS z7cU5lS~V}qV6ok6qwWOif5+wSD|guV>Rm%>a9G{gHh-f{UO=7y_7`KIFYc?wCAWCS=3u@kS5peZqdQ?!60Q8A6cB8Gqv zN(>Q$*@AN(K;$ireg8nzw0HmjDFM6ujL(BcvzETLSh9qF=zr3%5giu05A4TXXB3Ro zI&}YQr-2*Czui~TUJ-n9TXd+yr{)IJ03W?}gIhlBa|4`tvdfpU{&sBQ_NK-+=NuLm ztj&Y&W&F_C`O{OICY@8yJB=^gm|BF+jjcR-wC=pk-V3XYwwx+)@HkL9Z`|$^4r?YN zO}g*9daU-EZisrkFDp7BOy|9~Wn|5k`~AG-`1bxEs7Lod*^fVU`&@szOOD-blc$}_ z_poH~2c*Af)MN~S)c4QF8rKfC(sHgOJ(~3zV_!UmhM8VtT{o2SWC&aUiE&K8-VqrL zB*g@4TnGwKgSjK@5X5}b?V$28 zd5_44sw=%5>Yt}1eYj!Wey%5}b;Z~ur&V|LHjg%z6lr~7vgKWR_u<(UURh1;_F(L2m#Zd>soD{G;urmjHI6}h5{^H8 z?*W;OiZ9sX-4lg6lx5Ghytrup*7B*yv*5FO_gwb3G*8-Udu{6n%R||Fr3n2_%RN7I zAq#LIAZ4?Wf;|xoCn*6&3SpE&QAP++0tkT^F+&JJ5@ARLp{c=abhO5i;nf)i)$_a( zjB5txLDQ|%Uc2o%XyKIFQM<*YV^MajZJY0VU*cQe_Me>wN`#FKjYMQwbVpM4(-1dQ zZ={h;lDn^o&zQ_I9si{cZgq69k+{n6qQm?ng1XJ!vp?&89{XyN?MUL8Z}NbozB@BI ze@~*-&T~_ZwR>wX{dGoBt;sEicyL6w!=*$@eTKRUYNTXa z_TNA7nhE(WL_k~%wE5lpV&yHJskf|)E#v#ZE5-5dMzsYSZ)OPY*$4hqy(;vU#JxlR zUdE~&KJmvK0uP*Uv-^14S3R)%+<0Kh&$hnKxmh!-s`Kq@L#tCK79Ksd+aJHd#ed0A@_?=;6_Q;es7hmSp-z2wWlr+e&7mwEe7KeWo;t>R14vdrLm|IIHp zx|dtUyy=Y@NS~4XzDJtU--##Zb!;~5eRbUN+><1<<7Ixy<)c@tF9*Jl$c&nnyZiO6 z?&OBh`d!msH7qV(WO%2wripm&x%3VToi8ALuL&J0e>zot+6T?*1hezEN_Gm+20rNh E9rj>AO#lD@ delta 1761 zcmZvd-%Aux6vy}M?r7?&?Na$8s!6LYEcf0!_s)leKob$cMu}8|- zt@!k03S$H5gb@{CjZh<^GHXD|+5jfVjOkeSS#kwfRq=6&6GSu;QBf&YftVFd+;BKX z{Ez2mj0L>bjFlK>4x{`>e(tpuU{$V}kaA;246e(~Z2>GlYn+VA;f!fG2oKAqE)%lW zmCAbUGDI~$Zeuu(Ax3@H8fLoYj7j9&T$Yn)LC$UAI9@LpuiNJK(Pm_5OvO}pM^zdJ z!&;DkM>RCXhtpF}+S(T0d0&V`Qc;-iu)ef+H>9T{8)on4hQ(K~PHR_YXM+>hH}}$w z<6prBt*Oi*mmRj|T*kkGow9ypjxRGc&W|%tu7jc(E zh9ZRcs%^-H>mi*=I@4fpogR$Ki3w(2%QoCLD`kcZsmZ(HqRxzcLHEj-CN=pm%n*^{ z9*62-!*JFnntW;|Wi41|0qfracKXfnL^INnR%RJS^xui7WZ{^76Q+aA1$c zussaq91?NhK8(ZvN*Me>`Z~Q5YQjo7^tTQWcOBf}FQQG~^U@RTrTWO4}0; zgdcE!dqt>$x>%vU1>Anb#UxBa-kUB$D})xzp%mUGa{H0gznMkIK%G*k-(AWrX2q*h j?-3NC8z5GQ?g#f@h?hg7&P^8Ld&>$CzIPuUgyo8VetxR- From 9d1727f8f9fcc1cb991ea72fb54672c698ac7f74 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Mon, 23 Sep 2024 13:15:34 +0200 Subject: [PATCH 04/17] add changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb5aacf64b..877231bb53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ Main (unreleased) - Add support to `loki.source.syslog` for the RFC3164 format ("BSD syslog"). (@sushain97) - Add support to `loki.source.api` to be able to extract the tenant from the HTTP `X-Scope-OrgID` header (@QuentinBisson) -- + - (_Experimental_) Add a `loki.secretfilter` component to redact secrets from collected logs. ### Enhancements @@ -36,6 +36,8 @@ Main (unreleased) - SNMP exporter now supports labels in both `target` and `targets` parameters. (@mattdurham) +- Add support for relative paths to `import.file`. This new functionality allows users to use `import.file` blocks in modules + imported via `import.git` and other `import.file`. (@wildum) ### Bugfixes From 68559eb9bc5dd43e2cbe16d6b5aa76d78e347a13 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Mon, 23 Sep 2024 15:10:03 +0200 Subject: [PATCH 05/17] add docs --- .../reference/config-blocks/import.file.md | 126 +++++++++++++++++- 1 file changed, 124 insertions(+), 2 deletions(-) diff --git a/docs/sources/reference/config-blocks/import.file.md b/docs/sources/reference/config-blocks/import.file.md index 09046a0a43..dec94760d7 100644 --- a/docs/sources/reference/config-blocks/import.file.md +++ b/docs/sources/reference/config-blocks/import.file.md @@ -17,6 +17,9 @@ Imported directories are treated as single modules to support composability. That means that you can define a custom component in one file and use it in another custom component in another file in the same directory. +The keyword `module_path` can be used in combination with the stdlib function [file.path_join][] to import a module relative to the current module's path. +It works for modules that are imported via `import.file`, `import.git` and `import.string`. + ## Usage ```alloy @@ -41,6 +44,21 @@ The following arguments are supported: This example imports a module from a file and instantiates a custom component from the import that adds two numbers: +{{< collapse title="main.alloy" >}} + +```alloy +import.file "math" { + filename = "module.alloy" +} + +math.add "default" { + a = 15 + b = 45 +} +``` + +{{< /collapse >}} + {{< collapse title="module.alloy" >}} ```alloy @@ -56,11 +74,71 @@ declare "add" { {{< /collapse >}} -{{< collapse title="importer.alloy" >}} + +This example imports a module from a file inside of a module that is imported via [import.git][]: + +{{< collapse title="main.alloy" >}} + +```alloy +import.git "math" { + repository = "https://github.com/wildum/module.git" + path = "relative_math.alloy" +} + +math.add "default" { + a = 15 + b = 45 +} +``` + +{{< /collapse >}} + +{{< collapse title="relative_math.alloy" >}} + +```alloy +import.file "lib" { + filename = file.path_join(module_path, "lib.alloy") +} + +declare "add" { + argument "a" {} + argument "b" {} + + lib.plus "default" { + a = argument.a.value + b = argument.b.value + } + + export "output" { + value = lib.plus.default.sum + } +} +``` + +{{< /collapse >}} + +{{< collapse title="lib.alloy" >}} + +```alloy +declare "plus" { + argument "a" {} + argument "b" {} + + export "sum" { + value = argument.a.value + argument.b.value + } +} +``` + +{{< /collapse >}} + +This example imports a module from a file inside of a module that is imported via another `import.file`: + +{{< collapse title="main.alloy" >}} ```alloy import.file "math" { - filename = "module.alloy" + filename = "path/to/module/relative_math.alloy" } math.add "default" { @@ -70,3 +148,47 @@ math.add "default" { ``` {{< /collapse >}} + +{{< collapse title="relative_math.alloy" >}} + +```alloy +import.file "lib" { + filename = file.path_join(module_path, "lib.alloy") +} + +declare "add" { + argument "a" {} + argument "b" {} + + lib.plus "default" { + a = argument.a.value + b = argument.b.value + } + + export "output" { + value = lib.plus.default.sum + } +} +``` + +{{< /collapse >}} + +{{< collapse title="lib.alloy" >}} + +```alloy +declare "plus" { + argument "a" {} + argument "b" {} + + export "sum" { + value = argument.a.value + argument.b.value + } +} +``` + +{{< /collapse >}} + + + +[file.path_join]: ../../stdlib/file/ +[import.git]: ../import.git/ \ No newline at end of file From 0d482b91fe2c6873a9b8ad514cf34191e4a75424 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Tue, 24 Sep 2024 09:40:47 +0200 Subject: [PATCH 06/17] Update docs/sources/reference/config-blocks/import.file.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- docs/sources/reference/config-blocks/import.file.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sources/reference/config-blocks/import.file.md b/docs/sources/reference/config-blocks/import.file.md index dec94760d7..ce36fd2e06 100644 --- a/docs/sources/reference/config-blocks/import.file.md +++ b/docs/sources/reference/config-blocks/import.file.md @@ -17,8 +17,8 @@ Imported directories are treated as single modules to support composability. That means that you can define a custom component in one file and use it in another custom component in another file in the same directory. -The keyword `module_path` can be used in combination with the stdlib function [file.path_join][] to import a module relative to the current module's path. -It works for modules that are imported via `import.file`, `import.git` and `import.string`. +You can use the keyword `module_path` in combination with the `stdlib` function [file.path_join][] to import a module relative to the current module's path. +The `module_path` keyword works for modules that are imported via `import.file`, `import.git` and `import.string`. ## Usage From cc1c79fe75ed0ad1236a1cbdb311538cbb973cf8 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Mon, 30 Sep 2024 11:31:28 +0200 Subject: [PATCH 07/17] add module_path const --- internal/runtime/alloy.go | 3 ++- internal/runtime/internal/controller/node_config_import.go | 4 ++-- internal/runtime/internal/importsource/import_source.go | 2 ++ internal/runtime/internal/importsource/import_string.go | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/internal/runtime/alloy.go b/internal/runtime/alloy.go index 1d226e41d8..27fcf1e95b 100644 --- a/internal/runtime/alloy.go +++ b/internal/runtime/alloy.go @@ -56,6 +56,7 @@ import ( "github.com/grafana/alloy/internal/featuregate" "github.com/grafana/alloy/internal/runtime/internal/controller" + "github.com/grafana/alloy/internal/runtime/internal/importsource" "github.com/grafana/alloy/internal/runtime/internal/worker" "github.com/grafana/alloy/internal/runtime/logging" "github.com/grafana/alloy/internal/runtime/logging/level" @@ -307,7 +308,7 @@ func (f *Runtime) LoadSource(source *Source, args map[string]any, configPath str ArgScope: &vm.Scope{ Parent: nil, Variables: map[string]interface{}{ - "module_path": util.ExtractDirPath(configPath), + importsource.ModulePath: util.ExtractDirPath(configPath), }, }, }) diff --git a/internal/runtime/internal/controller/node_config_import.go b/internal/runtime/internal/controller/node_config_import.go index 785f65b059..3b7a7586a8 100644 --- a/internal/runtime/internal/controller/node_config_import.go +++ b/internal/runtime/internal/controller/node_config_import.go @@ -305,7 +305,7 @@ func (cn *ImportConfigNode) evaluateChildren() error { Parent: &vm.Scope{ Parent: nil, Variables: map[string]interface{}{ - "module_path": cn.source.ModulePath(), + importsource.ModulePath: cn.source.ModulePath(), }, }, Variables: make(map[string]interface{}), @@ -438,7 +438,7 @@ func (cn *ImportConfigNode) ImportedDeclares() map[string]ast.Body { func (cn *ImportConfigNode) Scope() *vm.Scope { return &vm.Scope{ Variables: map[string]interface{}{ - "module_path": cn.source.ModulePath(), + importsource.ModulePath: cn.source.ModulePath(), }, } } diff --git a/internal/runtime/internal/importsource/import_source.go b/internal/runtime/internal/importsource/import_source.go index 5e2322f375..ce3a369b98 100644 --- a/internal/runtime/internal/importsource/import_source.go +++ b/internal/runtime/internal/importsource/import_source.go @@ -24,6 +24,8 @@ const ( BlockImportGit = "import.git" ) +const ModulePath = "module_path" + // ImportSource retrieves a module from a source. type ImportSource interface { // Evaluate updates the arguments provided via the Alloy block. diff --git a/internal/runtime/internal/importsource/import_string.go b/internal/runtime/internal/importsource/import_string.go index 7c638f46a0..a8a1249fc4 100644 --- a/internal/runtime/internal/importsource/import_string.go +++ b/internal/runtime/internal/importsource/import_string.go @@ -42,7 +42,7 @@ func (im *ImportString) Evaluate(scope *vm.Scope) error { } im.arguments = arguments - im.modulePath, _ = scope.Variables["module_path"].(string) + im.modulePath, _ = scope.Variables[ModulePath].(string) // notifies that the content has changed im.onContentChange(map[string]string{"import_string": arguments.Content.Value}) From cfe7713e14d3bcee8c775dc3750b5600aec51fb9 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Mon, 30 Sep 2024 12:52:13 +0200 Subject: [PATCH 08/17] improve import git doc --- docs/sources/reference/config-blocks/import.git.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/sources/reference/config-blocks/import.git.md b/docs/sources/reference/config-blocks/import.git.md index 81ba649469..018ba90820 100644 --- a/docs/sources/reference/config-blocks/import.git.md +++ b/docs/sources/reference/config-blocks/import.git.md @@ -9,6 +9,9 @@ title: import.git The `import.git` block imports custom components from a Git repository and exposes them to the importer. `import.git` blocks must be given a label that determines the namespace where custom components are exposed. +The entire repository is cloned, and the module path is accessible via the `module_path` keyword. +This enables your module to import other modules within the repository by setting relative paths in the [import.file][] blocks. + ## Usage ```alloy @@ -101,5 +104,6 @@ math.add "default" { } ``` +[import.file]: ../import.file/ [basic_auth]: #basic_auth-block [ssh_key]: #ssh_key-block From 0a2cd6271f89951cf56adc8805e10f1b22c1dfd1 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Tue, 1 Oct 2024 14:42:32 +0200 Subject: [PATCH 09/17] use os.Stat to extract the dir path --- internal/runtime/alloy.go | 6 ++- internal/runtime/alloy_test.go | 38 +++++++++++++++++-- .../internal/importsource/import_file.go | 7 +++- internal/util/filepath.go | 22 +++++++---- 4 files changed, 61 insertions(+), 12 deletions(-) diff --git a/internal/runtime/alloy.go b/internal/runtime/alloy.go index 27fcf1e95b..6a21d9575c 100644 --- a/internal/runtime/alloy.go +++ b/internal/runtime/alloy.go @@ -300,6 +300,10 @@ func (f *Runtime) Run(ctx context.Context) { // without any configuration errors. // LoadSource uses default loader configuration. func (f *Runtime) LoadSource(source *Source, args map[string]any, configPath string) error { + modulePath, err := util.ExtractDirPath(configPath) + if err != nil { + level.Warn(f.log).Log("msg", "failed to extract directory path from configPath", "configPath", configPath, "err", err) + } return f.applyLoaderConfig(controller.ApplyOptions{ Args: args, ComponentBlocks: source.components, @@ -308,7 +312,7 @@ func (f *Runtime) LoadSource(source *Source, args map[string]any, configPath str ArgScope: &vm.Scope{ Parent: nil, Variables: map[string]interface{}{ - importsource.ModulePath: util.ExtractDirPath(configPath), + importsource.ModulePath: modulePath, }, }, }) diff --git a/internal/runtime/alloy_test.go b/internal/runtime/alloy_test.go index fb93e5853a..ebce0b656c 100644 --- a/internal/runtime/alloy_test.go +++ b/internal/runtime/alloy_test.go @@ -78,15 +78,47 @@ func TestController_LoadSource_WithModulePath_Evaluation(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil, "path/to/config/main.alloy") + filePath := "tmp_modulePath_test/test/main.alloy" + require.NoError(t, os.Mkdir("tmp_modulePath_test", 0700)) + require.NoError(t, os.Mkdir("tmp_modulePath_test/test", 0700)) + defer os.RemoveAll("tmp_modulePath_test") + require.NoError(t, os.WriteFile(filePath, []byte(""), 0664)) + + err = ctrl.LoadSource(f, nil, filePath) + require.NoError(t, err) + require.Len(t, ctrl.loader.Components(), 4) + + // Check the inputs and outputs of things that should be immediately resolved + // without having to run the components. + in, out := getFields(t, ctrl.loader.Graph(), "testcomponents.passthrough.static") + require.Equal(t, "tmp_modulePath_test/test", in.(testcomponents.PassthroughConfig).Input) + require.Equal(t, "tmp_modulePath_test/test", out.(testcomponents.PassthroughExports).Output) +} + +func TestController_LoadSource_WithModulePathWithoutFileExtension_Evaluation(t *testing.T) { + defer verifyNoGoroutineLeaks(t) + ctrl := New(testOptions(t)) + defer cleanUpController(ctrl) + + f, err := ParseSource(t.Name(), []byte(modulePathTestFile)) + require.NoError(t, err) + require.NotNil(t, f) + + filePath := "tmp_modulePath_test/test/main" + require.NoError(t, os.Mkdir("tmp_modulePath_test", 0700)) + require.NoError(t, os.Mkdir("tmp_modulePath_test/test", 0700)) + defer os.RemoveAll("tmp_modulePath_test") + require.NoError(t, os.WriteFile(filePath, []byte(""), 0664)) + + err = ctrl.LoadSource(f, nil, filePath) require.NoError(t, err) require.Len(t, ctrl.loader.Components(), 4) // Check the inputs and outputs of things that should be immediately resolved // without having to run the components. in, out := getFields(t, ctrl.loader.Graph(), "testcomponents.passthrough.static") - require.Equal(t, "path/to/config", in.(testcomponents.PassthroughConfig).Input) - require.Equal(t, "path/to/config", out.(testcomponents.PassthroughExports).Output) + require.Equal(t, "tmp_modulePath_test/test", in.(testcomponents.PassthroughConfig).Input) + require.Equal(t, "tmp_modulePath_test/test", out.(testcomponents.PassthroughExports).Output) } func getFields(t *testing.T, g *dag.Graph, nodeID string) (component.Arguments, component.Exports) { diff --git a/internal/runtime/internal/importsource/import_file.go b/internal/runtime/internal/importsource/import_file.go index 0eef81eae1..e4691d9ed5 100644 --- a/internal/runtime/internal/importsource/import_file.go +++ b/internal/runtime/internal/importsource/import_file.go @@ -257,5 +257,10 @@ func (im *ImportFile) SetEval(eval *vm.Evaluator) { } func (im *ImportFile) ModulePath() string { - return util.ExtractDirPath(im.args.Filename) + path, err := util.ExtractDirPath(im.args.Filename) + + if err != nil { + level.Error(im.managedOpts.Logger).Log("msg", "failed to extract module path", "module path", im.args.Filename, "err", err) + } + return path } diff --git a/internal/util/filepath.go b/internal/util/filepath.go index 7a0dde4891..b158e2cebc 100644 --- a/internal/util/filepath.go +++ b/internal/util/filepath.go @@ -1,13 +1,21 @@ package util -import "path/filepath" +import ( + "os" + "path/filepath" +) // ExtractDirPath removes the file part of a path if it exists. -func ExtractDirPath(p string) string { - // If the base of the path has an extension, it's likely a file. - if filepath.Ext(filepath.Base(p)) != "" { - return filepath.Dir(p) +func ExtractDirPath(p string) (string, error) { + info, err := os.Stat(p) + + if err != nil { + return "", err + } + + if !info.IsDir() { + return filepath.Dir(p), nil } - // Otherwise, return the path as is. - return p + + return p, nil } From 65d3cceeedc7eccf42b0cafd81f7018471dadb2b Mon Sep 17 00:00:00 2001 From: William Dumont Date: Tue, 1 Oct 2024 17:21:54 +0200 Subject: [PATCH 10/17] fix relative path with declare --- internal/runtime/declare_test.go | 36 +++++++++++++++++++ .../controller/custom_component_registry.go | 9 +++-- .../runtime/internal/controller/loader.go | 2 +- 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/internal/runtime/declare_test.go b/internal/runtime/declare_test.go index f8c58e5982..75d47b39b1 100644 --- a/internal/runtime/declare_test.go +++ b/internal/runtime/declare_test.go @@ -358,6 +358,42 @@ func TestDeclare(t *testing.T) { } } +func TestDeclareModulePath(t *testing.T) { + config := ` + declare "mod" { + export "output" { + value = module_path + } + } + + mod "myModule" {} + + testcomponents.passthrough "pass" { + input = mod.myModule.output + } + ` + ctrl := runtime.New(testOptions(t)) + f, err := runtime.ParseSource(t.Name(), []byte(config)) + require.NoError(t, err) + require.NotNil(t, f) + + err = ctrl.LoadSource(f, nil, "") + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + ctrl.Run(ctx) + close(done) + }() + defer func() { + cancel() + <-done + }() + passthrough := getExport[testcomponents.PassthroughExports](t, ctrl, "", "testcomponents.passthrough.pass") + require.Equal(t, passthrough.Output, "") +} + type errorTestCase struct { name string config string diff --git a/internal/runtime/internal/controller/custom_component_registry.go b/internal/runtime/internal/controller/custom_component_registry.go index 1ec4053d28..2f64423a3e 100644 --- a/internal/runtime/internal/controller/custom_component_registry.go +++ b/internal/runtime/internal/controller/custom_component_registry.go @@ -22,9 +22,10 @@ type CustomComponentRegistry struct { // NewCustomComponentRegistry creates a new CustomComponentRegistry with a parent. // parent can be nil. -func NewCustomComponentRegistry(parent *CustomComponentRegistry) *CustomComponentRegistry { +func NewCustomComponentRegistry(parent *CustomComponentRegistry, scope *vm.Scope) *CustomComponentRegistry { return &CustomComponentRegistry{ parent: parent, + scope: scope, declares: make(map[string]ast.Body), imports: make(map[string]*CustomComponentRegistry), } @@ -77,9 +78,8 @@ func (s *CustomComponentRegistry) updateImportContent(importNode *ImportConfigNo if _, exist := s.imports[importNode.label]; !exist { panic(fmt.Errorf("import %q was not registered", importNode.label)) } - importScope := NewCustomComponentRegistry(nil) + importScope := NewCustomComponentRegistry(nil, importNode.Scope()) importScope.declares = importNode.ImportedDeclares() - importScope.scope = importNode.Scope() importScope.updateImportContentChildren(importNode) s.imports[importNode.label] = importScope } @@ -88,9 +88,8 @@ func (s *CustomComponentRegistry) updateImportContent(importNode *ImportConfigNo // and update their scope with the imported declare blocks. func (s *CustomComponentRegistry) updateImportContentChildren(importNode *ImportConfigNode) { for _, child := range importNode.ImportConfigNodesChildren() { - childScope := NewCustomComponentRegistry(nil) + childScope := NewCustomComponentRegistry(nil, child.Scope()) childScope.declares = child.ImportedDeclares() - childScope.scope = child.Scope() childScope.updateImportContentChildren(child) s.imports[child.label] = childScope } diff --git a/internal/runtime/internal/controller/loader.go b/internal/runtime/internal/controller/loader.go index 659ffdf35c..48192299f9 100644 --- a/internal/runtime/internal/controller/loader.go +++ b/internal/runtime/internal/controller/loader.go @@ -158,7 +158,7 @@ func (l *Loader) Apply(options ApplyOptions) diag.Diagnostics { // Create a new CustomComponentRegistry based on the provided one. // The provided one should be nil for the root config. - l.componentNodeManager.setCustomComponentRegistry(NewCustomComponentRegistry(options.CustomComponentRegistry)) + l.componentNodeManager.setCustomComponentRegistry(NewCustomComponentRegistry(options.CustomComponentRegistry, options.ArgScope)) newGraph, diags := l.loadNewGraph(options.Args, options.ComponentBlocks, options.ConfigBlocks, options.DeclareBlocks) if diags.HasErrors() { return diags From 6357037c6bb77e116fb0697b436bf35302379fd0 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Tue, 1 Oct 2024 17:23:52 +0200 Subject: [PATCH 11/17] add support for remotecfg --- internal/alloycli/cmd_run.go | 1 + internal/runtime/alloy_services.go | 5 ++--- internal/service/remotecfg/remotecfg.go | 3 ++- internal/service/remotecfg/remotecfg_test.go | 4 ++-- internal/service/service.go | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/internal/alloycli/cmd_run.go b/internal/alloycli/cmd_run.go index f62aaec1fe..89357f2323 100644 --- a/internal/alloycli/cmd_run.go +++ b/internal/alloycli/cmd_run.go @@ -292,6 +292,7 @@ func (fr *alloyRun) Run(configPath string) error { remoteCfgService, err := remotecfgservice.New(remotecfgservice.Options{ Logger: log.With(l, "service", "remotecfg"), + ConfigPath: configPath, StoragePath: fr.storagePath, Metrics: reg, }) diff --git a/internal/runtime/alloy_services.go b/internal/runtime/alloy_services.go index bcded46bda..6c6171a7a6 100644 --- a/internal/runtime/alloy_services.go +++ b/internal/runtime/alloy_services.go @@ -93,13 +93,12 @@ type ServiceController struct { } func (sc ServiceController) Run(ctx context.Context) { sc.f.Run(ctx) } -func (sc ServiceController) LoadSource(b []byte, args map[string]any) error { +func (sc ServiceController) LoadSource(b []byte, args map[string]any, configPath string) error { source, err := ParseSource("", b) if err != nil { return err } - // The config is loaded via a service, the config path is left empty. - return sc.f.LoadSource(source, args, "") + return sc.f.LoadSource(source, args, configPath) } func (sc ServiceController) Ready() bool { return sc.f.Ready() } diff --git a/internal/service/remotecfg/remotecfg.go b/internal/service/remotecfg/remotecfg.go index cbe5dcedbd..50ef83aee0 100644 --- a/internal/service/remotecfg/remotecfg.go +++ b/internal/service/remotecfg/remotecfg.go @@ -90,6 +90,7 @@ const namespaceDelimiter = "." type Options struct { Logger log.Logger // Where to send logs. StoragePath string // Where to cache configuration on-disk. + ConfigPath string // Where the root config file is. Metrics prometheus.Registerer // Where to send metrics to. } @@ -467,7 +468,7 @@ func (s *Service) parseAndLoad(b []byte) error { return nil } - err := ctrl.LoadSource(b, nil) + err := ctrl.LoadSource(b, nil, s.opts.ConfigPath) if err != nil { return err } diff --git a/internal/service/remotecfg/remotecfg_test.go b/internal/service/remotecfg/remotecfg_test.go index 93890dfc57..5fe015fbb8 100644 --- a/internal/service/remotecfg/remotecfg_test.go +++ b/internal/service/remotecfg/remotecfg_test.go @@ -290,11 +290,11 @@ type serviceController struct { } func (sc serviceController) Run(ctx context.Context) { sc.f.Run(ctx) } -func (sc serviceController) LoadSource(b []byte, args map[string]any) error { +func (sc serviceController) LoadSource(b []byte, args map[string]any, configPath string) error { source, err := alloy_runtime.ParseSource("", b) if err != nil { return err } - return sc.f.LoadSource(source, args, "") + return sc.f.LoadSource(source, args, configPath) } func (sc serviceController) Ready() bool { return sc.f.Ready() } diff --git a/internal/service/service.go b/internal/service/service.go index c01d8562ec..b6fc24675f 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -75,7 +75,7 @@ type Host interface { // Controller is implemented by alloy.Alloy. type Controller interface { Run(ctx context.Context) - LoadSource(source []byte, args map[string]any) error + LoadSource(source []byte, args map[string]any, configPath string) error Ready() bool } From 86382ca253b2ee3bb1b35af0ad95425535791426 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Tue, 1 Oct 2024 17:54:56 +0200 Subject: [PATCH 12/17] use mutex to avoid data race --- internal/runtime/declare_test.go | 2 ++ internal/runtime/internal/controller/loader.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/internal/runtime/declare_test.go b/internal/runtime/declare_test.go index 75d47b39b1..d5bb56d74f 100644 --- a/internal/runtime/declare_test.go +++ b/internal/runtime/declare_test.go @@ -359,6 +359,7 @@ func TestDeclare(t *testing.T) { } func TestDeclareModulePath(t *testing.T) { + defer verifyNoGoroutineLeaks(t) config := ` declare "mod" { export "output" { @@ -390,6 +391,7 @@ func TestDeclareModulePath(t *testing.T) { cancel() <-done }() + time.Sleep(30 * time.Millisecond) passthrough := getExport[testcomponents.PassthroughExports](t, ctrl, "", "testcomponents.passthrough.pass") require.Equal(t, passthrough.Output, "") } diff --git a/internal/runtime/internal/controller/loader.go b/internal/runtime/internal/controller/loader.go index 48192299f9..8cbe0061fe 100644 --- a/internal/runtime/internal/controller/loader.go +++ b/internal/runtime/internal/controller/loader.go @@ -614,7 +614,9 @@ func (l *Loader) wireGraphEdges(g *dag.Graph) diag.Diagnostics { } // Finally, wire component references. + l.cache.mut.RLock() refs, nodeDiags := ComponentReferences(n, g, l.log, l.cache.scope) + l.cache.mut.RUnlock() for _, ref := range refs { g.AddEdge(dag.Edge{From: n, To: ref.Target}) } From cfff770776ac728c0cba2344d1c051ef10d365df Mon Sep 17 00:00:00 2001 From: William Dumont Date: Wed, 16 Oct 2024 12:27:01 +0200 Subject: [PATCH 13/17] add vm.scope constructors --- internal/runtime/alloy.go | 9 +++---- .../internal/controller/node_config_import.go | 20 +++++----------- .../internal/controller/value_cache.go | 5 +--- syntax/alloytypes/secret_test.go | 8 +++---- syntax/vm/op_binary_test.go | 12 ++++------ syntax/vm/vm.go | 13 ++++++++++ syntax/vm/vm_benchmarks_test.go | 8 +++---- syntax/vm/vm_errors_test.go | 12 ++++------ syntax/vm/vm_stdlib_test.go | 22 +++++++---------- syntax/vm/vm_test.go | 24 +++++++------------ 10 files changed, 56 insertions(+), 77 deletions(-) diff --git a/internal/runtime/alloy.go b/internal/runtime/alloy.go index 6a21d9575c..613fc204da 100644 --- a/internal/runtime/alloy.go +++ b/internal/runtime/alloy.go @@ -309,12 +309,9 @@ func (f *Runtime) LoadSource(source *Source, args map[string]any, configPath str ComponentBlocks: source.components, ConfigBlocks: source.configBlocks, DeclareBlocks: source.declareBlocks, - ArgScope: &vm.Scope{ - Parent: nil, - Variables: map[string]interface{}{ - importsource.ModulePath: modulePath, - }, - }, + ArgScope: vm.NewScope(map[string]interface{}{ + importsource.ModulePath: modulePath, + }), }) } diff --git a/internal/runtime/internal/controller/node_config_import.go b/internal/runtime/internal/controller/node_config_import.go index 3b7a7586a8..5d6e5a200a 100644 --- a/internal/runtime/internal/controller/node_config_import.go +++ b/internal/runtime/internal/controller/node_config_import.go @@ -301,15 +301,9 @@ func (cn *ImportConfigNode) processImportBlock(stmt *ast.BlockStmt, fullName str // evaluateChildren evaluates the import nodes managed by this import node. func (cn *ImportConfigNode) evaluateChildren() error { for _, child := range cn.importConfigNodesChildren { - err := child.Evaluate(&vm.Scope{ - Parent: &vm.Scope{ - Parent: nil, - Variables: map[string]interface{}{ - importsource.ModulePath: cn.source.ModulePath(), - }, - }, - Variables: make(map[string]interface{}), - }) + err := child.Evaluate(vm.NewScope(map[string]interface{}{ + importsource.ModulePath: cn.source.ModulePath(), + })) if err != nil { return fmt.Errorf("imported node %s failed to evaluate, %v", child.label, err) } @@ -436,11 +430,9 @@ func (cn *ImportConfigNode) ImportedDeclares() map[string]ast.Body { // Scope returns the scope associated with the import source. func (cn *ImportConfigNode) Scope() *vm.Scope { - return &vm.Scope{ - Variables: map[string]interface{}{ - importsource.ModulePath: cn.source.ModulePath(), - }, - } + return vm.NewScope(map[string]interface{}{ + importsource.ModulePath: cn.source.ModulePath(), + }) } // ImportConfigNodesChildren returns the ImportConfigNodesChildren of this ImportConfigNode. diff --git a/internal/runtime/internal/controller/value_cache.go b/internal/runtime/internal/controller/value_cache.go index 3815a98a44..6aae014ba9 100644 --- a/internal/runtime/internal/controller/value_cache.go +++ b/internal/runtime/internal/controller/value_cache.go @@ -171,10 +171,7 @@ func (vc *valueCache) BuildContext() *vm.Scope { vc.mut.RLock() defer vc.mut.RUnlock() - scope := &vm.Scope{ - Parent: vc.scope, - Variables: make(map[string]interface{}), - } + scope := vm.NewScopeWithParent(vc.scope, make(map[string]interface{})) // First, partition components by Alloy block name. var componentsByBlockName = make(map[string][]ComponentID) diff --git a/syntax/alloytypes/secret_test.go b/syntax/alloytypes/secret_test.go index 69b770a615..eedda3b48c 100644 --- a/syntax/alloytypes/secret_test.go +++ b/syntax/alloytypes/secret_test.go @@ -39,9 +39,7 @@ func decodeTo(t *testing.T, input interface{}, target interface{}) error { require.NoError(t, err) eval := vm.New(expr) - return eval.Evaluate(&vm.Scope{ - Variables: map[string]interface{}{ - "val": input, - }, - }, target) + return eval.Evaluate(vm.NewScope(map[string]interface{}{ + "val": input, + }), target) } diff --git a/syntax/vm/op_binary_test.go b/syntax/vm/op_binary_test.go index 11803c2283..015711323d 100644 --- a/syntax/vm/op_binary_test.go +++ b/syntax/vm/op_binary_test.go @@ -11,13 +11,11 @@ import ( ) func TestVM_OptionalSecret_Conversion(t *testing.T) { - scope := &vm.Scope{ - Variables: map[string]any{ - "string_val": "hello", - "non_secret_val": alloytypes.OptionalSecret{IsSecret: false, Value: "world"}, - "secret_val": alloytypes.OptionalSecret{IsSecret: true, Value: "secret"}, - }, - } + scope := vm.NewScope(map[string]any{ + "string_val": "hello", + "non_secret_val": alloytypes.OptionalSecret{IsSecret: false, Value: "world"}, + "secret_val": alloytypes.OptionalSecret{IsSecret: true, Value: "secret"}, + }) tt := []struct { name string diff --git a/syntax/vm/vm.go b/syntax/vm/vm.go index 9dfed95ac6..71b2893ccc 100644 --- a/syntax/vm/vm.go +++ b/syntax/vm/vm.go @@ -469,6 +469,19 @@ type Scope struct { Variables map[string]interface{} } +func NewScope(variables map[string]interface{}) *Scope { + return &Scope{ + Variables: variables, + } +} + +func NewScopeWithParent(parent *Scope, variables map[string]interface{}) *Scope { + return &Scope{ + Parent: parent, + Variables: variables, + } +} + // Lookup looks up a named identifier from the scope, all of the scope's // parents, and the stdlib. func (s *Scope) Lookup(name string) (interface{}, bool) { diff --git a/syntax/vm/vm_benchmarks_test.go b/syntax/vm/vm_benchmarks_test.go index 0d1e37335d..d5bfce790b 100644 --- a/syntax/vm/vm_benchmarks_test.go +++ b/syntax/vm/vm_benchmarks_test.go @@ -13,11 +13,9 @@ import ( func BenchmarkExprs(b *testing.B) { // Shared scope across all tests below - scope := &vm.Scope{ - Variables: map[string]interface{}{ - "foobar": int(42), - }, - } + scope := vm.NewScope(map[string]interface{}{ + "foobar": int(42), + }) tt := []struct { name string diff --git a/syntax/vm/vm_errors_test.go b/syntax/vm/vm_errors_test.go index 219d6d47c7..d2f2502eae 100644 --- a/syntax/vm/vm_errors_test.go +++ b/syntax/vm/vm_errors_test.go @@ -46,15 +46,13 @@ func TestVM_ExprErrors(t *testing.T) { name: "deeply nested indirect", input: `key = key_value`, into: &Target{}, - scope: &vm.Scope{ - Variables: map[string]interface{}{ - "key_value": map[string]interface{}{ - "object": map[string]interface{}{ - "field1": []interface{}{15, 30, "Hello, world!"}, - }, + scope: vm.NewScope(map[string]interface{}{ + "key_value": map[string]interface{}{ + "object": map[string]interface{}{ + "field1": []interface{}{15, 30, "Hello, world!"}, }, }, - }, + }), expect: `test:1:7: key_value.object.field1[2] should be number, got string`, }, { diff --git a/syntax/vm/vm_stdlib_test.go b/syntax/vm/vm_stdlib_test.go index 4454a31d89..e8009ae157 100644 --- a/syntax/vm/vm_stdlib_test.go +++ b/syntax/vm/vm_stdlib_test.go @@ -119,12 +119,10 @@ func TestStdlibJsonPath(t *testing.T) { } func TestStdlib_Nonsensitive(t *testing.T) { - scope := &vm.Scope{ - Variables: map[string]any{ - "secret": alloytypes.Secret("foo"), - "optionalSecret": alloytypes.OptionalSecret{Value: "bar"}, - }, - } + scope := vm.NewScope(map[string]any{ + "secret": alloytypes.Secret("foo"), + "optionalSecret": alloytypes.OptionalSecret{Value: "bar"}, + }) tt := []struct { name string @@ -152,9 +150,7 @@ func TestStdlib_Nonsensitive(t *testing.T) { } } func TestStdlib_StringFunc(t *testing.T) { - scope := &vm.Scope{ - Variables: map[string]any{}, - } + scope := vm.NewScope(make(map[string]interface{})) tt := []struct { name string @@ -276,11 +272,9 @@ func BenchmarkConcat(b *testing.B) { Attrs: data, }) } - scope := &vm.Scope{ - Variables: map[string]interface{}{ - "values_ref": valuesRef, - }, - } + scope := vm.NewScope(map[string]interface{}{ + "values_ref": valuesRef, + }) // Reset timer before running the actual test b.ResetTimer() diff --git a/syntax/vm/vm_test.go b/syntax/vm/vm_test.go index 877ac879b3..7fc8fc577d 100644 --- a/syntax/vm/vm_test.go +++ b/syntax/vm/vm_test.go @@ -63,11 +63,9 @@ func TestVM_Evaluate_Literals(t *testing.T) { func TestVM_Evaluate(t *testing.T) { // Shared scope across all tests below - scope := &vm.Scope{ - Variables: map[string]interface{}{ - "foobar": int(42), - }, - } + scope := vm.NewScope(map[string]interface{}{ + "foobar": int(42), + }) tt := []struct { input string @@ -176,11 +174,9 @@ func TestVM_Evaluate_Null(t *testing.T) { func TestVM_Evaluate_IdentifierExpr(t *testing.T) { t.Run("Valid lookup", func(t *testing.T) { - scope := &vm.Scope{ - Variables: map[string]interface{}{ - "foobar": 15, - }, - } + scope := vm.NewScope(map[string]interface{}{ + "foobar": 15, + }) expr, err := parser.ParseExpression(`foobar`) require.NoError(t, err) @@ -210,11 +206,9 @@ func TestVM_Evaluate_AccessExpr(t *testing.T) { Name string `alloy:"name,attr,optional"` } - scope := &vm.Scope{ - Variables: map[string]interface{}{ - "person": Person{}, - }, - } + scope := vm.NewScope(map[string]interface{}{ + "person": Person{}, + }) expr, err := parser.ParseExpression(`person.name`) require.NoError(t, err) From 87083cf4ed98e711313531c0b75ca887e1dccb13 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Wed, 16 Oct 2024 12:28:47 +0200 Subject: [PATCH 14/17] update import git doc --- docs/sources/reference/config-blocks/import.git.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/reference/config-blocks/import.git.md b/docs/sources/reference/config-blocks/import.git.md index 018ba90820..6aad5cd069 100644 --- a/docs/sources/reference/config-blocks/import.git.md +++ b/docs/sources/reference/config-blocks/import.git.md @@ -10,7 +10,7 @@ The `import.git` block imports custom components from a Git repository and expos `import.git` blocks must be given a label that determines the namespace where custom components are exposed. The entire repository is cloned, and the module path is accessible via the `module_path` keyword. -This enables your module to import other modules within the repository by setting relative paths in the [import.file][] blocks. +This enables, for example, your module to import other modules within the repository by setting relative paths in the [import.file][] blocks. ## Usage From 72440bdc29f60919da747aa5c15b7c463e08a40b Mon Sep 17 00:00:00 2001 From: William Dumont Date: Wed, 16 Oct 2024 12:53:36 +0200 Subject: [PATCH 15/17] update import examples --- .../reference/config-blocks/import.file.md | 43 ++++++------------- .../reference/config-blocks/import.http.md | 7 ++- 2 files changed, 16 insertions(+), 34 deletions(-) diff --git a/docs/sources/reference/config-blocks/import.file.md b/docs/sources/reference/config-blocks/import.file.md index ce36fd2e06..9f0b476b23 100644 --- a/docs/sources/reference/config-blocks/import.file.md +++ b/docs/sources/reference/config-blocks/import.file.md @@ -40,12 +40,13 @@ The following arguments are supported: {{< docs/shared lookup="reference/components/local-file-arguments-text.md" source="alloy" version="" >}} -## Example +## Examples -This example imports a module from a file and instantiates a custom component from the import that adds two numbers: +### Import a module from a local file -{{< collapse title="main.alloy" >}} +This example imports a module from a file and instantiates a custom component from the import that adds two numbers: +main.alloy ```alloy import.file "math" { filename = "module.alloy" @@ -57,10 +58,7 @@ math.add "default" { } ``` -{{< /collapse >}} - -{{< collapse title="module.alloy" >}} - +module.alloy ```alloy declare "add" { argument "a" {} @@ -72,13 +70,11 @@ declare "add" { } ``` -{{< /collapse >}} - +### Import a module in a module imported via import.git This example imports a module from a file inside of a module that is imported via [import.git][]: -{{< collapse title="main.alloy" >}} - +main.alloy ```alloy import.git "math" { repository = "https://github.com/wildum/module.git" @@ -91,10 +87,8 @@ math.add "default" { } ``` -{{< /collapse >}} - -{{< collapse title="relative_math.alloy" >}} +relative_math.alloy ```alloy import.file "lib" { filename = file.path_join(module_path, "lib.alloy") @@ -115,10 +109,7 @@ declare "add" { } ``` -{{< /collapse >}} - -{{< collapse title="lib.alloy" >}} - +lib.alloy ```alloy declare "plus" { argument "a" {} @@ -130,11 +121,11 @@ declare "plus" { } ``` -{{< /collapse >}} +### Import a module in a module imported via import.file This example imports a module from a file inside of a module that is imported via another `import.file`: -{{< collapse title="main.alloy" >}} +main.alloy ```alloy import.file "math" { @@ -147,10 +138,7 @@ math.add "default" { } ``` -{{< /collapse >}} - -{{< collapse title="relative_math.alloy" >}} - +relative_math.alloy ```alloy import.file "lib" { filename = file.path_join(module_path, "lib.alloy") @@ -171,10 +159,7 @@ declare "add" { } ``` -{{< /collapse >}} - -{{< collapse title="lib.alloy" >}} - +lib.alloy ```alloy declare "plus" { argument "a" {} @@ -186,8 +171,6 @@ declare "plus" { } ``` -{{< /collapse >}} - [file.path_join]: ../../stdlib/file/ diff --git a/docs/sources/reference/config-blocks/import.http.md b/docs/sources/reference/config-blocks/import.http.md index 77f791424d..552581851b 100644 --- a/docs/sources/reference/config-blocks/import.http.md +++ b/docs/sources/reference/config-blocks/import.http.md @@ -78,7 +78,7 @@ The `tls_config` block configures TLS settings for connecting to HTTPS servers. This example imports custom components from an HTTP response and instantiates a custom component for adding two numbers: -{{< collapse title="HTTP response" >}} +module.alloy ```alloy declare "add" { argument "a" {} @@ -89,9 +89,8 @@ declare "add" { } } ``` -{{< /collapse >}} -{{< collapse title="importer.alloy" >}} +main.alloy ```alloy import.http "math" { url = SERVER_URL @@ -102,7 +101,7 @@ math.add "default" { b = 45 } ``` -{{< /collapse >}} + [client]: #client-block [basic_auth]: #basic_auth-block From bae654a35594961dbe27d5609069bd6cf87c570a Mon Sep 17 00:00:00 2001 From: William Dumont Date: Wed, 16 Oct 2024 18:38:03 +0200 Subject: [PATCH 16/17] Update docs/sources/reference/config-blocks/import.file.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- docs/sources/reference/config-blocks/import.file.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/reference/config-blocks/import.file.md b/docs/sources/reference/config-blocks/import.file.md index 9f0b476b23..7e525833fd 100644 --- a/docs/sources/reference/config-blocks/import.file.md +++ b/docs/sources/reference/config-blocks/import.file.md @@ -18,7 +18,7 @@ That means that you can define a custom component in one file and use it in anot in the same directory. You can use the keyword `module_path` in combination with the `stdlib` function [file.path_join][] to import a module relative to the current module's path. -The `module_path` keyword works for modules that are imported via `import.file`, `import.git` and `import.string`. +The `module_path` keyword works for modules that are imported via `import.file`, `import.git`, and `import.string`. ## Usage From b873cf36c29ef588b7ed44404964712dab92b80a Mon Sep 17 00:00:00 2001 From: William Dumont Date: Thu, 17 Oct 2024 13:36:01 +0200 Subject: [PATCH 17/17] add missing revision in example --- docs/sources/reference/config-blocks/import.file.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/sources/reference/config-blocks/import.file.md b/docs/sources/reference/config-blocks/import.file.md index 9f0b476b23..4a2f876ef6 100644 --- a/docs/sources/reference/config-blocks/import.file.md +++ b/docs/sources/reference/config-blocks/import.file.md @@ -79,6 +79,7 @@ main.alloy import.git "math" { repository = "https://github.com/wildum/module.git" path = "relative_math.alloy" + revision = "master" } math.add "default" {