From 062074df21dc55eba1f861cea047178ad8e7a87b Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 28 Aug 2023 17:13:09 +0200 Subject: [PATCH 01/29] Simplified compileFiles --- arduino/builder/compilation_database.go | 27 ++++++++++------------- legacy/builder/builder_utils/utils.go | 29 ++++++++++--------------- 2 files changed, 23 insertions(+), 33 deletions(-) diff --git a/arduino/builder/compilation_database.go b/arduino/builder/compilation_database.go index 54a91d0c707..9d82cb88382 100644 --- a/arduino/builder/compilation_database.go +++ b/arduino/builder/compilation_database.go @@ -67,24 +67,21 @@ func (db *CompilationDatabase) SaveToFile() { } } -func dirForCommand(command *exec.Cmd) string { - // This mimics what Cmd.Run also does: Use Dir if specified, - // current directory otherwise - if command.Dir != "" { - return command.Dir - } - dir, err := os.Getwd() - if err != nil { - fmt.Println(tr("Error getting current directory for compilation database: %s", err)) - return "" - } - return dir -} - // Add adds a new CompilationDatabase entry func (db *CompilationDatabase) Add(target *paths.Path, command *exec.Cmd) { + commandDir := command.Dir + if commandDir == "" { + // This mimics what Cmd.Run also does: Use Dir if specified, + // current directory otherwise + dir, err := os.Getwd() + if err != nil { + fmt.Println(tr("Error getting current directory for compilation database: %s", err)) + } + commandDir = dir + } + entry := CompilationCommand{ - Directory: dirForCommand(command), + Directory: commandDir, Arguments: command.Args, File: target.String(), } diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go index 4a227e76154..f7cd0348c55 100644 --- a/legacy/builder/builder_utils/utils.go +++ b/legacy/builder/builder_utils/utils.go @@ -64,32 +64,25 @@ func DirContentIsOlderThan(dir *paths.Path, target *paths.Path, extensions ...st return true, nil } -func CompileFiles(ctx *types.Context, sourcePath *paths.Path, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { - return compileFiles(ctx, sourcePath, false, buildPath, buildProperties, includes) +func CompileFiles(ctx *types.Context, sourceDir *paths.Path, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { + return compileFiles(ctx, sourceDir, false, buildPath, buildProperties, includes) } -func CompileFilesRecursive(ctx *types.Context, sourcePath *paths.Path, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { - return compileFiles(ctx, sourcePath, true, buildPath, buildProperties, includes) +func CompileFilesRecursive(ctx *types.Context, sourceDir *paths.Path, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { + return compileFiles(ctx, sourceDir, true, buildPath, buildProperties, includes) } -func compileFiles(ctx *types.Context, sourcePath *paths.Path, recurse bool, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { - var sources paths.PathList - var err error - if recurse { - sources, err = sourcePath.ReadDirRecursive() - } else { - sources, err = sourcePath.ReadDir() - } - if err != nil { - return nil, err - } - +func compileFiles(ctx *types.Context, sourceDir *paths.Path, recurse bool, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { validExtensions := []string{} for ext := range globals.SourceFilesValidExtensions { validExtensions = append(validExtensions, ext) } - sources.FilterSuffix(validExtensions...) + sources, err := utils.FindFilesInFolder(sourceDir, recurse, validExtensions...) + if err != nil { + return nil, err + } + ctx.Progress.AddSubSteps(len(sources)) defer ctx.Progress.RemoveSubSteps() @@ -107,7 +100,7 @@ func compileFiles(ctx *types.Context, sourcePath *paths.Path, recurse bool, buil if !buildProperties.ContainsKey(recipe) { recipe = fmt.Sprintf("recipe%s.o.pattern", globals.SourceFilesValidExtensions[source.Ext()]) } - objectFile, err := compileFileWithRecipe(ctx, sourcePath, source, buildPath, buildProperties, includes, recipe) + objectFile, err := compileFileWithRecipe(ctx, sourceDir, source, buildPath, buildProperties, includes, recipe) if err != nil { errorsMux.Lock() errorsList = append(errorsList, err) From 1b66c2df3891b0db0a61f7b1ef1d1a1ede04d60e Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 28 Aug 2023 18:10:13 +0200 Subject: [PATCH 02/29] Starting new Builder (part 1/n) --- arduino/builder/builder.go | 30 +++++++++++++++++++++++++++++ arduino/builder/sketch.go | 8 ++++---- commands/compile/compile.go | 1 + legacy/builder/builder.go | 5 ++--- legacy/builder/test/builder_test.go | 1 + legacy/builder/types/context.go | 2 ++ 6 files changed, 40 insertions(+), 7 deletions(-) create mode 100644 arduino/builder/builder.go diff --git a/arduino/builder/builder.go b/arduino/builder/builder.go new file mode 100644 index 00000000000..7408ece5c3f --- /dev/null +++ b/arduino/builder/builder.go @@ -0,0 +1,30 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package builder + +import "github.com/arduino/arduino-cli/arduino/sketch" + +// Builder is a Sketch builder. +type Builder struct { + sketch *sketch.Sketch +} + +// NewBuilder creates a sketch Builder. +func NewBuilder(sk *sketch.Sketch) *Builder { + return &Builder{ + sketch: sk, + } +} diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index a79bbac4bef..69184edab94 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -37,12 +37,12 @@ var ( // PrepareSketchBuildPath copies the sketch source files in the build path. // The .ino files are merged together to create a .cpp file (by the way, the // .cpp file still needs to be Arduino-preprocessed to compile). -func PrepareSketchBuildPath(sketch *sketch.Sketch, sourceOverrides map[string]string, buildPath *paths.Path) (int, error) { - if offset, mergedSource, err := sketchMergeSources(sketch, sourceOverrides); err != nil { +func (b *Builder) PrepareSketchBuildPath(sourceOverrides map[string]string, buildPath *paths.Path) (int, error) { + if offset, mergedSource, err := sketchMergeSources(b.sketch, sourceOverrides); err != nil { return 0, err - } else if err := SketchSaveItemCpp(sketch.MainFile, []byte(mergedSource), buildPath); err != nil { + } else if err := SketchSaveItemCpp(b.sketch.MainFile, []byte(mergedSource), buildPath); err != nil { return 0, err - } else if err := sketchCopyAdditionalFiles(sketch, buildPath, sourceOverrides); err != nil { + } else if err := sketchCopyAdditionalFiles(b.sketch, buildPath, sourceOverrides); err != nil { return 0, err } else { return offset, nil diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 92858dd68d6..4bd2447b691 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -169,6 +169,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream } builderCtx := &types.Context{} + builderCtx.Builder = bldr.NewBuilder(sk) builderCtx.PackageManager = pme if pme.GetProfile() != nil { builderCtx.LibrariesManager = lm diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index 90d92938811..06f4371f524 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -19,7 +19,6 @@ import ( "reflect" "time" - "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/builder/preprocessor" "github.com/arduino/arduino-cli/i18n" "github.com/arduino/arduino-cli/legacy/builder/phases" @@ -50,7 +49,7 @@ func (s *Builder) Run(ctx *types.Context) error { &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, _err = builder.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), @@ -170,7 +169,7 @@ func (s *Preprocess) Run(ctx *types.Context) error { &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, _err = builder.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = ctx.Builder.PrepareSketchBuildPath(ctx.SourceOverride, ctx.SketchBuildPath) return _err }), diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index 58a372d6e41..a6b6a00d7c8 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -98,6 +98,7 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat ctx.RequiredTools = requiredTools } + ctx.Builder = bldr.NewBuilder(ctx.Sketch) return ctx } diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 0a160324a46..0c887ee6125 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -64,6 +64,8 @@ func (p *ProgressStruct) CompleteStep() { // Context structure type Context struct { + Builder *builder.Builder + // Build options HardwareDirs paths.PathList BuiltInToolsDirs paths.PathList From c720bec176f0b42f81ec00b151fb154169061a84 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Tue, 29 Aug 2023 09:35:35 +0200 Subject: [PATCH 03/29] builder: renamed variable and moved dir creation up --- arduino/builder/sketch.go | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index 69184edab94..9a95500a1c7 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -38,6 +38,9 @@ var ( // The .ino files are merged together to create a .cpp file (by the way, the // .cpp file still needs to be Arduino-preprocessed to compile). func (b *Builder) PrepareSketchBuildPath(sourceOverrides map[string]string, buildPath *paths.Path) (int, error) { + if err := buildPath.MkdirAll(); err != nil { + return 0, errors.Wrap(err, tr("unable to create a folder to save the sketch")) + } if offset, mergedSource, err := sketchMergeSources(b.sketch, sourceOverrides); err != nil { return 0, err } else if err := SketchSaveItemCpp(b.sketch.MainFile, []byte(mergedSource), buildPath); err != nil { @@ -50,13 +53,10 @@ func (b *Builder) PrepareSketchBuildPath(sourceOverrides map[string]string, buil } // SketchSaveItemCpp saves a preprocessed .cpp sketch file on disk -func SketchSaveItemCpp(path *paths.Path, contents []byte, destPath *paths.Path) error { +func SketchSaveItemCpp(path *paths.Path, contents []byte, buildPath *paths.Path) error { sketchName := path.Base() - if err := destPath.MkdirAll(); err != nil { - return errors.Wrap(err, tr("unable to create a folder to save the sketch")) - } - destFile := destPath.Join(fmt.Sprintf("%s.cpp", sketchName)) + destFile := buildPath.Join(fmt.Sprintf("%s.cpp", sketchName)) if err := destFile.WriteFile(contents); err != nil { return errors.Wrap(err, tr("unable to save the sketch on disk")) @@ -114,18 +114,14 @@ func sketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, st // sketchCopyAdditionalFiles copies the additional files for a sketch to the // specified destination directory. -func sketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath *paths.Path, overrides map[string]string) error { - if err := destPath.MkdirAll(); err != nil { - return errors.Wrap(err, tr("unable to create a folder to save the sketch files")) - } - +func sketchCopyAdditionalFiles(sketch *sketch.Sketch, buildPath *paths.Path, overrides map[string]string) error { for _, file := range sketch.AdditionalFiles { relpath, err := sketch.FullPath.RelTo(file) if err != nil { return errors.Wrap(err, tr("unable to compute relative path to the sketch for the item")) } - targetPath := destPath.JoinPath(relpath) + targetPath := buildPath.JoinPath(relpath) // create the directory containing the target if err = targetPath.Parent().MkdirAll(); err != nil { return errors.Wrap(err, tr("unable to create the folder containing the item")) From 15e419a5262fc9abf09836485393661b9f7bdd8a Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Tue, 29 Aug 2023 10:49:01 +0200 Subject: [PATCH 04/29] builder: made a sketch-prepress function part of Builder --- arduino/builder/sketch.go | 12 +++---- arduino/builder/sketch_test.go | 60 +++++++++++++++------------------- 2 files changed, 32 insertions(+), 40 deletions(-) diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index 9a95500a1c7..dc30f9752dc 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -41,7 +41,7 @@ func (b *Builder) PrepareSketchBuildPath(sourceOverrides map[string]string, buil if err := buildPath.MkdirAll(); err != nil { return 0, errors.Wrap(err, tr("unable to create a folder to save the sketch")) } - if offset, mergedSource, err := sketchMergeSources(b.sketch, sourceOverrides); err != nil { + if offset, mergedSource, err := b.sketchMergeSources(sourceOverrides); err != nil { return 0, err } else if err := SketchSaveItemCpp(b.sketch.MainFile, []byte(mergedSource), buildPath); err != nil { return 0, err @@ -67,12 +67,12 @@ func SketchSaveItemCpp(path *paths.Path, contents []byte, buildPath *paths.Path) // sketchMergeSources merges all the .ino source files included in a sketch to produce // a single .cpp file. -func sketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, string, error) { +func (b *Builder) sketchMergeSources(overrides map[string]string) (int, string, error) { lineOffset := 0 mergedSource := "" getSource := func(f *paths.Path) (string, error) { - path, err := sk.FullPath.RelTo(f) + path, err := b.sketch.FullPath.RelTo(f) if err != nil { return "", errors.Wrap(err, tr("unable to compute relative path to the sketch for the item")) } @@ -87,7 +87,7 @@ func sketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, st } // add Arduino.h inclusion directive if missing - mainSrc, err := getSource(sk.MainFile) + mainSrc, err := getSource(b.sketch.MainFile) if err != nil { return 0, "", err } @@ -96,11 +96,11 @@ func sketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, st lineOffset++ } - mergedSource += "#line 1 " + cpp.QuoteString(sk.MainFile.String()) + "\n" + mergedSource += "#line 1 " + cpp.QuoteString(b.sketch.MainFile.String()) + "\n" mergedSource += mainSrc + "\n" lineOffset++ - for _, file := range sk.OtherSketchFiles { + for _, file := range b.sketch.OtherSketchFiles { src, err := getSource(file) if err != nil { return 0, "", err diff --git a/arduino/builder/sketch_test.go b/arduino/builder/sketch_test.go index 5466cdff819..aa2e02f404a 100644 --- a/arduino/builder/sketch_test.go +++ b/arduino/builder/sketch_test.go @@ -17,8 +17,6 @@ package builder import ( "fmt" - "os" - "path/filepath" "runtime" "strings" "testing" @@ -28,21 +26,14 @@ import ( "github.com/stretchr/testify/require" ) -func tmpDirOrDie() *paths.Path { - dir, err := os.MkdirTemp(os.TempDir(), "builder_test") - if err != nil { - panic(fmt.Sprintf("error creating tmp dir: %v", err)) - } - return paths.New(dir) -} - func TestSaveSketch(t *testing.T) { - sketchName := t.Name() + ".ino" - outName := sketchName + ".cpp" - sketchFile := filepath.Join("testdata", sketchName) - tmp := tmpDirOrDie() + sketchName := t.Name() + ".ino" // "TestSaveSketch.ino" + outName := sketchName + ".cpp" // "TestSaveSketch.ino.cpp" + sketchFile := paths.New("testdata", sketchName) // "testdata/TestSaveSketch.ino.cpp" + tmp, err := paths.MkTempDir("", "") + require.NoError(t, err) defer tmp.RemoveAll() - source, err := os.ReadFile(sketchFile) + source, err := sketchFile.ReadFile() if err != nil { t.Fatalf("unable to read golden file %s: %v", sketchFile, err) } @@ -59,9 +50,9 @@ func TestSaveSketch(t *testing.T) { func TestMergeSketchSources(t *testing.T) { // borrow the sketch from TestLoadSketchFolder to avoid boilerplate - s, err := sketch.New(paths.New("testdata", "TestLoadSketchFolder")) + sk, err := sketch.New(paths.New("testdata", "TestLoadSketchFolder")) require.Nil(t, err) - require.NotNil(t, s) + require.NotNil(t, sk) // load expected result suffix := ".txt" @@ -69,66 +60,67 @@ func TestMergeSketchSources(t *testing.T) { suffix = "_win.txt" } mergedPath := paths.New("testdata", t.Name()+suffix) + require.NoError(t, mergedPath.ToAbs()) mergedBytes, err := mergedPath.ReadFile() - if err != nil { - t.Fatalf("unable to read golden file %s: %v", mergedPath, err) - } + require.NoError(t, err, "reading golden file %s: %v", mergedPath, err) - mergedPath.ToAbs() pathToGoldenSource := mergedPath.Parent().Parent().String() if runtime.GOOS == "windows" { pathToGoldenSource = strings.ReplaceAll(pathToGoldenSource, `\`, `\\`) } mergedSources := strings.ReplaceAll(string(mergedBytes), "%s", pathToGoldenSource) - offset, source, err := sketchMergeSources(s, nil) + b := NewBuilder(sk) + offset, source, err := b.sketchMergeSources(nil) require.Nil(t, err) require.Equal(t, 2, offset) require.Equal(t, mergedSources, source) } func TestMergeSketchSourcesArduinoIncluded(t *testing.T) { - s, err := sketch.New(paths.New("testdata", t.Name())) + sk, err := sketch.New(paths.New("testdata", t.Name())) require.Nil(t, err) - require.NotNil(t, s) + require.NotNil(t, sk) // ensure not to include Arduino.h when it's already there - _, source, err := sketchMergeSources(s, nil) + b := NewBuilder(sk) + _, source, err := b.sketchMergeSources(nil) require.Nil(t, err) require.Equal(t, 1, strings.Count(source, "")) } func TestCopyAdditionalFiles(t *testing.T) { - tmp := tmpDirOrDie() + tmp, err := paths.MkTempDir("", "") + require.NoError(t, err) defer tmp.RemoveAll() // load the golden sketch - s1, err := sketch.New(paths.New("testdata", t.Name())) + sk1, err := sketch.New(paths.New("testdata", t.Name())) require.Nil(t, err) - require.Equal(t, s1.AdditionalFiles.Len(), 1) + require.Equal(t, sk1.AdditionalFiles.Len(), 1) // copy the sketch over, create a fake main file we don't care about it // but we need it for `SketchLoad` to succeed later - err = sketchCopyAdditionalFiles(s1, tmp, nil) + err = sketchCopyAdditionalFiles(sk1, tmp, nil) require.Nil(t, err) fakeIno := tmp.Join(fmt.Sprintf("%s.ino", tmp.Base())) require.Nil(t, fakeIno.WriteFile([]byte{})) // compare - s2, err := sketch.New(tmp) + sk2, err := sketch.New(tmp) require.Nil(t, err) - require.Equal(t, s2.AdditionalFiles.Len(), 1) + require.Equal(t, sk2.AdditionalFiles.Len(), 1) // save file info - info1, err := s2.AdditionalFiles[0].Stat() + info1, err := sk2.AdditionalFiles[0].Stat() require.Nil(t, err) // copy again - err = sketchCopyAdditionalFiles(s1, tmp, nil) + err = sketchCopyAdditionalFiles(sk1, tmp, nil) require.Nil(t, err) // verify file hasn't changed - info2, err := s2.AdditionalFiles[0].Stat() + info2, err := sk2.AdditionalFiles[0].Stat() require.NoError(t, err) require.Equal(t, info1.ModTime(), info2.ModTime()) } From 0fae845dff879c6550218399c85583c4913455fc Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Tue, 29 Aug 2023 12:46:02 +0200 Subject: [PATCH 05/29] Removed ctx dependency in PreprocessSketchWithArduinoPreprocessor --- legacy/builder/builder.go | 20 +++++--- legacy/builder/preprocess_sketch.go | 80 ++++++++++++++--------------- 2 files changed, 51 insertions(+), 49 deletions(-) diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index 06f4371f524..1c1676b9f3a 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -138,19 +138,23 @@ func (s *Builder) Run(ctx *types.Context) error { } func PreprocessSketch(ctx *types.Context) error { + var normalOutput, verboseOutput []byte + var err error if ctx.UseArduinoPreprocessor { - return PreprocessSketchWithArduinoPreprocessor(ctx) + normalOutput, verboseOutput, err = PreprocessSketchWithArduinoPreprocessor( + ctx.Sketch, ctx.BuildPath, ctx.IncludeFolders, ctx.SketchBuildPath, + ctx.BuildProperties) } else { - normalOutput, verboseOutput, err := preprocessor.PreprocessSketchWithCtags( + normalOutput, verboseOutput, err = preprocessor.PreprocessSketchWithCtags( ctx.Sketch, ctx.BuildPath, ctx.IncludeFolders, ctx.LineOffset, ctx.BuildProperties, ctx.OnlyUpdateCompilationDatabase) - if ctx.Verbose { - ctx.WriteStdout(verboseOutput) - } else { - ctx.WriteStdout(normalOutput) - } - return err } + if ctx.Verbose { + ctx.WriteStdout(verboseOutput) + } else { + ctx.WriteStdout(normalOutput) + } + return err } type Preprocess struct{} diff --git a/legacy/builder/preprocess_sketch.go b/legacy/builder/preprocess_sketch.go index 7dff77c7dde..b2723e53f3d 100644 --- a/legacy/builder/preprocess_sketch.go +++ b/legacy/builder/preprocess_sketch.go @@ -16,74 +16,72 @@ package builder import ( - "fmt" - "os" - "os/exec" + "bytes" + "context" "path/filepath" "runtime" bldr "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/builder/preprocessor" - "github.com/arduino/arduino-cli/legacy/builder/types" + "github.com/arduino/arduino-cli/arduino/sketch" + "github.com/arduino/arduino-cli/executils" "github.com/arduino/arduino-cli/legacy/builder/utils" + "github.com/arduino/go-paths-helper" properties "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" ) -func PreprocessSketchWithArduinoPreprocessor(ctx *types.Context) error { - if err := ctx.BuildPath.Join("preproc").MkdirAll(); err != nil { - return errors.WithStack(err) +func PreprocessSketchWithArduinoPreprocessor(sk *sketch.Sketch, buildPath *paths.Path, includeFolders paths.PathList, sketchBuildPath *paths.Path, buildProperties *properties.Map) ([]byte, []byte, error) { + verboseOut := &bytes.Buffer{} + normalOut := &bytes.Buffer{} + if err := buildPath.Join("preproc").MkdirAll(); err != nil { + return nil, nil, err } - sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp") - targetFile := ctx.BuildPath.Join("preproc", "sketch_merged.cpp") - gccStdout, gccStderr, err := preprocessor.GCC(sourceFile, targetFile, ctx.IncludeFolders, ctx.BuildProperties) - if ctx.Verbose { - ctx.WriteStdout(gccStdout) - ctx.WriteStderr(gccStderr) - } + sourceFile := sketchBuildPath.Join(sk.MainFile.Base() + ".cpp") + targetFile := buildPath.Join("preproc", "sketch_merged.cpp") + gccStdout, gccStderr, err := preprocessor.GCC(sourceFile, targetFile, includeFolders, buildProperties) + verboseOut.Write(gccStdout) + verboseOut.Write(gccStderr) if err != nil { - return err + return nil, nil, err } - buildProperties := properties.NewMap() - buildProperties.Set("tools.arduino-preprocessor.path", "{runtime.tools.arduino-preprocessor.path}") - buildProperties.Set("tools.arduino-preprocessor.cmd.path", "{path}/arduino-preprocessor") - buildProperties.Set("tools.arduino-preprocessor.pattern", `"{cmd.path}" "{source_file}" -- -std=gnu++11`) - buildProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC") - buildProperties.Merge(ctx.BuildProperties) - buildProperties.Merge(buildProperties.SubTree("tools").SubTree("arduino-preprocessor")) - buildProperties.SetPath("source_file", targetFile) - - pattern := buildProperties.Get("pattern") + arduiniPreprocessorProperties := properties.NewMap() + arduiniPreprocessorProperties.Set("tools.arduino-preprocessor.path", "{runtime.tools.arduino-preprocessor.path}") + arduiniPreprocessorProperties.Set("tools.arduino-preprocessor.cmd.path", "{path}/arduino-preprocessor") + arduiniPreprocessorProperties.Set("tools.arduino-preprocessor.pattern", `"{cmd.path}" "{source_file}" -- -std=gnu++11`) + arduiniPreprocessorProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC") + arduiniPreprocessorProperties.Merge(buildProperties) + arduiniPreprocessorProperties.Merge(arduiniPreprocessorProperties.SubTree("tools").SubTree("arduino-preprocessor")) + arduiniPreprocessorProperties.SetPath("source_file", targetFile) + pattern := arduiniPreprocessorProperties.Get("pattern") if pattern == "" { - return errors.New(tr("arduino-preprocessor pattern is missing")) + return nil, nil, errors.New(tr("arduino-preprocessor pattern is missing")) } - commandLine := buildProperties.ExpandPropsInString(pattern) + commandLine := arduiniPreprocessorProperties.ExpandPropsInString(pattern) parts, err := properties.SplitQuotedString(commandLine, `"'`, false) if err != nil { - return errors.WithStack(err) + return nil, nil, errors.WithStack(err) } - command := exec.Command(parts[0], parts[1:]...) - command.Env = append(os.Environ(), ctx.PackageManager.GetEnvVarsForSpawnedProcess()...) + command, err := executils.NewProcess(nil, parts...) + if err != nil { + return nil, nil, err + } if runtime.GOOS == "windows" { // chdir in the uppermost directory to avoid UTF-8 bug in clang (https://github.com/arduino/arduino-preprocessor/issues/2) - command.Dir = filepath.VolumeName(command.Args[0]) + "/" - //command.Args[0], _ = filepath.Rel(command.Dir, command.Args[0]) - } - - verbose := ctx.Verbose - if verbose { - fmt.Println(commandLine) + command.SetDir(filepath.VolumeName(parts[0]) + "/") } - buf, err := command.Output() + verboseOut.WriteString(commandLine) + commandStdOut, commandStdErr, err := command.RunAndCaptureOutput(context.Background()) + verboseOut.Write(commandStdErr) if err != nil { - return errors.New(errors.WithStack(err).Error() + string(err.(*exec.ExitError).Stderr)) + return normalOut.Bytes(), verboseOut.Bytes(), err } + result := utils.NormalizeUTF8(commandStdOut) - result := utils.NormalizeUTF8(buf) - return bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, result, ctx.SketchBuildPath) + return normalOut.Bytes(), verboseOut.Bytes(), bldr.SketchSaveItemCpp(sk.MainFile, result, sketchBuildPath) } From d3253852e53d075c6a3abc72b154e510253fa907 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Tue, 29 Aug 2023 12:51:01 +0200 Subject: [PATCH 06/29] uniform parameters between preprocesors --- legacy/builder/builder.go | 4 ++-- legacy/builder/preprocess_sketch.go | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index 1c1676b9f3a..dfd84ac5b98 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -142,8 +142,8 @@ func PreprocessSketch(ctx *types.Context) error { var err error if ctx.UseArduinoPreprocessor { normalOutput, verboseOutput, err = PreprocessSketchWithArduinoPreprocessor( - ctx.Sketch, ctx.BuildPath, ctx.IncludeFolders, ctx.SketchBuildPath, - ctx.BuildProperties) + ctx.Sketch, ctx.BuildPath, ctx.IncludeFolders, ctx.LineOffset, + ctx.BuildProperties, ctx.OnlyUpdateCompilationDatabase) } else { normalOutput, verboseOutput, err = preprocessor.PreprocessSketchWithCtags( ctx.Sketch, ctx.BuildPath, ctx.IncludeFolders, ctx.LineOffset, diff --git a/legacy/builder/preprocess_sketch.go b/legacy/builder/preprocess_sketch.go index b2723e53f3d..d22a6abdd13 100644 --- a/legacy/builder/preprocess_sketch.go +++ b/legacy/builder/preprocess_sketch.go @@ -31,14 +31,14 @@ import ( "github.com/pkg/errors" ) -func PreprocessSketchWithArduinoPreprocessor(sk *sketch.Sketch, buildPath *paths.Path, includeFolders paths.PathList, sketchBuildPath *paths.Path, buildProperties *properties.Map) ([]byte, []byte, error) { +func PreprocessSketchWithArduinoPreprocessor(sk *sketch.Sketch, buildPath *paths.Path, includeFolders paths.PathList, lineOffset int, buildProperties *properties.Map, onlyUpdateCompilationDatabase bool) ([]byte, []byte, error) { verboseOut := &bytes.Buffer{} normalOut := &bytes.Buffer{} if err := buildPath.Join("preproc").MkdirAll(); err != nil { return nil, nil, err } - sourceFile := sketchBuildPath.Join(sk.MainFile.Base() + ".cpp") + sourceFile := buildPath.Join("sketch", sk.MainFile.Base()+".cpp") targetFile := buildPath.Join("preproc", "sketch_merged.cpp") gccStdout, gccStderr, err := preprocessor.GCC(sourceFile, targetFile, includeFolders, buildProperties) verboseOut.Write(gccStdout) @@ -83,5 +83,6 @@ func PreprocessSketchWithArduinoPreprocessor(sk *sketch.Sketch, buildPath *paths } result := utils.NormalizeUTF8(commandStdOut) - return normalOut.Bytes(), verboseOut.Bytes(), bldr.SketchSaveItemCpp(sk.MainFile, result, sketchBuildPath) + err = bldr.SketchSaveItemCpp(sk.MainFile, result, buildPath.Join("sketch")) + return normalOut.Bytes(), verboseOut.Bytes(), err } From 394890b56dc3fe6889a8657c79f041a516b9aef5 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Tue, 29 Aug 2023 13:03:47 +0200 Subject: [PATCH 07/29] Moved PreprocessSketchWithArduinoPreprocessor into proper place --- .../builder/preprocessor/arduino_preprocessor.go | 15 ++++++++------- legacy/builder/builder.go | 14 +++++--------- 2 files changed, 13 insertions(+), 16 deletions(-) rename legacy/builder/preprocess_sketch.go => arduino/builder/preprocessor/arduino_preprocessor.go (86%) diff --git a/legacy/builder/preprocess_sketch.go b/arduino/builder/preprocessor/arduino_preprocessor.go similarity index 86% rename from legacy/builder/preprocess_sketch.go rename to arduino/builder/preprocessor/arduino_preprocessor.go index d22a6abdd13..ec450f869f4 100644 --- a/legacy/builder/preprocess_sketch.go +++ b/arduino/builder/preprocessor/arduino_preprocessor.go @@ -1,6 +1,6 @@ // This file is part of arduino-cli. // -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// Copyright 2023 ARDUINO SA (http://www.arduino.cc/) // // This software is released under the GNU General Public License version 3, // which covers the main part of arduino-cli. @@ -13,7 +13,7 @@ // Arduino software without disclosing the source code of your own applications. // To purchase a commercial license, send an email to license@arduino.cc. -package builder +package preprocessor import ( "bytes" @@ -21,16 +21,17 @@ import ( "path/filepath" "runtime" - bldr "github.com/arduino/arduino-cli/arduino/builder" - "github.com/arduino/arduino-cli/arduino/builder/preprocessor" + "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/executils" "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/arduino/go-paths-helper" - properties "github.com/arduino/go-properties-orderedmap" + "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" ) +// PreprocessSketchWithArduinoPreprocessor performs preprocessing of the arduino sketch +// using arduino-preprocessor (https://github.com/arduino/arduino-preprocessor). func PreprocessSketchWithArduinoPreprocessor(sk *sketch.Sketch, buildPath *paths.Path, includeFolders paths.PathList, lineOffset int, buildProperties *properties.Map, onlyUpdateCompilationDatabase bool) ([]byte, []byte, error) { verboseOut := &bytes.Buffer{} normalOut := &bytes.Buffer{} @@ -40,7 +41,7 @@ func PreprocessSketchWithArduinoPreprocessor(sk *sketch.Sketch, buildPath *paths sourceFile := buildPath.Join("sketch", sk.MainFile.Base()+".cpp") targetFile := buildPath.Join("preproc", "sketch_merged.cpp") - gccStdout, gccStderr, err := preprocessor.GCC(sourceFile, targetFile, includeFolders, buildProperties) + gccStdout, gccStderr, err := GCC(sourceFile, targetFile, includeFolders, buildProperties) verboseOut.Write(gccStdout) verboseOut.Write(gccStderr) if err != nil { @@ -83,6 +84,6 @@ func PreprocessSketchWithArduinoPreprocessor(sk *sketch.Sketch, buildPath *paths } result := utils.NormalizeUTF8(commandStdOut) - err = bldr.SketchSaveItemCpp(sk.MainFile, result, buildPath.Join("sketch")) + err = builder.SketchSaveItemCpp(sk.MainFile, result, buildPath.Join("sketch")) return normalOut.Bytes(), verboseOut.Bytes(), err } diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index dfd84ac5b98..bce8bd24d6b 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -138,17 +138,13 @@ func (s *Builder) Run(ctx *types.Context) error { } func PreprocessSketch(ctx *types.Context) error { - var normalOutput, verboseOutput []byte - var err error + preprocessorImpl := preprocessor.PreprocessSketchWithCtags if ctx.UseArduinoPreprocessor { - normalOutput, verboseOutput, err = PreprocessSketchWithArduinoPreprocessor( - ctx.Sketch, ctx.BuildPath, ctx.IncludeFolders, ctx.LineOffset, - ctx.BuildProperties, ctx.OnlyUpdateCompilationDatabase) - } else { - normalOutput, verboseOutput, err = preprocessor.PreprocessSketchWithCtags( - ctx.Sketch, ctx.BuildPath, ctx.IncludeFolders, ctx.LineOffset, - ctx.BuildProperties, ctx.OnlyUpdateCompilationDatabase) + preprocessorImpl = preprocessor.PreprocessSketchWithArduinoPreprocessor } + normalOutput, verboseOutput, err := preprocessorImpl( + ctx.Sketch, ctx.BuildPath, ctx.IncludeFolders, ctx.LineOffset, + ctx.BuildProperties, ctx.OnlyUpdateCompilationDatabase) if ctx.Verbose { ctx.WriteStdout(verboseOutput) } else { From b013d1684bb603e0e0685054a684c77914a376a5 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Tue, 29 Aug 2023 17:55:11 +0200 Subject: [PATCH 08/29] Inlined function --- .../preprocessor/arduino_preprocessor.go | 7 +++-- arduino/builder/sketch.go | 26 +++++++------------ arduino/builder/sketch_test.go | 22 ---------------- 3 files changed, 15 insertions(+), 40 deletions(-) diff --git a/arduino/builder/preprocessor/arduino_preprocessor.go b/arduino/builder/preprocessor/arduino_preprocessor.go index ec450f869f4..8629ec211ba 100644 --- a/arduino/builder/preprocessor/arduino_preprocessor.go +++ b/arduino/builder/preprocessor/arduino_preprocessor.go @@ -21,7 +21,6 @@ import ( "path/filepath" "runtime" - "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/executils" "github.com/arduino/arduino-cli/legacy/builder/utils" @@ -84,6 +83,10 @@ func PreprocessSketchWithArduinoPreprocessor(sk *sketch.Sketch, buildPath *paths } result := utils.NormalizeUTF8(commandStdOut) - err = builder.SketchSaveItemCpp(sk.MainFile, result, buildPath.Join("sketch")) + destFile := buildPath.Join(sk.MainFile.Base() + ".cpp") + if err := destFile.WriteFile(result); err != nil { + return normalOut.Bytes(), verboseOut.Bytes(), err + } + return normalOut.Bytes(), verboseOut.Bytes(), err } diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index dc30f9752dc..793786968fb 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -41,28 +41,22 @@ func (b *Builder) PrepareSketchBuildPath(sourceOverrides map[string]string, buil if err := buildPath.MkdirAll(); err != nil { return 0, errors.Wrap(err, tr("unable to create a folder to save the sketch")) } - if offset, mergedSource, err := b.sketchMergeSources(sourceOverrides); err != nil { - return 0, err - } else if err := SketchSaveItemCpp(b.sketch.MainFile, []byte(mergedSource), buildPath); err != nil { - return 0, err - } else if err := sketchCopyAdditionalFiles(b.sketch, buildPath, sourceOverrides); err != nil { + + offset, mergedSource, err := b.sketchMergeSources(sourceOverrides) + if err != nil { return 0, err - } else { - return offset, nil } -} -// SketchSaveItemCpp saves a preprocessed .cpp sketch file on disk -func SketchSaveItemCpp(path *paths.Path, contents []byte, buildPath *paths.Path) error { - sketchName := path.Base() - - destFile := buildPath.Join(fmt.Sprintf("%s.cpp", sketchName)) + destFile := buildPath.Join(b.sketch.MainFile.Base() + ".cpp") + if err := destFile.WriteFile([]byte(mergedSource)); err != nil { + return 0, err + } - if err := destFile.WriteFile(contents); err != nil { - return errors.Wrap(err, tr("unable to save the sketch on disk")) + if err := sketchCopyAdditionalFiles(b.sketch, buildPath, sourceOverrides); err != nil { + return 0, err } - return nil + return offset, nil } // sketchMergeSources merges all the .ino source files included in a sketch to produce diff --git a/arduino/builder/sketch_test.go b/arduino/builder/sketch_test.go index aa2e02f404a..a877d9a6fd0 100644 --- a/arduino/builder/sketch_test.go +++ b/arduino/builder/sketch_test.go @@ -26,28 +26,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestSaveSketch(t *testing.T) { - sketchName := t.Name() + ".ino" // "TestSaveSketch.ino" - outName := sketchName + ".cpp" // "TestSaveSketch.ino.cpp" - sketchFile := paths.New("testdata", sketchName) // "testdata/TestSaveSketch.ino.cpp" - tmp, err := paths.MkTempDir("", "") - require.NoError(t, err) - defer tmp.RemoveAll() - source, err := sketchFile.ReadFile() - if err != nil { - t.Fatalf("unable to read golden file %s: %v", sketchFile, err) - } - - SketchSaveItemCpp(paths.New(sketchName), source, tmp) - - out, err := tmp.Join(outName).ReadFile() - if err != nil { - t.Fatalf("unable to read output file %s: %v", outName, err) - } - - require.Equal(t, source, out) -} - func TestMergeSketchSources(t *testing.T) { // borrow the sketch from TestLoadSketchFolder to avoid boilerplate sk, err := sketch.New(paths.New("testdata", "TestLoadSketchFolder")) From fccb3b4104614c4d967d322a61d81bf9215d280d Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Tue, 29 Aug 2023 18:04:29 +0200 Subject: [PATCH 09/29] Converted sketchCopyAdditionalFiles into a Builder method --- arduino/builder/sketch.go | 8 ++++---- arduino/builder/sketch_test.go | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index 793786968fb..bc740721f88 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -52,7 +52,7 @@ func (b *Builder) PrepareSketchBuildPath(sourceOverrides map[string]string, buil return 0, err } - if err := sketchCopyAdditionalFiles(b.sketch, buildPath, sourceOverrides); err != nil { + if err := b.sketchCopyAdditionalFiles(buildPath, sourceOverrides); err != nil { return 0, err } @@ -108,9 +108,9 @@ func (b *Builder) sketchMergeSources(overrides map[string]string) (int, string, // sketchCopyAdditionalFiles copies the additional files for a sketch to the // specified destination directory. -func sketchCopyAdditionalFiles(sketch *sketch.Sketch, buildPath *paths.Path, overrides map[string]string) error { - for _, file := range sketch.AdditionalFiles { - relpath, err := sketch.FullPath.RelTo(file) +func (b *Builder) sketchCopyAdditionalFiles(buildPath *paths.Path, overrides map[string]string) error { + for _, file := range b.sketch.AdditionalFiles { + relpath, err := b.sketch.FullPath.RelTo(file) if err != nil { return errors.Wrap(err, tr("unable to compute relative path to the sketch for the item")) } diff --git a/arduino/builder/sketch_test.go b/arduino/builder/sketch_test.go index a877d9a6fd0..b1fda4e6e8c 100644 --- a/arduino/builder/sketch_test.go +++ b/arduino/builder/sketch_test.go @@ -76,10 +76,11 @@ func TestCopyAdditionalFiles(t *testing.T) { sk1, err := sketch.New(paths.New("testdata", t.Name())) require.Nil(t, err) require.Equal(t, sk1.AdditionalFiles.Len(), 1) + b1 := NewBuilder(sk1) // copy the sketch over, create a fake main file we don't care about it // but we need it for `SketchLoad` to succeed later - err = sketchCopyAdditionalFiles(sk1, tmp, nil) + err = b1.sketchCopyAdditionalFiles(tmp, nil) require.Nil(t, err) fakeIno := tmp.Join(fmt.Sprintf("%s.ino", tmp.Base())) require.Nil(t, fakeIno.WriteFile([]byte{})) @@ -94,7 +95,7 @@ func TestCopyAdditionalFiles(t *testing.T) { require.Nil(t, err) // copy again - err = sketchCopyAdditionalFiles(sk1, tmp, nil) + err = b1.sketchCopyAdditionalFiles(tmp, nil) require.Nil(t, err) // verify file hasn't changed From 1408d7fe77e814a87549537a7a073f9d30c4aa69 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 30 Aug 2023 14:27:01 +0200 Subject: [PATCH 10/29] Made SetupBuildProperties a method of the new Builder --- arduino/builder/sketch.go | 9 ++++----- commands/compile/compile.go | 6 ++++-- legacy/builder/test/builder_test.go | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index bc740721f88..144164cf31b 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -21,7 +21,6 @@ import ( "regexp" "github.com/arduino/arduino-cli/arduino/builder/cpp" - "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/i18n" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" @@ -170,16 +169,16 @@ func writeIfDifferent(source []byte, destPath *paths.Path) error { // SetupBuildProperties adds the build properties related to the sketch to the // default board build properties map. -func SetupBuildProperties(boardBuildProperties *properties.Map, buildPath *paths.Path, sketch *sketch.Sketch, optimizeForDebug bool) *properties.Map { +func (b *Builder) SetupBuildProperties(boardBuildProperties *properties.Map, buildPath *paths.Path, optimizeForDebug bool) *properties.Map { buildProperties := properties.NewMap() buildProperties.Merge(boardBuildProperties) if buildPath != nil { buildProperties.SetPath("build.path", buildPath) } - if sketch != nil { - buildProperties.Set("build.project_name", sketch.MainFile.Base()) - buildProperties.SetPath("build.source.path", sketch.FullPath) + if b.sketch != nil { + buildProperties.Set("build.project_name", b.sketch.MainFile.Base()) + buildProperties.SetPath("build.source.path", b.sketch.FullPath) } if optimizeForDebug { if debugFlags, ok := buildProperties.GetOk("compiler.optimization_flags.debug"); ok { diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 4bd2447b691..b374213f2d3 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -152,8 +152,10 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream // cache is purged after compilation to not remove entries that might be required defer maybePurgeBuildCache() + sketchBuilder := bldr.NewBuilder(sk) + // Add build properites related to sketch data - buildProperties = bldr.SetupBuildProperties(buildProperties, buildPath, sk, req.GetOptimizeForDebug()) + buildProperties = sketchBuilder.SetupBuildProperties(buildProperties, buildPath, req.GetOptimizeForDebug()) // Add user provided custom build properties customBuildPropertiesArgs := append(req.GetBuildProperties(), "build.warn_data_percentage=75") @@ -169,7 +171,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream } builderCtx := &types.Context{} - builderCtx.Builder = bldr.NewBuilder(sk) + builderCtx.Builder = sketchBuilder builderCtx.PackageManager = pme if pme.GetProfile() != nil { builderCtx.LibrariesManager = lm diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index a6b6a00d7c8..5ec00979cfb 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -81,6 +81,7 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat ctx.Sketch = sk } + ctx.Builder = bldr.NewBuilder(ctx.Sketch) if fqbn != "" { ctx.FQBN = parseFQBN(t, fqbn) targetPackage, targetPlatform, targetBoard, buildProperties, buildPlatform, err := pme.ResolveFQBN(ctx.FQBN) @@ -88,7 +89,7 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat requiredTools, err := pme.FindToolsRequiredForBuild(targetPlatform, buildPlatform) require.NoError(t, err) - buildProperties = bldr.SetupBuildProperties(buildProperties, ctx.BuildPath, ctx.Sketch, false /*OptimizeForDebug*/) + buildProperties = ctx.Builder.SetupBuildProperties(buildProperties, ctx.BuildPath, false /*OptimizeForDebug*/) ctx.PackageManager = pme ctx.TargetBoard = targetBoard ctx.BuildProperties = buildProperties @@ -98,7 +99,6 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat ctx.RequiredTools = requiredTools } - ctx.Builder = bldr.NewBuilder(ctx.Sketch) return ctx } From 5534d6decc747e9ce4892ec0cb5a669880875900 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Thu, 31 Aug 2023 20:22:05 +0200 Subject: [PATCH 11/29] Refactor AddAdditionalEntriesToContext https://github.com/cmaglie/arduino-cli/pull/32 * refactor AddAdditionalEntriesToContext in a function * use the new function in all the tests * Move the assignaton of LibrariesResolutionResults inside the ResolveLibrary func The ResolveLibrary func is only called by the ContainerFindIncludes * rename bPath to buildPath * cleanup usless tests * remove shadowed variable --- .../add_additional_entries_to_context.go | 34 ++++++++----------- legacy/builder/container_find_includes.go | 4 +++ legacy/builder/container_setup.go | 11 +++++- .../add_additional_entries_to_context_test.go | 30 ++++++++-------- legacy/builder/test/hardware_loader_test.go | 9 ----- legacy/builder/test/libraries_loader_test.go | 4 --- legacy/builder/test/recipe_runner_test.go | 1 - .../test/setup_build_properties_test.go | 16 --------- 8 files changed, 42 insertions(+), 67 deletions(-) diff --git a/legacy/builder/add_additional_entries_to_context.go b/legacy/builder/add_additional_entries_to_context.go index 9e9fdd80669..843cbdcd79e 100644 --- a/legacy/builder/add_additional_entries_to_context.go +++ b/legacy/builder/add_additional_entries_to_context.go @@ -17,38 +17,32 @@ package builder import ( "github.com/arduino/arduino-cli/legacy/builder/constants" - "github.com/arduino/arduino-cli/legacy/builder/types" + "github.com/arduino/go-paths-helper" "github.com/pkg/errors" ) -type AddAdditionalEntriesToContext struct{} +func AddAdditionalEntriesToContext(buildPath *paths.Path, warningLevel string) (*paths.Path, *paths.Path, *paths.Path, string, error) { + var sketchBuildPath, librariesBuildPath, coreBuildPath *paths.Path + var err error -func (*AddAdditionalEntriesToContext) Run(ctx *types.Context) error { - if ctx.BuildPath != nil { - buildPath := ctx.BuildPath - sketchBuildPath, err := buildPath.Join(constants.FOLDER_SKETCH).Abs() + if buildPath != nil { + sketchBuildPath, err = buildPath.Join(constants.FOLDER_SKETCH).Abs() if err != nil { - return errors.WithStack(err) + return nil, nil, nil, "", errors.WithStack(err) } - librariesBuildPath, err := buildPath.Join(constants.FOLDER_LIBRARIES).Abs() + librariesBuildPath, err = buildPath.Join(constants.FOLDER_LIBRARIES).Abs() if err != nil { - return errors.WithStack(err) + return nil, nil, nil, "", errors.WithStack(err) } - coreBuildPath, err := buildPath.Join(constants.FOLDER_CORE).Abs() + coreBuildPath, err = buildPath.Join(constants.FOLDER_CORE).Abs() if err != nil { - return errors.WithStack(err) + return nil, nil, nil, "", errors.WithStack(err) } - - ctx.SketchBuildPath = sketchBuildPath - ctx.LibrariesBuildPath = librariesBuildPath - ctx.CoreBuildPath = coreBuildPath } - if ctx.WarningsLevel == "" { - ctx.WarningsLevel = DEFAULT_WARNINGS_LEVEL + if warningLevel == "" { + warningLevel = DEFAULT_WARNINGS_LEVEL } - ctx.LibrariesResolutionResults = map[string]types.LibraryResolutionResult{} - - return nil + return sketchBuildPath, librariesBuildPath, coreBuildPath, warningLevel, nil } diff --git a/legacy/builder/container_find_includes.go b/legacy/builder/container_find_includes.go index 5d849b927ed..55b60347898 100644 --- a/legacy/builder/container_find_includes.go +++ b/legacy/builder/container_find_includes.go @@ -447,6 +447,10 @@ func ResolveLibrary(ctx *types.Context, header string) *libraries.Library { resolver := ctx.LibrariesResolver importedLibraries := ctx.ImportedLibraries + if ctx.LibrariesResolutionResults == nil { + ctx.LibrariesResolutionResults = map[string]types.LibraryResolutionResult{} + } + candidates := resolver.AlternativesFor(header) if ctx.Verbose { diff --git a/legacy/builder/container_setup.go b/legacy/builder/container_setup.go index 3a00ff6edaa..8a99df5c8a9 100644 --- a/legacy/builder/container_setup.go +++ b/legacy/builder/container_setup.go @@ -23,8 +23,17 @@ import ( type ContainerSetupHardwareToolsLibsSketchAndProps struct{} func (s *ContainerSetupHardwareToolsLibsSketchAndProps) Run(ctx *types.Context) error { + sketchBuildPath, librariesBuildPath, coreBuildPath, + warningsLevel, err := AddAdditionalEntriesToContext(ctx.BuildPath, ctx.WarningsLevel) + if err != nil { + return errors.WithStack(err) + } + ctx.SketchBuildPath = sketchBuildPath + ctx.LibrariesBuildPath = librariesBuildPath + ctx.CoreBuildPath = coreBuildPath + ctx.WarningsLevel = warningsLevel + commands := []types.Command{ - &AddAdditionalEntriesToContext{}, &FailIfBuildPathEqualsSketchPath{}, &LibrariesLoader{}, } diff --git a/legacy/builder/test/add_additional_entries_to_context_test.go b/legacy/builder/test/add_additional_entries_to_context_test.go index fc1edb5acb2..dc15e16697c 100644 --- a/legacy/builder/test/add_additional_entries_to_context_test.go +++ b/legacy/builder/test/add_additional_entries_to_context_test.go @@ -28,30 +28,28 @@ import ( func TestAddAdditionalEntriesToContextNoBuildPath(t *testing.T) { ctx := &types.Context{} - command := builder.AddAdditionalEntriesToContext{} - NoError(t, command.Run(ctx)) + sketchBuildPath, librariesBuildPath, coreBuildPath, + warningsLevel, err := builder.AddAdditionalEntriesToContext(ctx.BuildPath, ctx.WarningsLevel) + NoError(t, err) - require.Empty(t, ctx.SketchBuildPath) - require.Empty(t, ctx.LibrariesBuildPath) - require.Empty(t, ctx.CoreBuildPath) + require.Empty(t, sketchBuildPath) + require.Empty(t, librariesBuildPath) + require.Empty(t, coreBuildPath) - require.NotNil(t, ctx.WarningsLevel) - - require.Equal(t, 0, len(ctx.LibrariesResolutionResults)) + require.NotNil(t, warningsLevel) } func TestAddAdditionalEntriesToContextWithBuildPath(t *testing.T) { ctx := &types.Context{} ctx.BuildPath = paths.New("folder") - command := builder.AddAdditionalEntriesToContext{} - NoError(t, command.Run(ctx)) - - require.Equal(t, Abs(t, paths.New("folder", constants.FOLDER_SKETCH)), ctx.SketchBuildPath) - require.Equal(t, Abs(t, paths.New("folder", "libraries")), ctx.LibrariesBuildPath) - require.Equal(t, Abs(t, paths.New("folder", constants.FOLDER_CORE)), ctx.CoreBuildPath) + sketchBuildPath, librariesBuildPath, coreBuildPath, + warningsLevel, err := builder.AddAdditionalEntriesToContext(ctx.BuildPath, ctx.WarningsLevel) + NoError(t, err) - require.NotNil(t, ctx.WarningsLevel) + require.Equal(t, Abs(t, paths.New("folder", constants.FOLDER_SKETCH)), sketchBuildPath) + require.Equal(t, Abs(t, paths.New("folder", "libraries")), librariesBuildPath) + require.Equal(t, Abs(t, paths.New("folder", constants.FOLDER_CORE)), coreBuildPath) - require.Equal(t, 0, len(ctx.LibrariesResolutionResults)) + require.NotNil(t, warningsLevel) } diff --git a/legacy/builder/test/hardware_loader_test.go b/legacy/builder/test/hardware_loader_test.go index f900ad38ac3..c7e4b0ba387 100644 --- a/legacy/builder/test/hardware_loader_test.go +++ b/legacy/builder/test/hardware_loader_test.go @@ -20,7 +20,6 @@ import ( "runtime" "testing" - "github.com/arduino/arduino-cli/legacy/builder" "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" @@ -69,14 +68,6 @@ func TestLoadHardwareMixingUserHardwareFolder(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, nil, "") defer cleanUpBuilderTestContext(t, ctx) - commands := []types.Command{ - &builder.AddAdditionalEntriesToContext{}, - } - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } - packages := ctx.PackageManager.GetPackages() if runtime.GOOS == "windows" { diff --git a/legacy/builder/test/libraries_loader_test.go b/legacy/builder/test/libraries_loader_test.go index 143c1866221..a18880fed92 100644 --- a/legacy/builder/test/libraries_loader_test.go +++ b/legacy/builder/test/libraries_loader_test.go @@ -48,7 +48,6 @@ func TestLoadLibrariesAVR(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) commands := []types.Command{ - &builder.AddAdditionalEntriesToContext{}, &builder.LibrariesLoader{}, } for _, command := range commands { @@ -154,7 +153,6 @@ func TestLoadLibrariesSAM(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) commands := []types.Command{ - &builder.AddAdditionalEntriesToContext{}, &builder.LibrariesLoader{}, } for _, command := range commands { @@ -233,7 +231,6 @@ func TestLoadLibrariesAVRNoDuplicateLibrariesFolders(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) commands := []types.Command{ - &builder.AddAdditionalEntriesToContext{}, &builder.LibrariesLoader{}, } for _, command := range commands { @@ -258,7 +255,6 @@ func TestLoadLibrariesMyAVRPlatform(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) commands := []types.Command{ - &builder.AddAdditionalEntriesToContext{}, &builder.LibrariesLoader{}, } for _, command := range commands { diff --git a/legacy/builder/test/recipe_runner_test.go b/legacy/builder/test/recipe_runner_test.go index 6fb90bf7e68..5d9a53ef68e 100644 --- a/legacy/builder/test/recipe_runner_test.go +++ b/legacy/builder/test/recipe_runner_test.go @@ -35,7 +35,6 @@ func TestRecipeRunner(t *testing.T) { buildProperties.Set("recipe.hooks.prebuild.1.pattern", "echo") commands := []types.Command{ - &builder.AddAdditionalEntriesToContext{}, &builder.RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"}, } diff --git a/legacy/builder/test/setup_build_properties_test.go b/legacy/builder/test/setup_build_properties_test.go index 1b595d5d69c..d26a217d398 100644 --- a/legacy/builder/test/setup_build_properties_test.go +++ b/legacy/builder/test/setup_build_properties_test.go @@ -34,14 +34,6 @@ func TestSetupBuildProperties(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, paths.New("sketch1", "sketch1.ino"), "arduino:avr:uno") defer cleanUpBuilderTestContext(t, ctx) - commands := []types.Command{ - &builder.AddAdditionalEntriesToContext{}, - } - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } - buildProperties := ctx.BuildProperties require.Equal(t, "ARDUINO", buildProperties.Get("software")) @@ -108,14 +100,6 @@ func TestSetupBuildPropertiesUserHardware(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, paths.New("sketch1", "sketch1.ino"), "my_avr_platform:avr:custom_yun") defer cleanUpBuilderTestContext(t, ctx) - commands := []types.Command{ - &builder.AddAdditionalEntriesToContext{}, - } - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } - buildProperties := ctx.BuildProperties require.Equal(t, "ARDUINO", buildProperties.Get("software")) From b86ce45c87e0a82aec29a2a57940127784971606 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Thu, 31 Aug 2023 19:21:41 +0200 Subject: [PATCH 12/29] Refactor legacy LibrariesLoader command --- legacy/builder/container_setup.go | 29 +++---- legacy/builder/libraries_loader.go | 71 +++++++++-------- legacy/builder/test/libraries_loader_test.go | 82 ++++++++++---------- 3 files changed, 92 insertions(+), 90 deletions(-) diff --git a/legacy/builder/container_setup.go b/legacy/builder/container_setup.go index 8a99df5c8a9..4d2d8a6d22d 100644 --- a/legacy/builder/container_setup.go +++ b/legacy/builder/container_setup.go @@ -33,20 +33,23 @@ func (s *ContainerSetupHardwareToolsLibsSketchAndProps) Run(ctx *types.Context) ctx.CoreBuildPath = coreBuildPath ctx.WarningsLevel = warningsLevel - commands := []types.Command{ - &FailIfBuildPathEqualsSketchPath{}, - &LibrariesLoader{}, + if ctx.BuildPath.Canonical().EqualsTo(ctx.Sketch.FullPath.Canonical()) { + return errors.New(tr("Sketch cannot be located in build path. Please specify a different build path")) } - ctx.Progress.AddSubSteps(len(commands)) - defer ctx.Progress.RemoveSubSteps() - for _, command := range commands { - PrintRingNameIfDebug(ctx, command) - err := command.Run(ctx) - if err != nil { - return errors.WithStack(err) - } - ctx.Progress.CompleteStep() - ctx.PushProgress() + + lm, libsResolver, verboseOut, err := LibrariesLoader( + ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, + ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.ActualPlatform, ctx.TargetPlatform, + ) + if err != nil { + return errors.WithStack(err) + } + + ctx.LibrariesManager = lm + ctx.LibrariesResolver = libsResolver + if ctx.Verbose { + ctx.Warn(string(verboseOut)) } return nil } diff --git a/legacy/builder/libraries_loader.go b/legacy/builder/libraries_loader.go index b41c49edcb1..fcdf7ed89f9 100644 --- a/legacy/builder/libraries_loader.go +++ b/legacy/builder/libraries_loader.go @@ -16,41 +16,48 @@ package builder import ( + "bytes" + + "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/libraries" "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" "github.com/arduino/arduino-cli/arduino/libraries/librariesresolver" - "github.com/arduino/arduino-cli/legacy/builder/types" + "github.com/arduino/go-paths-helper" "github.com/pkg/errors" ) -type LibrariesLoader struct{} - -func (s *LibrariesLoader) Run(ctx *types.Context) error { - if ctx.UseCachedLibrariesResolution { +func LibrariesLoader( + useCachedLibrariesResolution bool, + librariesManager *librariesmanager.LibrariesManager, + builtInLibrariesDirs *paths.Path, libraryDirs, otherLibrariesDirs paths.PathList, + actualPlatform, targetPlatform *cores.PlatformRelease, +) (*librariesmanager.LibrariesManager, *librariesresolver.Cpp, []byte, error) { + verboseOut := &bytes.Buffer{} + lm := librariesManager + if useCachedLibrariesResolution { // Since we are using the cached libraries resolution // the library manager is not needed. - lm := librariesmanager.NewLibraryManager(nil, nil) - ctx.LibrariesManager = lm - } else if ctx.LibrariesManager == nil { - lm := librariesmanager.NewLibraryManager(nil, nil) - ctx.LibrariesManager = lm + lm = librariesmanager.NewLibraryManager(nil, nil) + } + if librariesManager == nil { + lm = librariesmanager.NewLibraryManager(nil, nil) - builtInLibrariesFolders := ctx.BuiltInLibrariesDirs + builtInLibrariesFolders := builtInLibrariesDirs if builtInLibrariesFolders != nil { if err := builtInLibrariesFolders.ToAbs(); err != nil { - return errors.WithStack(err) + return nil, nil, nil, errors.WithStack(err) } lm.AddLibrariesDir(builtInLibrariesFolders, libraries.IDEBuiltIn) } - if ctx.ActualPlatform != ctx.TargetPlatform { - lm.AddPlatformReleaseLibrariesDir(ctx.ActualPlatform, libraries.ReferencedPlatformBuiltIn) + if actualPlatform != targetPlatform { + lm.AddPlatformReleaseLibrariesDir(actualPlatform, libraries.ReferencedPlatformBuiltIn) } - lm.AddPlatformReleaseLibrariesDir(ctx.TargetPlatform, libraries.PlatformBuiltIn) + lm.AddPlatformReleaseLibrariesDir(targetPlatform, libraries.PlatformBuiltIn) - librariesFolders := ctx.OtherLibrariesDirs + librariesFolders := otherLibrariesDirs if err := librariesFolders.ToAbs(); err != nil { - return errors.WithStack(err) + return nil, nil, nil, errors.WithStack(err) } for _, folder := range librariesFolders { lm.AddLibrariesDir(folder, libraries.User) @@ -63,35 +70,31 @@ func (s *LibrariesLoader) Run(ctx *types.Context) error { // I have no intention right now to start a refactoring of the legacy package too, so // here's this shitty solution for now. // When we're gonna refactor the legacy package this will be gone. - if ctx.Verbose { - ctx.Warn(status.Message()) - } + verboseOut.Write([]byte(status.Message())) } - for _, dir := range ctx.LibraryDirs { + for _, dir := range libraryDirs { // Libraries specified this way have top priority if err := lm.LoadLibraryFromDir(dir, libraries.Unmanaged); err != nil { - return err + return nil, nil, nil, errors.WithStack(err) } } } resolver := librariesresolver.NewCppResolver() - if err := resolver.ScanIDEBuiltinLibraries(ctx.LibrariesManager); err != nil { - return errors.WithStack(err) + if err := resolver.ScanIDEBuiltinLibraries(lm); err != nil { + return nil, nil, nil, errors.WithStack(err) } - if err := resolver.ScanUserAndUnmanagedLibraries(ctx.LibrariesManager); err != nil { - return errors.WithStack(err) + if err := resolver.ScanUserAndUnmanagedLibraries(lm); err != nil { + return nil, nil, nil, errors.WithStack(err) } - if err := resolver.ScanPlatformLibraries(ctx.LibrariesManager, ctx.TargetPlatform); err != nil { - return errors.WithStack(err) + if err := resolver.ScanPlatformLibraries(lm, targetPlatform); err != nil { + return nil, nil, nil, errors.WithStack(err) } - if ctx.ActualPlatform != ctx.TargetPlatform { - if err := resolver.ScanPlatformLibraries(ctx.LibrariesManager, ctx.ActualPlatform); err != nil { - return errors.WithStack(err) + if actualPlatform != targetPlatform { + if err := resolver.ScanPlatformLibraries(lm, actualPlatform); err != nil { + return nil, nil, nil, errors.WithStack(err) } } - ctx.LibrariesResolver = resolver - - return nil + return lm, resolver, verboseOut.Bytes(), nil } diff --git a/legacy/builder/test/libraries_loader_test.go b/legacy/builder/test/libraries_loader_test.go index a18880fed92..7b7fcb12750 100644 --- a/legacy/builder/test/libraries_loader_test.go +++ b/legacy/builder/test/libraries_loader_test.go @@ -28,9 +28,9 @@ import ( "github.com/stretchr/testify/require" ) -func extractLibraries(ctx *types.Context) []*libraries.Library { +func extractLibraries(libs map[string]libraries.List) []*libraries.Library { res := []*libraries.Library{} - for _, lib := range ctx.LibrariesManager.Libraries { + for _, lib := range libs { for _, libAlternative := range lib { res = append(res, libAlternative) } @@ -47,21 +47,20 @@ func TestLoadLibrariesAVR(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, nil, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - commands := []types.Command{ - &builder.LibrariesLoader{}, - } - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } + lm, libsResolver, _, err := builder.LibrariesLoader( + ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, + ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.ActualPlatform, ctx.TargetPlatform, + ) + NoError(t, err) - librariesFolders := ctx.LibrariesManager.LibrariesDir + librariesFolders := lm.LibrariesDir require.Equal(t, 3, len(librariesFolders)) require.True(t, Abs(t, paths.New("downloaded_libraries")).EquivalentTo(librariesFolders[0].Path)) require.True(t, Abs(t, paths.New("downloaded_hardware", "arduino", "avr", "libraries")).EquivalentTo(librariesFolders[1].Path)) require.True(t, Abs(t, paths.New("libraries")).EquivalentTo(librariesFolders[2].Path)) - libs := extractLibraries(ctx) + libs := extractLibraries(lm.Libraries) require.Equal(t, 24, len(libs)) sort.Sort(ByLibraryName(libs)) @@ -124,21 +123,21 @@ func TestLoadLibrariesAVR(t *testing.T) { idx++ require.Equal(t, "Wire", libs[idx].Name) - libs = ctx.LibrariesResolver.AlternativesFor("Audio.h") + libs = libsResolver.AlternativesFor("Audio.h") require.Len(t, libs, 2) sort.Sort(ByLibraryName(libs)) require.Equal(t, "Audio", libs[0].Name) require.Equal(t, "FakeAudio", libs[1].Name) - libs = ctx.LibrariesResolver.AlternativesFor("FakeAudio.h") + libs = libsResolver.AlternativesFor("FakeAudio.h") require.Len(t, libs, 1) require.Equal(t, "FakeAudio", libs[0].Name) - libs = ctx.LibrariesResolver.AlternativesFor("Adafruit_PN532.h") + libs = libsResolver.AlternativesFor("Adafruit_PN532.h") require.Len(t, libs, 1) require.Equal(t, "Adafruit PN532", libs[0].Name) - libs = ctx.LibrariesResolver.AlternativesFor("IRremote.h") + libs = libsResolver.AlternativesFor("IRremote.h") require.Len(t, libs, 1) require.Equal(t, "IRremote", libs[0].Name) } @@ -152,21 +151,20 @@ func TestLoadLibrariesSAM(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, nil, "arduino:sam:arduino_due_x_dbg") defer cleanUpBuilderTestContext(t, ctx) - commands := []types.Command{ - &builder.LibrariesLoader{}, - } - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } + lm, libsResolver, _, err := builder.LibrariesLoader( + ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, + ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.ActualPlatform, ctx.TargetPlatform, + ) + NoError(t, err) - librariesFolders := ctx.LibrariesManager.LibrariesDir + librariesFolders := lm.LibrariesDir require.Equal(t, 3, len(librariesFolders)) require.True(t, Abs(t, paths.New("downloaded_libraries")).EquivalentTo(librariesFolders[0].Path)) require.True(t, Abs(t, paths.New("downloaded_hardware", "arduino", "sam", "libraries")).EquivalentTo(librariesFolders[1].Path)) require.True(t, Abs(t, paths.New("libraries")).EquivalentTo(librariesFolders[2].Path)) - libraries := extractLibraries(ctx) + libraries := extractLibraries(lm.Libraries) require.Equal(t, 22, len(libraries)) sort.Sort(ByLibraryName(libraries)) @@ -206,17 +204,17 @@ func TestLoadLibrariesSAM(t *testing.T) { idx++ require.Equal(t, "Wire", libraries[idx].Name) - libs := ctx.LibrariesResolver.AlternativesFor("Audio.h") + libs := libsResolver.AlternativesFor("Audio.h") require.Len(t, libs, 2) sort.Sort(ByLibraryName(libs)) require.Equal(t, "Audio", libs[0].Name) require.Equal(t, "FakeAudio", libs[1].Name) - libs = ctx.LibrariesResolver.AlternativesFor("FakeAudio.h") + libs = libsResolver.AlternativesFor("FakeAudio.h") require.Len(t, libs, 1) require.Equal(t, "FakeAudio", libs[0].Name) - libs = ctx.LibrariesResolver.AlternativesFor("IRremote.h") + libs = libsResolver.AlternativesFor("IRremote.h") require.Len(t, libs, 1) require.Equal(t, "IRremote", libs[0].Name) } @@ -230,15 +228,14 @@ func TestLoadLibrariesAVRNoDuplicateLibrariesFolders(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, nil, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - commands := []types.Command{ - &builder.LibrariesLoader{}, - } - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } + lm, _, _, err := builder.LibrariesLoader( + ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, + ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.ActualPlatform, ctx.TargetPlatform, + ) + NoError(t, err) - librariesFolders := ctx.LibrariesManager.LibrariesDir + librariesFolders := lm.LibrariesDir require.Equal(t, 3, len(librariesFolders)) require.True(t, Abs(t, paths.New("downloaded_libraries")).EquivalentTo(librariesFolders[0].Path)) require.True(t, Abs(t, paths.New("downloaded_hardware", "arduino", "avr", "libraries")).EquivalentTo(librariesFolders[1].Path)) @@ -254,15 +251,14 @@ func TestLoadLibrariesMyAVRPlatform(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, nil, "my_avr_platform:avr:custom_yun") defer cleanUpBuilderTestContext(t, ctx) - commands := []types.Command{ - &builder.LibrariesLoader{}, - } - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } + lm, _, _, err := builder.LibrariesLoader( + ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, + ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.ActualPlatform, ctx.TargetPlatform, + ) + NoError(t, err) - librariesFolders := ctx.LibrariesManager.LibrariesDir + librariesFolders := lm.LibrariesDir require.Equal(t, 4, len(librariesFolders)) require.True(t, Abs(t, paths.New("downloaded_libraries")).EquivalentTo(librariesFolders[0].Path)) require.True(t, Abs(t, paths.New("downloaded_hardware", "arduino", "avr", "libraries")).EquivalentTo(librariesFolders[1].Path)) From 341ffe295219f4b7c59f0701900eca508131cf98 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Thu, 31 Aug 2023 19:34:28 +0200 Subject: [PATCH 13/29] move LibrariesLoader under arduino/builder --- .../libraries_loader.go => arduino/builder/libraries.go | 0 legacy/builder/container_setup.go | 3 ++- legacy/builder/test/libraries_loader_test.go | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) rename legacy/builder/libraries_loader.go => arduino/builder/libraries.go (100%) diff --git a/legacy/builder/libraries_loader.go b/arduino/builder/libraries.go similarity index 100% rename from legacy/builder/libraries_loader.go rename to arduino/builder/libraries.go diff --git a/legacy/builder/container_setup.go b/legacy/builder/container_setup.go index 4d2d8a6d22d..a0ee98d86f6 100644 --- a/legacy/builder/container_setup.go +++ b/legacy/builder/container_setup.go @@ -16,6 +16,7 @@ package builder import ( + "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/pkg/errors" ) @@ -37,7 +38,7 @@ func (s *ContainerSetupHardwareToolsLibsSketchAndProps) Run(ctx *types.Context) return errors.New(tr("Sketch cannot be located in build path. Please specify a different build path")) } - lm, libsResolver, verboseOut, err := LibrariesLoader( + lm, libsResolver, verboseOut, err := builder.LibrariesLoader( ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, diff --git a/legacy/builder/test/libraries_loader_test.go b/legacy/builder/test/libraries_loader_test.go index 7b7fcb12750..8ef2c629a24 100644 --- a/legacy/builder/test/libraries_loader_test.go +++ b/legacy/builder/test/libraries_loader_test.go @@ -21,7 +21,7 @@ import ( "testing" "github.com/arduino/arduino-cli/arduino/libraries" - "github.com/arduino/arduino-cli/legacy/builder" + "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" From 96426b9df992684f1967156dd98be682ec411337 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Fri, 1 Sep 2023 15:50:04 +0200 Subject: [PATCH 14/29] remove usless nil check --- .../add_additional_entries_to_context.go | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/legacy/builder/add_additional_entries_to_context.go b/legacy/builder/add_additional_entries_to_context.go index 843cbdcd79e..0f3a2fb3f57 100644 --- a/legacy/builder/add_additional_entries_to_context.go +++ b/legacy/builder/add_additional_entries_to_context.go @@ -22,22 +22,17 @@ import ( ) func AddAdditionalEntriesToContext(buildPath *paths.Path, warningLevel string) (*paths.Path, *paths.Path, *paths.Path, string, error) { - var sketchBuildPath, librariesBuildPath, coreBuildPath *paths.Path - var err error - - if buildPath != nil { - sketchBuildPath, err = buildPath.Join(constants.FOLDER_SKETCH).Abs() - if err != nil { - return nil, nil, nil, "", errors.WithStack(err) - } - librariesBuildPath, err = buildPath.Join(constants.FOLDER_LIBRARIES).Abs() - if err != nil { - return nil, nil, nil, "", errors.WithStack(err) - } - coreBuildPath, err = buildPath.Join(constants.FOLDER_CORE).Abs() - if err != nil { - return nil, nil, nil, "", errors.WithStack(err) - } + sketchBuildPath, err := buildPath.Join(constants.FOLDER_SKETCH).Abs() + if err != nil { + return nil, nil, nil, "", errors.WithStack(err) + } + librariesBuildPath, err := buildPath.Join(constants.FOLDER_LIBRARIES).Abs() + if err != nil { + return nil, nil, nil, "", errors.WithStack(err) + } + coreBuildPath, err := buildPath.Join(constants.FOLDER_CORE).Abs() + if err != nil { + return nil, nil, nil, "", errors.WithStack(err) } if warningLevel == "" { From 298ebb5b837d52ce1480d4f935b16221036257a9 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Fri, 1 Sep 2023 16:02:32 +0200 Subject: [PATCH 15/29] remove AddAdditionalEntries func, in favour of initializing them in the compile command --- commands/compile/compile.go | 25 +++++++++ .../add_additional_entries_to_context.go | 43 --------------- legacy/builder/container_setup.go | 10 ---- .../add_additional_entries_to_context_test.go | 55 ------------------- legacy/builder/test/builder_test.go | 12 ++++ 5 files changed, 37 insertions(+), 108 deletions(-) delete mode 100644 legacy/builder/add_additional_entries_to_context.go delete mode 100644 legacy/builder/test/add_additional_entries_to_context_test.go diff --git a/commands/compile/compile.go b/commands/compile/compile.go index b374213f2d3..4a652b8dd6e 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -18,6 +18,7 @@ package compile import ( "context" "fmt" + "go/build" "io" "sort" "strings" @@ -33,6 +34,7 @@ import ( "github.com/arduino/arduino-cli/i18n" "github.com/arduino/arduino-cli/internal/inventory" "github.com/arduino/arduino-cli/legacy/builder" + "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" paths "github.com/arduino/go-paths-helper" @@ -189,6 +191,25 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx.BuildPath = buildPath builderCtx.ProgressCB = progressCB + sketchBuildPath, err := buildPath.Join(constants.FOLDER_SKETCH).Abs() + if err != nil { + return nil, err + } + builderCtx.SketchBuildPath = sketchBuildPath + + librariesBuildPath, err := buildPath.Join(constants.FOLDER_LIBRARIES).Abs() + if err != nil { + return nil, err + } + builderCtx.LibrariesBuildPath = librariesBuildPath + + coreBuildPath, err := buildPath.Join(constants.FOLDER_CORE).Abs() + if err != nil { + return nil, err + } + builderCtx.CoreBuildPath = coreBuildPath + //ctx.WarningsLevel = warningsLevel + // FIXME: This will be redundant when arduino-builder will be part of the cli builderCtx.HardwareDirs = configuration.HardwareDirectories(configuration.Settings) builderCtx.BuiltInToolsDirs = configuration.BuiltinToolsDirectories(configuration.Settings) @@ -202,7 +223,11 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx.Verbose = req.GetVerbose() builderCtx.Jobs = int(req.GetJobs()) + builderCtx.WarningsLevel = req.GetWarnings() + if builderCtx.WarningsLevel == "" { + builderCtx.WarningsLevel = builder.DEFAULT_WARNINGS_LEVEL + } if req.GetBuildCachePath() == "" { builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "cores") diff --git a/legacy/builder/add_additional_entries_to_context.go b/legacy/builder/add_additional_entries_to_context.go deleted file mode 100644 index 0f3a2fb3f57..00000000000 --- a/legacy/builder/add_additional_entries_to_context.go +++ /dev/null @@ -1,43 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "github.com/arduino/arduino-cli/legacy/builder/constants" - "github.com/arduino/go-paths-helper" - "github.com/pkg/errors" -) - -func AddAdditionalEntriesToContext(buildPath *paths.Path, warningLevel string) (*paths.Path, *paths.Path, *paths.Path, string, error) { - sketchBuildPath, err := buildPath.Join(constants.FOLDER_SKETCH).Abs() - if err != nil { - return nil, nil, nil, "", errors.WithStack(err) - } - librariesBuildPath, err := buildPath.Join(constants.FOLDER_LIBRARIES).Abs() - if err != nil { - return nil, nil, nil, "", errors.WithStack(err) - } - coreBuildPath, err := buildPath.Join(constants.FOLDER_CORE).Abs() - if err != nil { - return nil, nil, nil, "", errors.WithStack(err) - } - - if warningLevel == "" { - warningLevel = DEFAULT_WARNINGS_LEVEL - } - - return sketchBuildPath, librariesBuildPath, coreBuildPath, warningLevel, nil -} diff --git a/legacy/builder/container_setup.go b/legacy/builder/container_setup.go index a0ee98d86f6..0a3448352e3 100644 --- a/legacy/builder/container_setup.go +++ b/legacy/builder/container_setup.go @@ -24,16 +24,6 @@ import ( type ContainerSetupHardwareToolsLibsSketchAndProps struct{} func (s *ContainerSetupHardwareToolsLibsSketchAndProps) Run(ctx *types.Context) error { - sketchBuildPath, librariesBuildPath, coreBuildPath, - warningsLevel, err := AddAdditionalEntriesToContext(ctx.BuildPath, ctx.WarningsLevel) - if err != nil { - return errors.WithStack(err) - } - ctx.SketchBuildPath = sketchBuildPath - ctx.LibrariesBuildPath = librariesBuildPath - ctx.CoreBuildPath = coreBuildPath - ctx.WarningsLevel = warningsLevel - if ctx.BuildPath.Canonical().EqualsTo(ctx.Sketch.FullPath.Canonical()) { return errors.New(tr("Sketch cannot be located in build path. Please specify a different build path")) } diff --git a/legacy/builder/test/add_additional_entries_to_context_test.go b/legacy/builder/test/add_additional_entries_to_context_test.go deleted file mode 100644 index dc15e16697c..00000000000 --- a/legacy/builder/test/add_additional_entries_to_context_test.go +++ /dev/null @@ -1,55 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package test - -import ( - "testing" - - "github.com/arduino/arduino-cli/legacy/builder" - "github.com/arduino/arduino-cli/legacy/builder/constants" - "github.com/arduino/arduino-cli/legacy/builder/types" - paths "github.com/arduino/go-paths-helper" - "github.com/stretchr/testify/require" -) - -func TestAddAdditionalEntriesToContextNoBuildPath(t *testing.T) { - ctx := &types.Context{} - - sketchBuildPath, librariesBuildPath, coreBuildPath, - warningsLevel, err := builder.AddAdditionalEntriesToContext(ctx.BuildPath, ctx.WarningsLevel) - NoError(t, err) - - require.Empty(t, sketchBuildPath) - require.Empty(t, librariesBuildPath) - require.Empty(t, coreBuildPath) - - require.NotNil(t, warningsLevel) -} - -func TestAddAdditionalEntriesToContextWithBuildPath(t *testing.T) { - ctx := &types.Context{} - ctx.BuildPath = paths.New("folder") - - sketchBuildPath, librariesBuildPath, coreBuildPath, - warningsLevel, err := builder.AddAdditionalEntriesToContext(ctx.BuildPath, ctx.WarningsLevel) - NoError(t, err) - - require.Equal(t, Abs(t, paths.New("folder", constants.FOLDER_SKETCH)), sketchBuildPath) - require.Equal(t, Abs(t, paths.New("folder", "libraries")), librariesBuildPath) - require.Equal(t, Abs(t, paths.New("folder", constants.FOLDER_CORE)), coreBuildPath) - - require.NotNil(t, warningsLevel) -} diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index 5ec00979cfb..3e33cba8b7f 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -57,6 +57,18 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat ctx.BuildPath = buildPath } + buildPath := ctx.BuildPath + sketchBuildPath, err := buildPath.Join(constants.FOLDER_SKETCH).Abs() + NoError(t, err) + librariesBuildPath, err := buildPath.Join(constants.FOLDER_LIBRARIES).Abs() + NoError(t, err) + coreBuildPath, err := buildPath.Join(constants.FOLDER_CORE).Abs() + NoError(t, err) + + ctx.SketchBuildPath = sketchBuildPath + ctx.LibrariesBuildPath = librariesBuildPath + ctx.CoreBuildPath = coreBuildPath + // Create a Package Manager from the given context // This should happen only on legacy arduino-builder. // Hopefully this piece will be removed once the legacy package will be cleanedup. From ea05e2cd764ecd4b95196025605c0e177dff3cb6 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Fri, 1 Sep 2023 16:06:52 +0200 Subject: [PATCH 16/29] move a check directly in the compile command --- commands/compile/compile.go | 7 +++++-- legacy/builder/container_setup.go | 4 ---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 4a652b8dd6e..b860b1bd9dc 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -17,8 +17,8 @@ package compile import ( "context" + "errors" "fmt" - "go/build" "io" "sort" "strings" @@ -208,7 +208,10 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream return nil, err } builderCtx.CoreBuildPath = coreBuildPath - //ctx.WarningsLevel = warningsLevel + + if builderCtx.BuildPath.Canonical().EqualsTo(builderCtx.Sketch.FullPath.Canonical()) { + return nil, errors.New(tr("Sketch cannot be located in build path. Please specify a different build path")) + } // FIXME: This will be redundant when arduino-builder will be part of the cli builderCtx.HardwareDirs = configuration.HardwareDirectories(configuration.Settings) diff --git a/legacy/builder/container_setup.go b/legacy/builder/container_setup.go index 0a3448352e3..03c9cce88d4 100644 --- a/legacy/builder/container_setup.go +++ b/legacy/builder/container_setup.go @@ -24,10 +24,6 @@ import ( type ContainerSetupHardwareToolsLibsSketchAndProps struct{} func (s *ContainerSetupHardwareToolsLibsSketchAndProps) Run(ctx *types.Context) error { - if ctx.BuildPath.Canonical().EqualsTo(ctx.Sketch.FullPath.Canonical()) { - return errors.New(tr("Sketch cannot be located in build path. Please specify a different build path")) - } - lm, libsResolver, verboseOut, err := builder.LibrariesLoader( ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, From 45ee0825a11877f61366faed6821aef8f1146bac Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Fri, 1 Sep 2023 16:23:54 +0200 Subject: [PATCH 17/29] create the SketchLibrariesDetector struct --- arduino/builder/libraries.go | 92 +++++++++++++++++++++++++++++++++ legacy/builder/types/context.go | 3 +- 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/arduino/builder/libraries.go b/arduino/builder/libraries.go index fcdf7ed89f9..ffee5539ec4 100644 --- a/arduino/builder/libraries.go +++ b/arduino/builder/libraries.go @@ -17,6 +17,7 @@ package builder import ( "bytes" + "fmt" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/libraries" @@ -26,6 +27,97 @@ import ( "github.com/pkg/errors" ) +type libraryResolutionResult struct { + Library *libraries.Library + NotUsedLibraries []*libraries.Library +} + +// SketchLibrariesDetector todo +type SketchLibrariesDetector struct { + librariesManager *librariesmanager.LibrariesManager + librariesResolver *librariesresolver.Cpp + useCachedLibrariesResolution bool + verbose bool + verboseInfoFn func(msg string) + verboseWarnFn func(msg string) + + importedLibraries libraries.List + librariesResolutionResults map[string]libraryResolutionResult +} + +// NewSketchLibrariesDetector todo +func NewSketchLibrariesDetector( + lm *librariesmanager.LibrariesManager, + libsResolver *librariesresolver.Cpp, + importedLibraries libraries.List, + verbose, useCachedLibrariesResolution bool, + verboseInfoFn func(msg string), + verboseWarnFn func(msg string), +) *SketchLibrariesDetector { + + return &SketchLibrariesDetector{ + librariesManager: lm, + librariesResolver: libsResolver, + useCachedLibrariesResolution: useCachedLibrariesResolution, + librariesResolutionResults: map[string]libraryResolutionResult{}, + verbose: verbose, + verboseInfoFn: verboseInfoFn, + verboseWarnFn: verboseWarnFn, + + importedLibraries: importedLibraries, + } +} + +// ResolveLibrary todo +func (l *SketchLibrariesDetector) ResolveLibrary(header, platformArch string) *libraries.Library { + importedLibraries := l.importedLibraries + candidates := l.librariesResolver.AlternativesFor(header) + + if l.verbose { + l.verboseInfoFn(tr("Alternatives for %[1]s: %[2]s", header, candidates)) + l.verboseInfoFn(fmt.Sprintf("ResolveLibrary(%s)", header)) + l.verboseInfoFn(fmt.Sprintf(" -> %s: %s", tr("candidates"), candidates)) + } + + if len(candidates) == 0 { + return nil + } + + for _, candidate := range candidates { + if importedLibraries.Contains(candidate) { + return nil + } + } + + selected := l.librariesResolver.ResolveFor(header, platformArch) + if alreadyImported := importedLibraries.FindByName(selected.Name); alreadyImported != nil { + // Certain libraries might have the same name but be different. + // This usually happens when the user includes two or more custom libraries that have + // different header name but are stored in a parent folder with identical name, like + // ./libraries1/Lib/lib1.h and ./libraries2/Lib/lib2.h + // Without this check the library resolution would be stuck in a loop. + // This behaviour has been reported in this issue: + // https://github.com/arduino/arduino-cli/issues/973 + if selected == alreadyImported { + selected = alreadyImported + } + } + + candidates.Remove(selected) + l.librariesResolutionResults[header] = libraryResolutionResult{ + Library: selected, + NotUsedLibraries: candidates, + } + + return selected +} + +func (l *SketchLibrariesDetector) ImportedLibraries() libraries.List { + // TODO understand if we have to do a deepcopy + return l.importedLibraries +} + +// LibrariesLoader todo func LibrariesLoader( useCachedLibrariesResolution bool, librariesManager *librariesmanager.LibrariesManager, diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 0c887ee6125..9e022e315e9 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -64,7 +64,8 @@ func (p *ProgressStruct) CompleteStep() { // Context structure type Context struct { - Builder *builder.Builder + Builder *builder.Builder + SketchLibrariesDetector *builder.SketchLibrariesDetector // Build options HardwareDirs paths.PathList From 1e902cf544c081f9a69863d17d71b0760c276ace Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Fri, 1 Sep 2023 16:56:00 +0200 Subject: [PATCH 18/29] move all the logic of ContainerSetupHardwareToolsLibsSketchAndProps in the compile command --- commands/compile/compile.go | 86 +++++++++++++++++++------------ legacy/builder/builder.go | 11 ---- legacy/builder/container_setup.go | 16 ------ 3 files changed, 53 insertions(+), 60 deletions(-) diff --git a/commands/compile/compile.go b/commands/compile/compile.go index b860b1bd9dc..2ca455946c0 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -17,7 +17,6 @@ package compile import ( "context" - "errors" "fmt" "io" "sort" @@ -175,9 +174,6 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx := &types.Context{} builderCtx.Builder = sketchBuilder builderCtx.PackageManager = pme - if pme.GetProfile() != nil { - builderCtx.LibrariesManager = lm - } builderCtx.TargetBoard = targetBoard builderCtx.TargetPlatform = targetPlatform builderCtx.TargetPackage = targetPackage @@ -185,34 +181,11 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx.RequiredTools = requiredTools builderCtx.BuildProperties = buildProperties builderCtx.CustomBuildProperties = customBuildPropertiesArgs - builderCtx.UseCachedLibrariesResolution = req.GetSkipLibrariesDiscovery() builderCtx.FQBN = fqbn builderCtx.Sketch = sk builderCtx.BuildPath = buildPath builderCtx.ProgressCB = progressCB - sketchBuildPath, err := buildPath.Join(constants.FOLDER_SKETCH).Abs() - if err != nil { - return nil, err - } - builderCtx.SketchBuildPath = sketchBuildPath - - librariesBuildPath, err := buildPath.Join(constants.FOLDER_LIBRARIES).Abs() - if err != nil { - return nil, err - } - builderCtx.LibrariesBuildPath = librariesBuildPath - - coreBuildPath, err := buildPath.Join(constants.FOLDER_CORE).Abs() - if err != nil { - return nil, err - } - builderCtx.CoreBuildPath = coreBuildPath - - if builderCtx.BuildPath.Canonical().EqualsTo(builderCtx.Sketch.FullPath.Canonical()) { - return nil, errors.New(tr("Sketch cannot be located in build path. Please specify a different build path")) - } - // FIXME: This will be redundant when arduino-builder will be part of the cli builderCtx.HardwareDirs = configuration.HardwareDirectories(configuration.Settings) builderCtx.BuiltInToolsDirs = configuration.BuiltinToolsDirectories(configuration.Settings) @@ -253,6 +226,57 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx.OnlyUpdateCompilationDatabase = req.GetCreateCompilationDatabaseOnly() builderCtx.SourceOverride = req.GetSourceOverride() + sketchBuildPath, err := buildPath.Join(constants.FOLDER_SKETCH).Abs() + if err != nil { + return r, &arduino.CompileFailedError{Message: err.Error()} + } + librariesBuildPath, err := buildPath.Join(constants.FOLDER_LIBRARIES).Abs() + if err != nil { + return r, &arduino.CompileFailedError{Message: err.Error()} + } + coreBuildPath, err := buildPath.Join(constants.FOLDER_CORE).Abs() + if err != nil { + return r, &arduino.CompileFailedError{Message: err.Error()} + } + builderCtx.SketchBuildPath = sketchBuildPath + builderCtx.LibrariesBuildPath = librariesBuildPath + builderCtx.CoreBuildPath = coreBuildPath + + if builderCtx.BuildPath.Canonical().EqualsTo(builderCtx.Sketch.FullPath.Canonical()) { + return r, &arduino.CompileFailedError{ + Message: tr("Sketch cannot be located in build path. Please specify a different build path"), + } + } + + // TODO replace all UseCache call with our SketchLibrariesDetector + builderCtx.UseCachedLibrariesResolution = req.GetSkipLibrariesDiscovery() + if pme.GetProfile() != nil { + builderCtx.LibrariesManager = lm + } + lm, libsResolver, verboseOut, err := bldr.LibrariesLoader( + builderCtx.UseCachedLibrariesResolution, builderCtx.LibrariesManager, + builderCtx.BuiltInLibrariesDirs, builderCtx.LibraryDirs, builderCtx.OtherLibrariesDirs, + builderCtx.ActualPlatform, builderCtx.TargetPlatform, + ) + if err != nil { + return r, &arduino.CompileFailedError{Message: err.Error()} + } + + builderCtx.LibrariesManager = lm + builderCtx.LibrariesResolver = libsResolver + if builderCtx.Verbose { + builderCtx.Warn(string(verboseOut)) + } + + builderCtx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( + lm, libsResolver, + builderCtx.ImportedLibraries, + builderCtx.Verbose, + builderCtx.UseCachedLibrariesResolution, + func(msg string) { builderCtx.Info(msg) }, + func(msg string) { builderCtx.Warn(msg) }, + ) + defer func() { if p := builderCtx.BuildPath; p != nil { r.BuildPath = p.String() @@ -274,13 +298,9 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream } }() + // Just get build properties and exit if req.GetShowProperties() { - // Just get build properties and exit - compileErr := builder.RunParseHardware(builderCtx) - if compileErr != nil { - compileErr = &arduino.CompileFailedError{Message: compileErr.Error()} - } - return r, compileErr + return r, nil } if req.GetPreprocess() { diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index bce8bd24d6b..f914e11140f 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -42,8 +42,6 @@ func (s *Builder) Run(ctx *types.Context) error { var _err, mainErr error commands := []types.Command{ - &ContainerSetupHardwareToolsLibsSketchAndProps{}, - &ContainerBuildOptions{}, &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"}, @@ -162,8 +160,6 @@ func (s *Preprocess) Run(ctx *types.Context) error { var _err error commands := []types.Command{ - &ContainerSetupHardwareToolsLibsSketchAndProps{}, - &ContainerBuildOptions{}, &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"}, @@ -217,13 +213,6 @@ func RunBuilder(ctx *types.Context) error { return runCommands(ctx, []types.Command{&Builder{}}) } -func RunParseHardware(ctx *types.Context) error { - commands := []types.Command{ - &ContainerSetupHardwareToolsLibsSketchAndProps{}, - } - return runCommands(ctx, commands) -} - func RunPreprocess(ctx *types.Context) error { command := Preprocess{} return command.Run(ctx) diff --git a/legacy/builder/container_setup.go b/legacy/builder/container_setup.go index 03c9cce88d4..3179ef6c1b2 100644 --- a/legacy/builder/container_setup.go +++ b/legacy/builder/container_setup.go @@ -16,27 +16,11 @@ package builder import ( - "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/pkg/errors" ) type ContainerSetupHardwareToolsLibsSketchAndProps struct{} func (s *ContainerSetupHardwareToolsLibsSketchAndProps) Run(ctx *types.Context) error { - lm, libsResolver, verboseOut, err := builder.LibrariesLoader( - ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, - ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, - ctx.ActualPlatform, ctx.TargetPlatform, - ) - if err != nil { - return errors.WithStack(err) - } - - ctx.LibrariesManager = lm - ctx.LibrariesResolver = libsResolver - if ctx.Verbose { - ctx.Warn(string(verboseOut)) - } return nil } From 1c18baf4981d8e7b4cbc3313054ed3662b378dab Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Fri, 1 Sep 2023 18:02:27 +0200 Subject: [PATCH 19/29] remove container_setup and adjust relative tests --- legacy/builder/container_setup.go | 26 -------------- legacy/builder/test/builder_test.go | 35 ++++++++++++++++++- legacy/builder/test/hardware_loader_test.go | 9 ++--- .../test/merge_sketch_with_bootloader_test.go | 12 ------- .../test/setup_build_properties_test.go | 10 ------ legacy/builder/test/tools_loader_test.go | 6 ++-- 6 files changed, 42 insertions(+), 56 deletions(-) delete mode 100644 legacy/builder/container_setup.go diff --git a/legacy/builder/container_setup.go b/legacy/builder/container_setup.go deleted file mode 100644 index 3179ef6c1b2..00000000000 --- a/legacy/builder/container_setup.go +++ /dev/null @@ -1,26 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "github.com/arduino/arduino-cli/legacy/builder/types" -) - -type ContainerSetupHardwareToolsLibsSketchAndProps struct{} - -func (s *ContainerSetupHardwareToolsLibsSketchAndProps) Run(ctx *types.Context) error { - return nil -} diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index 3e33cba8b7f..084d885564c 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -39,9 +39,18 @@ func cleanUpBuilderTestContext(t *testing.T, ctx *types.Context) { } } -func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *paths.Path, fqbn string) *types.Context { +type skipContextPreparationStepName string + +const skipLibraries = skipContextPreparationStepName("libraries") + +func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *paths.Path, fqbn string, skips ...skipContextPreparationStepName) *types.Context { DownloadCoresAndToolsAndLibraries(t) + stepToSkip := map[skipContextPreparationStepName]bool{} + for _, skip := range skips { + stepToSkip[skip] = true + } + if ctx == nil { ctx = &types.Context{} } @@ -111,6 +120,30 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat ctx.RequiredTools = requiredTools } + if ctx.Sketch != nil { + require.False(t, ctx.BuildPath.Canonical().EqualsTo(ctx.Sketch.FullPath.Canonical())) + } + + if !stepToSkip[skipLibraries] { + lm, libsResolver, _, err := bldr.LibrariesLoader( + ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, + ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.ActualPlatform, ctx.TargetPlatform, + ) + NoError(t, err) + + ctx.LibrariesManager = lm + ctx.LibrariesResolver = libsResolver + ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( + lm, libsResolver, + ctx.ImportedLibraries, + ctx.Verbose, + ctx.UseCachedLibrariesResolution, + func(msg string) { ctx.Info(msg) }, + func(msg string) { ctx.Warn(msg) }, + ) + } + return ctx } diff --git a/legacy/builder/test/hardware_loader_test.go b/legacy/builder/test/hardware_loader_test.go index c7e4b0ba387..ae87e1c41c7 100644 --- a/legacy/builder/test/hardware_loader_test.go +++ b/legacy/builder/test/hardware_loader_test.go @@ -33,7 +33,8 @@ func TestLoadHardware(t *testing.T) { ctx := &types.Context{ HardwareDirs: paths.NewPathList("downloaded_hardware", filepath.Join("..", "hardware")), } - ctx = prepareBuilderTestContext(t, ctx, nil, "") + + ctx = prepareBuilderTestContext(t, ctx, nil, "", skipLibraries) defer cleanUpBuilderTestContext(t, ctx) packages := ctx.PackageManager.GetPackages() @@ -65,7 +66,7 @@ func TestLoadHardwareMixingUserHardwareFolder(t *testing.T) { ctx := &types.Context{ HardwareDirs: paths.NewPathList("downloaded_hardware", filepath.Join("..", "hardware"), "user_hardware"), } - ctx = prepareBuilderTestContext(t, ctx, nil, "") + ctx = prepareBuilderTestContext(t, ctx, nil, "", skipLibraries) defer cleanUpBuilderTestContext(t, ctx) packages := ctx.PackageManager.GetPackages() @@ -125,7 +126,7 @@ func TestLoadHardwareWithBoardManagerFolderStructure(t *testing.T) { ctx := &types.Context{ HardwareDirs: paths.NewPathList("downloaded_board_manager_stuff"), } - ctx = prepareBuilderTestContext(t, ctx, nil, "") + ctx = prepareBuilderTestContext(t, ctx, nil, "", skipLibraries) defer cleanUpBuilderTestContext(t, ctx) packages := ctx.PackageManager.GetPackages() @@ -166,7 +167,7 @@ func TestLoadLotsOfHardware(t *testing.T) { ctx := &types.Context{ HardwareDirs: paths.NewPathList("downloaded_board_manager_stuff", "downloaded_hardware", filepath.Join("..", "hardware"), "user_hardware"), } - ctx = prepareBuilderTestContext(t, ctx, nil, "") + ctx = prepareBuilderTestContext(t, ctx, nil, "", skipLibraries) defer cleanUpBuilderTestContext(t, ctx) packages := ctx.PackageManager.GetPackages() diff --git a/legacy/builder/test/merge_sketch_with_bootloader_test.go b/legacy/builder/test/merge_sketch_with_bootloader_test.go index b36102f6c40..43387eeaa8d 100644 --- a/legacy/builder/test/merge_sketch_with_bootloader_test.go +++ b/legacy/builder/test/merge_sketch_with_bootloader_test.go @@ -70,7 +70,6 @@ func TestMergeSketchWithBootloader(t *testing.T) { NoError(t, err) commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.MergeSketchWithBootloader{}, } @@ -129,7 +128,6 @@ func TestMergeSketchWithBootloaderSketchInBuildPath(t *testing.T) { NoError(t, err) commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.MergeSketchWithBootloader{}, } @@ -152,15 +150,6 @@ func TestMergeSketchWithBootloaderWhenNoBootloaderAvailable(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) buildPath := ctx.BuildPath - commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - } - - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } - buildProperties := ctx.BuildProperties buildProperties.Remove(constants.BUILD_PROPERTIES_BOOTLOADER_NOBLINK) buildProperties.Remove(constants.BUILD_PROPERTIES_BOOTLOADER_FILE) @@ -222,7 +211,6 @@ func TestMergeSketchWithBootloaderPathIsParameterized(t *testing.T) { NoError(t, err) commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.MergeSketchWithBootloader{}, } diff --git a/legacy/builder/test/setup_build_properties_test.go b/legacy/builder/test/setup_build_properties_test.go index d26a217d398..2c0344812a2 100644 --- a/legacy/builder/test/setup_build_properties_test.go +++ b/legacy/builder/test/setup_build_properties_test.go @@ -19,7 +19,6 @@ import ( "path/filepath" "testing" - "github.com/arduino/arduino-cli/legacy/builder" "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" @@ -118,15 +117,6 @@ func TestSetupBuildPropertiesWithMissingPropsFromParentPlatformTxtFiles(t *testi ctx = prepareBuilderTestContext(t, ctx, paths.New("sketch1", "sketch1.ino"), "my_avr_platform:avr:custom_yun") defer cleanUpBuilderTestContext(t, ctx) - commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - } - - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } - buildProperties := ctx.BuildProperties require.Equal(t, "ARDUINO", buildProperties.Get("software")) diff --git a/legacy/builder/test/tools_loader_test.go b/legacy/builder/test/tools_loader_test.go index 7138df09afc..e28f828401d 100644 --- a/legacy/builder/test/tools_loader_test.go +++ b/legacy/builder/test/tools_loader_test.go @@ -66,7 +66,7 @@ func TestLoadTools(t *testing.T) { HardwareDirs: paths.NewPathList(filepath.Join("..", "hardware"), "downloaded_hardware"), BuiltInToolsDirs: paths.NewPathList("downloaded_tools", "tools_builtin"), } - ctx = prepareBuilderTestContext(t, ctx, nil, "") + ctx = prepareBuilderTestContext(t, ctx, nil, "", skipLibraries) defer cleanUpBuilderTestContext(t, ctx) tools := ctx.PackageManager.GetAllInstalledToolsReleases() @@ -107,7 +107,7 @@ func TestLoadToolsWithBoardManagerFolderStructure(t *testing.T) { ctx := &types.Context{ HardwareDirs: paths.NewPathList("downloaded_board_manager_stuff"), } - ctx = prepareBuilderTestContext(t, ctx, nil, "") + ctx = prepareBuilderTestContext(t, ctx, nil, "", skipLibraries) defer cleanUpBuilderTestContext(t, ctx) tools := ctx.PackageManager.GetAllInstalledToolsReleases() @@ -131,7 +131,7 @@ func TestLoadLotsOfTools(t *testing.T) { HardwareDirs: paths.NewPathList("downloaded_board_manager_stuff"), BuiltInToolsDirs: paths.NewPathList("downloaded_tools", "tools_builtin"), } - ctx = prepareBuilderTestContext(t, ctx, nil, "") + ctx = prepareBuilderTestContext(t, ctx, nil, "", skipLibraries) defer cleanUpBuilderTestContext(t, ctx) tools := ctx.PackageManager.GetAllInstalledToolsReleases() From 3b829da20db78a67dea2f565a32c7d1eb0d2c57a Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Fri, 1 Sep 2023 22:26:01 +0200 Subject: [PATCH 20/29] remove LibraryResolver property from context --- commands/compile/compile.go | 1 - legacy/builder/container_find_includes.go | 47 +---------------------- legacy/builder/test/builder_test.go | 1 - legacy/builder/types/context.go | 1 - 4 files changed, 1 insertion(+), 49 deletions(-) diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 2ca455946c0..445f35b7a9b 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -263,7 +263,6 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream } builderCtx.LibrariesManager = lm - builderCtx.LibrariesResolver = libsResolver if builderCtx.Verbose { builderCtx.Warn(string(verboseOut)) } diff --git a/legacy/builder/container_find_includes.go b/legacy/builder/container_find_includes.go index 55b60347898..142263fa421 100644 --- a/legacy/builder/container_find_includes.go +++ b/legacy/builder/container_find_includes.go @@ -444,52 +444,7 @@ func queueSourceFilesFromFolder(ctx *types.Context, sourceFileQueue *types.Uniqu } func ResolveLibrary(ctx *types.Context, header string) *libraries.Library { - resolver := ctx.LibrariesResolver - importedLibraries := ctx.ImportedLibraries - - if ctx.LibrariesResolutionResults == nil { - ctx.LibrariesResolutionResults = map[string]types.LibraryResolutionResult{} - } - - candidates := resolver.AlternativesFor(header) - - if ctx.Verbose { - ctx.Info(tr("Alternatives for %[1]s: %[2]s", header, candidates)) - ctx.Info(fmt.Sprintf("ResolveLibrary(%s)", header)) - ctx.Info(fmt.Sprintf(" -> %s: %s", tr("candidates"), candidates)) - } - - if len(candidates) == 0 { - return nil - } - - for _, candidate := range candidates { - if importedLibraries.Contains(candidate) { - return nil - } - } - - selected := resolver.ResolveFor(header, ctx.TargetPlatform.Platform.Architecture) - if alreadyImported := importedLibraries.FindByName(selected.Name); alreadyImported != nil { - // Certain libraries might have the same name but be different. - // This usually happens when the user includes two or more custom libraries that have - // different header name but are stored in a parent folder with identical name, like - // ./libraries1/Lib/lib1.h and ./libraries2/Lib/lib2.h - // Without this check the library resolution would be stuck in a loop. - // This behaviour has been reported in this issue: - // https://github.com/arduino/arduino-cli/issues/973 - if selected == alreadyImported { - selected = alreadyImported - } - } - - candidates.Remove(selected) - ctx.LibrariesResolutionResults[header] = types.LibraryResolutionResult{ - Library: selected, - NotUsedLibraries: candidates, - } - - return selected + return ctx.SketchLibrariesDetector.ResolveLibrary(header, ctx.TargetPlatform.Platform.Architecture) } func failIfImportedLibraryIsWrong(ctx *types.Context) error { diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index 084d885564c..05b1d84bbac 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -133,7 +133,6 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat NoError(t, err) ctx.LibrariesManager = lm - ctx.LibrariesResolver = libsResolver ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( lm, libsResolver, ctx.ImportedLibraries, diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 9e022e315e9..901ef26b75a 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -105,7 +105,6 @@ type Context struct { // Libraries handling LibrariesManager *librariesmanager.LibrariesManager - LibrariesResolver *librariesresolver.Cpp ImportedLibraries libraries.List LibrariesResolutionResults map[string]LibraryResolutionResult IncludeFolders paths.PathList From e2f7c7b74536f8a7d968ace953b9616fae79f109 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Fri, 1 Sep 2023 22:31:48 +0200 Subject: [PATCH 21/29] remove UseCachedLibrariesResolution for context --- arduino/builder/libraries.go | 4 ++++ commands/compile/compile.go | 6 +++--- legacy/builder/container_find_includes.go | 4 ++-- legacy/builder/test/builder_test.go | 4 ++-- legacy/builder/test/libraries_loader_test.go | 8 ++++---- legacy/builder/types/context.go | 2 -- 6 files changed, 15 insertions(+), 13 deletions(-) diff --git a/arduino/builder/libraries.go b/arduino/builder/libraries.go index ffee5539ec4..b620280940e 100644 --- a/arduino/builder/libraries.go +++ b/arduino/builder/libraries.go @@ -117,6 +117,10 @@ func (l *SketchLibrariesDetector) ImportedLibraries() libraries.List { return l.importedLibraries } +func (l *SketchLibrariesDetector) UseCachedLibrariesResolution() bool { + return l.useCachedLibrariesResolution +} + // LibrariesLoader todo func LibrariesLoader( useCachedLibrariesResolution bool, diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 445f35b7a9b..56ea6728d0a 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -249,12 +249,12 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream } // TODO replace all UseCache call with our SketchLibrariesDetector - builderCtx.UseCachedLibrariesResolution = req.GetSkipLibrariesDiscovery() + useCachedLibrariesResolution := req.GetSkipLibrariesDiscovery() if pme.GetProfile() != nil { builderCtx.LibrariesManager = lm } lm, libsResolver, verboseOut, err := bldr.LibrariesLoader( - builderCtx.UseCachedLibrariesResolution, builderCtx.LibrariesManager, + useCachedLibrariesResolution, builderCtx.LibrariesManager, builderCtx.BuiltInLibrariesDirs, builderCtx.LibraryDirs, builderCtx.OtherLibrariesDirs, builderCtx.ActualPlatform, builderCtx.TargetPlatform, ) @@ -271,7 +271,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream lm, libsResolver, builderCtx.ImportedLibraries, builderCtx.Verbose, - builderCtx.UseCachedLibrariesResolution, + useCachedLibrariesResolution, func(msg string) { builderCtx.Info(msg) }, func(msg string) { builderCtx.Warn(msg) }, ) diff --git a/legacy/builder/container_find_includes.go b/legacy/builder/container_find_includes.go index 142263fa421..c810a30622a 100644 --- a/legacy/builder/container_find_includes.go +++ b/legacy/builder/container_find_includes.go @@ -124,7 +124,7 @@ func (s *ContainerFindIncludes) Run(ctx *types.Context) error { func (s *ContainerFindIncludes) findIncludes(ctx *types.Context) error { librariesResolutionCache := ctx.BuildPath.Join("libraries.cache") - if ctx.UseCachedLibrariesResolution && librariesResolutionCache.Exist() { + if ctx.SketchLibrariesDetector.UseCachedLibrariesResolution() && librariesResolutionCache.Exist() { if d, err := librariesResolutionCache.ReadFile(); err != nil { return err } else if err := json.Unmarshal(d, &ctx.IncludeFolders); err != nil { @@ -146,7 +146,7 @@ func (s *ContainerFindIncludes) findIncludes(ctx *types.Context) error { sourceFileQueue := &types.UniqueSourceFileQueue{} - if !ctx.UseCachedLibrariesResolution { + if !ctx.SketchLibrariesDetector.UseCachedLibrariesResolution() { sketch := ctx.Sketch mergedfile, err := types.MakeSourceFile(ctx, sketch, paths.New(sketch.MainFile.Base()+".cpp")) if err != nil { diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index 05b1d84bbac..f7f1166d43c 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -126,7 +126,7 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat if !stepToSkip[skipLibraries] { lm, libsResolver, _, err := bldr.LibrariesLoader( - ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, + false, ctx.LibrariesManager, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) @@ -137,7 +137,7 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat lm, libsResolver, ctx.ImportedLibraries, ctx.Verbose, - ctx.UseCachedLibrariesResolution, + false, func(msg string) { ctx.Info(msg) }, func(msg string) { ctx.Warn(msg) }, ) diff --git a/legacy/builder/test/libraries_loader_test.go b/legacy/builder/test/libraries_loader_test.go index 8ef2c629a24..66e27202fa5 100644 --- a/legacy/builder/test/libraries_loader_test.go +++ b/legacy/builder/test/libraries_loader_test.go @@ -48,7 +48,7 @@ func TestLoadLibrariesAVR(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) lm, libsResolver, _, err := builder.LibrariesLoader( - ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, + false, ctx.LibrariesManager, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) @@ -152,7 +152,7 @@ func TestLoadLibrariesSAM(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) lm, libsResolver, _, err := builder.LibrariesLoader( - ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, + false, ctx.LibrariesManager, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) @@ -229,7 +229,7 @@ func TestLoadLibrariesAVRNoDuplicateLibrariesFolders(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) lm, _, _, err := builder.LibrariesLoader( - ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, + false, ctx.LibrariesManager, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) @@ -252,7 +252,7 @@ func TestLoadLibrariesMyAVRPlatform(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) lm, _, _, err := builder.LibrariesLoader( - ctx.UseCachedLibrariesResolution, ctx.LibrariesManager, + false, ctx.LibrariesManager, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 901ef26b75a..edf938877bb 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -27,7 +27,6 @@ import ( "github.com/arduino/arduino-cli/arduino/cores/packagemanager" "github.com/arduino/arduino-cli/arduino/libraries" "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" - "github.com/arduino/arduino-cli/arduino/libraries/librariesresolver" "github.com/arduino/arduino-cli/arduino/sketch" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" paths "github.com/arduino/go-paths-helper" @@ -108,7 +107,6 @@ type Context struct { ImportedLibraries libraries.List LibrariesResolutionResults map[string]LibraryResolutionResult IncludeFolders paths.PathList - UseCachedLibrariesResolution bool // C++ Parsing LineOffset int From e3e2007cc70b9759d8f656383d384b4196cc54d4 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Fri, 1 Sep 2023 22:56:05 +0200 Subject: [PATCH 22/29] remove ImportedLibraries from context --- arduino/builder/libraries.go | 25 +++++++++++++------ commands/compile/compile.go | 3 +-- legacy/builder/container_find_includes.go | 6 ++--- legacy/builder/create_cmake_rule.go | 2 +- legacy/builder/phases/libraries_builder.go | 2 +- .../print_used_libraries_if_verbose.go | 4 +-- legacy/builder/test/builder_test.go | 1 - .../unused_compiled_libraries_remover_test.go | 15 ++++++++--- legacy/builder/types/context.go | 2 -- .../unused_compiled_libraries_remover.go | 2 +- .../warn_about_arch_incompatible_libraries.go | 2 +- 11 files changed, 40 insertions(+), 24 deletions(-) diff --git a/arduino/builder/libraries.go b/arduino/builder/libraries.go index b620280940e..d6f273f5f4c 100644 --- a/arduino/builder/libraries.go +++ b/arduino/builder/libraries.go @@ -40,21 +40,18 @@ type SketchLibrariesDetector struct { verbose bool verboseInfoFn func(msg string) verboseWarnFn func(msg string) - - importedLibraries libraries.List - librariesResolutionResults map[string]libraryResolutionResult + importedLibraries libraries.List + librariesResolutionResults map[string]libraryResolutionResult } // NewSketchLibrariesDetector todo func NewSketchLibrariesDetector( lm *librariesmanager.LibrariesManager, libsResolver *librariesresolver.Cpp, - importedLibraries libraries.List, verbose, useCachedLibrariesResolution bool, verboseInfoFn func(msg string), verboseWarnFn func(msg string), ) *SketchLibrariesDetector { - return &SketchLibrariesDetector{ librariesManager: lm, librariesResolver: libsResolver, @@ -63,8 +60,7 @@ func NewSketchLibrariesDetector( verbose: verbose, verboseInfoFn: verboseInfoFn, verboseWarnFn: verboseWarnFn, - - importedLibraries: importedLibraries, + importedLibraries: libraries.List{}, } } @@ -112,15 +108,30 @@ func (l *SketchLibrariesDetector) ResolveLibrary(header, platformArch string) *l return selected } +// ImportedLibraries todo func (l *SketchLibrariesDetector) ImportedLibraries() libraries.List { // TODO understand if we have to do a deepcopy return l.importedLibraries } +// AppendImportedLibraries todo should rename this, probably after refactoring the +// container_find_includes command. +func (l *SketchLibrariesDetector) AppendImportedLibraries(library *libraries.Library) { + l.importedLibraries = append(l.importedLibraries, library) +} + +// UseCachedLibrariesResolution todo func (l *SketchLibrariesDetector) UseCachedLibrariesResolution() bool { return l.useCachedLibrariesResolution } +// AppendIncludeFolder todo should rename this, probably after refactoring the +// container_find_includes command. +//func (l *SketchLibrariesDetector) AppendIncludeFolder(ctx *types.Context, cache *includeCache, sourceFilePath *paths.Path, include string, folder *paths.Path) { +// ctx.IncludeFolders = append(ctx.IncludeFolders, folder) +// cache.ExpectEntry(sourceFilePath, include, folder) +//} + // LibrariesLoader todo func LibrariesLoader( useCachedLibrariesResolution bool, diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 56ea6728d0a..c5f36dbcc7c 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -269,7 +269,6 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( lm, libsResolver, - builderCtx.ImportedLibraries, builderCtx.Verbose, useCachedLibrariesResolution, func(msg string) { builderCtx.Info(msg) }, @@ -313,7 +312,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream defer func() { importedLibs := []*rpc.Library{} - for _, lib := range builderCtx.ImportedLibraries { + for _, lib := range builderCtx.SketchLibrariesDetector.ImportedLibraries() { rpcLib, err := lib.ToRPCLibrary() if err != nil { msg := tr("Error getting information for library %s", lib.Name) + ": " + err.Error() + "\n" diff --git a/legacy/builder/container_find_includes.go b/legacy/builder/container_find_includes.go index c810a30622a..b90c25d46d5 100644 --- a/legacy/builder/container_find_includes.go +++ b/legacy/builder/container_find_includes.go @@ -405,7 +405,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFileQu // Add this library to the list of libraries, the // include path and queue its source files for further // include scanning - ctx.ImportedLibraries = append(ctx.ImportedLibraries, library) + ctx.SketchLibrariesDetector.AppendImportedLibraries(library) appendIncludeFolder(ctx, cache, sourcePath, missingIncludeH, library.SourceDir) if library.Precompiled && library.PrecompiledWithSources { @@ -448,11 +448,11 @@ func ResolveLibrary(ctx *types.Context, header string) *libraries.Library { } func failIfImportedLibraryIsWrong(ctx *types.Context) error { - if len(ctx.ImportedLibraries) == 0 { + if len(ctx.SketchLibrariesDetector.ImportedLibraries()) == 0 { return nil } - for _, library := range ctx.ImportedLibraries { + for _, library := range ctx.SketchLibrariesDetector.ImportedLibraries() { if !library.IsLegacy { if library.InstallDir.Join("arch").IsDir() { return errors.New(tr("%[1]s folder is no longer supported! See %[2]s for more information", "'arch'", "http://goo.gl/gfFJzU")) diff --git a/legacy/builder/create_cmake_rule.go b/legacy/builder/create_cmake_rule.go index acbce09e2cb..37b60e92fac 100644 --- a/legacy/builder/create_cmake_rule.go +++ b/legacy/builder/create_cmake_rule.go @@ -198,7 +198,7 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { cmakeFile := cmakeFolder.Join("CMakeLists.txt") dynamicLibsFromPkgConfig := map[string]bool{} - for _, library := range ctx.ImportedLibraries { + for _, library := range ctx.SketchLibrariesDetector.ImportedLibraries() { // Copy used libraries in the correct folder libDir := libBaseFolder.Join(library.DirName) mcu := ctx.BuildProperties.Get(constants.BUILD_PROPERTIES_BUILD_MCU) diff --git a/legacy/builder/phases/libraries_builder.go b/legacy/builder/phases/libraries_builder.go index 5ee81bb4385..3d28fbf5520 100644 --- a/legacy/builder/phases/libraries_builder.go +++ b/legacy/builder/phases/libraries_builder.go @@ -38,7 +38,7 @@ func (s *LibrariesBuilder) Run(ctx *types.Context) error { librariesBuildPath := ctx.LibrariesBuildPath buildProperties := ctx.BuildProperties includes := f.Map(ctx.IncludeFolders.AsStrings(), utils.WrapWithHyphenI) - libs := ctx.ImportedLibraries + libs := ctx.SketchLibrariesDetector.ImportedLibraries() if err := librariesBuildPath.MkdirAll(); err != nil { return errors.WithStack(err) diff --git a/legacy/builder/print_used_libraries_if_verbose.go b/legacy/builder/print_used_libraries_if_verbose.go index 8c32c925995..15211a46519 100644 --- a/legacy/builder/print_used_libraries_if_verbose.go +++ b/legacy/builder/print_used_libraries_if_verbose.go @@ -24,11 +24,11 @@ import ( type PrintUsedLibrariesIfVerbose struct{} func (s *PrintUsedLibrariesIfVerbose) Run(ctx *types.Context) error { - if !ctx.Verbose || len(ctx.ImportedLibraries) == 0 { + if !ctx.Verbose || len(ctx.SketchLibrariesDetector.ImportedLibraries()) == 0 { return nil } - for _, library := range ctx.ImportedLibraries { + for _, library := range ctx.SketchLibrariesDetector.ImportedLibraries() { legacy := "" if library.IsLegacy { legacy = tr("(legacy)") diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index f7f1166d43c..4577da79639 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -135,7 +135,6 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat ctx.LibrariesManager = lm ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( lm, libsResolver, - ctx.ImportedLibraries, ctx.Verbose, false, func(msg string) { ctx.Info(msg) }, diff --git a/legacy/builder/test/unused_compiled_libraries_remover_test.go b/legacy/builder/test/unused_compiled_libraries_remover_test.go index f6965cee0ff..5c04b1cee10 100644 --- a/legacy/builder/test/unused_compiled_libraries_remover_test.go +++ b/legacy/builder/test/unused_compiled_libraries_remover_test.go @@ -18,6 +18,7 @@ package test import ( "testing" + bldr "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/libraries" "github.com/arduino/arduino-cli/legacy/builder" "github.com/arduino/arduino-cli/legacy/builder/types" @@ -36,7 +37,10 @@ func TestUnusedCompiledLibrariesRemover(t *testing.T) { ctx := &types.Context{} ctx.LibrariesBuildPath = temp - ctx.ImportedLibraries = []*libraries.Library{{Name: "Bridge"}} + ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( + nil, nil, false, false, nil, nil, + ) + ctx.SketchLibrariesDetector.AppendImportedLibraries(&libraries.Library{Name: "Bridge"}) cmd := builder.UnusedCompiledLibrariesRemover{} err = cmd.Run(ctx) @@ -56,7 +60,10 @@ func TestUnusedCompiledLibrariesRemover(t *testing.T) { func TestUnusedCompiledLibrariesRemoverLibDoesNotExist(t *testing.T) { ctx := &types.Context{} ctx.LibrariesBuildPath = paths.TempDir().Join("test") - ctx.ImportedLibraries = []*libraries.Library{{Name: "Bridge"}} + ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( + nil, nil, false, false, nil, nil, + ) + ctx.SketchLibrariesDetector.AppendImportedLibraries(&libraries.Library{Name: "Bridge"}) cmd := builder.UnusedCompiledLibrariesRemover{} err := cmd.Run(ctx) @@ -73,8 +80,10 @@ func TestUnusedCompiledLibrariesRemoverNoUsedLibraries(t *testing.T) { NoError(t, temp.Join("dummy_file").WriteFile([]byte{})) ctx := &types.Context{} + ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( + nil, nil, false, false, nil, nil, + ) ctx.LibrariesBuildPath = temp - ctx.ImportedLibraries = []*libraries.Library{} cmd := builder.UnusedCompiledLibrariesRemover{} err = cmd.Run(ctx) diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index edf938877bb..beb4336fac8 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -25,7 +25,6 @@ import ( "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" - "github.com/arduino/arduino-cli/arduino/libraries" "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" "github.com/arduino/arduino-cli/arduino/sketch" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" @@ -104,7 +103,6 @@ type Context struct { // Libraries handling LibrariesManager *librariesmanager.LibrariesManager - ImportedLibraries libraries.List LibrariesResolutionResults map[string]LibraryResolutionResult IncludeFolders paths.PathList diff --git a/legacy/builder/unused_compiled_libraries_remover.go b/legacy/builder/unused_compiled_libraries_remover.go index d1b36b147ac..03ea5b85d01 100644 --- a/legacy/builder/unused_compiled_libraries_remover.go +++ b/legacy/builder/unused_compiled_libraries_remover.go @@ -31,7 +31,7 @@ func (s *UnusedCompiledLibrariesRemover) Run(ctx *types.Context) error { return nil } - libraryNames := toLibraryNames(ctx.ImportedLibraries) + libraryNames := toLibraryNames(ctx.SketchLibrariesDetector.ImportedLibraries()) files, err := librariesBuildPath.ReadDir() if err != nil { diff --git a/legacy/builder/warn_about_arch_incompatible_libraries.go b/legacy/builder/warn_about_arch_incompatible_libraries.go index 712f79b2fe5..c815a54f3aa 100644 --- a/legacy/builder/warn_about_arch_incompatible_libraries.go +++ b/legacy/builder/warn_about_arch_incompatible_libraries.go @@ -33,7 +33,7 @@ func (s *WarnAboutArchIncompatibleLibraries) Run(ctx *types.Context) error { archs = append(archs, strings.Split(overrides, ",")...) } - for _, importedLibrary := range ctx.ImportedLibraries { + for _, importedLibrary := range ctx.SketchLibrariesDetector.ImportedLibraries() { if !importedLibrary.SupportsAnyArchitectureIn(archs...) { ctx.Info( tr("WARNING: library %[1]s claims to run on %[2]s architecture(s) and may be incompatible with your current board which runs on %[3]s architecture(s).", From 4346e165655a8df0973ace8bee836ff5b1bb59fb Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Fri, 1 Sep 2023 23:03:30 +0200 Subject: [PATCH 23/29] remove LibrariesResolutionResults from context --- arduino/builder/libraries.go | 33 +++++++++++ legacy/builder/builder.go | 5 +- .../print_used_and_not_used_libraries.go | 59 ------------------- legacy/builder/types/context.go | 5 +- legacy/builder/types/types.go | 5 -- 5 files changed, 39 insertions(+), 68 deletions(-) delete mode 100644 legacy/builder/print_used_and_not_used_libraries.go diff --git a/arduino/builder/libraries.go b/arduino/builder/libraries.go index d6f273f5f4c..2f8219c5241 100644 --- a/arduino/builder/libraries.go +++ b/arduino/builder/libraries.go @@ -18,6 +18,8 @@ package builder import ( "bytes" "fmt" + "strings" + "time" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/libraries" @@ -125,6 +127,37 @@ func (l *SketchLibrariesDetector) UseCachedLibrariesResolution() bool { return l.useCachedLibrariesResolution } +// PrintUsedAndNotUsedLibraries todo +func (l *SketchLibrariesDetector) PrintUsedAndNotUsedLibraries(sketchError bool) { + // Print this message: + // - as warning, when the sketch didn't compile + // - as info, when verbose is on + // - otherwise, output nothing + if !sketchError && !l.verbose { + return + } + + res := "" + for header, libResResult := range l.librariesResolutionResults { + if len(libResResult.NotUsedLibraries) == 0 { + continue + } + res += fmt.Sprintln(tr(`Multiple libraries were found for "%[1]s"`, header)) + res += fmt.Sprintln(" " + tr("Used: %[1]s", libResResult.Library.InstallDir)) + for _, notUsedLibrary := range libResResult.NotUsedLibraries { + res += fmt.Sprintln(" " + tr("Not used: %[1]s", notUsedLibrary.InstallDir)) + } + } + res = strings.TrimSpace(res) + if sketchError { + l.verboseWarnFn(res) + } else { + l.verboseInfoFn(res) + } + // todo why?? should we remove this? + time.Sleep(100 * time.Millisecond) +} + // AppendIncludeFolder todo should rename this, probably after refactoring the // container_find_includes command. //func (l *SketchLibrariesDetector) AppendIncludeFolder(ctx *types.Context, cache *includeCache, sourceFilePath *paths.Path, include string, folder *paths.Path) { diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index f914e11140f..1af79855d5a 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -109,7 +109,10 @@ func (s *Builder) Run(ctx *types.Context) error { var otherErr error commands = []types.Command{ - &PrintUsedAndNotUsedLibraries{SketchError: mainErr != nil}, + types.BareCommand(func(ctx *types.Context) error { + ctx.SketchLibrariesDetector.PrintUsedAndNotUsedLibraries(mainErr != nil) + return nil + }), &PrintUsedLibrariesIfVerbose{}, diff --git a/legacy/builder/print_used_and_not_used_libraries.go b/legacy/builder/print_used_and_not_used_libraries.go deleted file mode 100644 index 971036bea45..00000000000 --- a/legacy/builder/print_used_and_not_used_libraries.go +++ /dev/null @@ -1,59 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "fmt" - "strings" - "time" - - "github.com/arduino/arduino-cli/legacy/builder/types" -) - -type PrintUsedAndNotUsedLibraries struct { - // Was there an error while compiling the sketch? - SketchError bool -} - -func (s *PrintUsedAndNotUsedLibraries) Run(ctx *types.Context) error { - // Print this message: - // - as warning, when the sketch didn't compile - // - as info, when verbose is on - // - otherwise, output nothing - if !s.SketchError && !ctx.Verbose { - return nil - } - - res := "" - for header, libResResult := range ctx.LibrariesResolutionResults { - if len(libResResult.NotUsedLibraries) == 0 { - continue - } - res += fmt.Sprintln(tr(`Multiple libraries were found for "%[1]s"`, header)) - res += fmt.Sprintln(" " + tr("Used: %[1]s", libResResult.Library.InstallDir)) - for _, notUsedLibrary := range libResResult.NotUsedLibraries { - res += fmt.Sprintln(" " + tr("Not used: %[1]s", notUsedLibrary.InstallDir)) - } - } - res = strings.TrimSpace(res) - if s.SketchError { - ctx.Warn(res) - } else { - ctx.Info(res) - } - time.Sleep(100 * time.Millisecond) - return nil -} diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index beb4336fac8..6a50ecd21de 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -102,9 +102,8 @@ type Context struct { WarningsLevel string // Libraries handling - LibrariesManager *librariesmanager.LibrariesManager - LibrariesResolutionResults map[string]LibraryResolutionResult - IncludeFolders paths.PathList + LibrariesManager *librariesmanager.LibrariesManager + IncludeFolders paths.PathList // C++ Parsing LineOffset int diff --git a/legacy/builder/types/types.go b/legacy/builder/types/types.go index 2ebf92bae5a..d84864eecfe 100644 --- a/legacy/builder/types/types.go +++ b/legacy/builder/types/types.go @@ -95,11 +95,6 @@ func (f *SourceFile) DepfilePath() *paths.Path { return f.buildRoot.Join(f.relativePath.String() + ".d") } -type LibraryResolutionResult struct { - Library *libraries.Library - NotUsedLibraries []*libraries.Library -} - type Command interface { Run(ctx *Context) error } From 161e7473a9af0416f09deff348972a10f8e1dd1a Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Fri, 1 Sep 2023 23:12:34 +0200 Subject: [PATCH 24/29] remove LibrariesManager from context --- commands/compile/compile.go | 14 +++++++------- legacy/builder/test/builder_test.go | 3 +-- legacy/builder/test/libraries_loader_test.go | 8 ++++---- legacy/builder/types/context.go | 2 -- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/commands/compile/compile.go b/commands/compile/compile.go index c5f36dbcc7c..96f30a83f39 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -25,6 +25,7 @@ import ( "github.com/arduino/arduino-cli/arduino" bldr "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/cores" + "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/arduino/utils" "github.com/arduino/arduino-cli/buildcache" @@ -248,13 +249,13 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream } } - // TODO replace all UseCache call with our SketchLibrariesDetector - useCachedLibrariesResolution := req.GetSkipLibrariesDiscovery() + var libsManager *librariesmanager.LibrariesManager if pme.GetProfile() != nil { - builderCtx.LibrariesManager = lm + libsManager = lm } - lm, libsResolver, verboseOut, err := bldr.LibrariesLoader( - useCachedLibrariesResolution, builderCtx.LibrariesManager, + useCachedLibrariesResolution := req.GetSkipLibrariesDiscovery() + libsManager, libsResolver, verboseOut, err := bldr.LibrariesLoader( + useCachedLibrariesResolution, libsManager, builderCtx.BuiltInLibrariesDirs, builderCtx.LibraryDirs, builderCtx.OtherLibrariesDirs, builderCtx.ActualPlatform, builderCtx.TargetPlatform, ) @@ -262,13 +263,12 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream return r, &arduino.CompileFailedError{Message: err.Error()} } - builderCtx.LibrariesManager = lm if builderCtx.Verbose { builderCtx.Warn(string(verboseOut)) } builderCtx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( - lm, libsResolver, + libsManager, libsResolver, builderCtx.Verbose, useCachedLibrariesResolution, func(msg string) { builderCtx.Info(msg) }, diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index 4577da79639..d4a6d4901a5 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -126,13 +126,12 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat if !stepToSkip[skipLibraries] { lm, libsResolver, _, err := bldr.LibrariesLoader( - false, ctx.LibrariesManager, + false, nil, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) NoError(t, err) - ctx.LibrariesManager = lm ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( lm, libsResolver, ctx.Verbose, diff --git a/legacy/builder/test/libraries_loader_test.go b/legacy/builder/test/libraries_loader_test.go index 66e27202fa5..dc4707455ff 100644 --- a/legacy/builder/test/libraries_loader_test.go +++ b/legacy/builder/test/libraries_loader_test.go @@ -48,7 +48,7 @@ func TestLoadLibrariesAVR(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) lm, libsResolver, _, err := builder.LibrariesLoader( - false, ctx.LibrariesManager, + false, nil, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) @@ -152,7 +152,7 @@ func TestLoadLibrariesSAM(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) lm, libsResolver, _, err := builder.LibrariesLoader( - false, ctx.LibrariesManager, + false, nil, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) @@ -229,7 +229,7 @@ func TestLoadLibrariesAVRNoDuplicateLibrariesFolders(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) lm, _, _, err := builder.LibrariesLoader( - false, ctx.LibrariesManager, + false, nil, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) @@ -252,7 +252,7 @@ func TestLoadLibrariesMyAVRPlatform(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) lm, _, _, err := builder.LibrariesLoader( - false, ctx.LibrariesManager, + false, nil, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 6a50ecd21de..a78530387e9 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -25,7 +25,6 @@ import ( "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" - "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" "github.com/arduino/arduino-cli/arduino/sketch" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" paths "github.com/arduino/go-paths-helper" @@ -102,7 +101,6 @@ type Context struct { WarningsLevel string // Libraries handling - LibrariesManager *librariesmanager.LibrariesManager IncludeFolders paths.PathList // C++ Parsing From 3aa1e006edf0dd68f76c07a5df50d79cf1b01df3 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Sat, 2 Sep 2023 00:01:19 +0200 Subject: [PATCH 25/29] fix regression exceeding 100% status bar --- legacy/builder/builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index 1af79855d5a..8a8c8253d26 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -89,7 +89,7 @@ func (s *Builder) Run(ctx *types.Context) error { &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.postbuild", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true}, } - ctx.Progress.AddSubSteps(len(commands) + 4) + ctx.Progress.AddSubSteps(len(commands) + 5) defer ctx.Progress.RemoveSubSteps() for _, command := range commands { From 27fbcfb94ce345f98dc3396a064e970aef9f610e Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Sat, 2 Sep 2023 21:21:39 +0200 Subject: [PATCH 26/29] refactor find_includes --- arduino/builder/detector/detector.go | 795 ++++++++++++++++++ arduino/builder/libraries.go | 240 ------ .../preprocessor/arduino_preprocessor.go | 2 +- arduino/builder/preprocessor/gcc.go | 2 +- arduino/builder/utils/utils.go | 185 ++++ commands/compile/compile.go | 8 +- legacy/builder/builder.go | 22 +- legacy/builder/builder_utils/utils.go | 123 +-- legacy/builder/container_find_includes.go | 474 ----------- legacy/builder/create_cmake_rule.go | 2 +- legacy/builder/includes_finder_with_regexp.go | 44 - legacy/builder/phases/core_builder.go | 2 +- legacy/builder/phases/libraries_builder.go | 5 +- legacy/builder/phases/sketch_builder.go | 5 +- legacy/builder/test/builder_test.go | 3 + .../unused_compiled_libraries_remover_test.go | 6 +- legacy/builder/types/accessories.go | 41 - legacy/builder/types/context.go | 6 +- legacy/builder/types/types.go | 80 -- legacy/builder/utils/utils.go | 61 -- 20 files changed, 1027 insertions(+), 1079 deletions(-) create mode 100644 arduino/builder/detector/detector.go delete mode 100644 arduino/builder/libraries.go create mode 100644 arduino/builder/utils/utils.go delete mode 100644 legacy/builder/container_find_includes.go delete mode 100644 legacy/builder/includes_finder_with_regexp.go delete mode 100644 legacy/builder/types/accessories.go diff --git a/arduino/builder/detector/detector.go b/arduino/builder/detector/detector.go new file mode 100644 index 00000000000..aa00a02e718 --- /dev/null +++ b/arduino/builder/detector/detector.go @@ -0,0 +1,795 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package detector + +import ( + "bytes" + "encoding/json" + "fmt" + "os/exec" + "regexp" + "strings" + "time" + + "golang.org/x/exp/slices" + + "github.com/arduino/arduino-cli/arduino/builder/preprocessor" + "github.com/arduino/arduino-cli/arduino/builder/utils" + "github.com/arduino/arduino-cli/arduino/cores" + "github.com/arduino/arduino-cli/arduino/globals" + "github.com/arduino/arduino-cli/arduino/libraries" + "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" + "github.com/arduino/arduino-cli/arduino/libraries/librariesresolver" + "github.com/arduino/arduino-cli/arduino/sketch" + "github.com/arduino/arduino-cli/i18n" + "github.com/arduino/go-paths-helper" + "github.com/arduino/go-properties-orderedmap" + "github.com/pkg/errors" +) + +var tr = i18n.Tr + +type libraryResolutionResult struct { + Library *libraries.Library + NotUsedLibraries []*libraries.Library +} + +// SketchLibrariesDetector todo +type SketchLibrariesDetector struct { + librariesManager *librariesmanager.LibrariesManager + librariesResolver *librariesresolver.Cpp + useCachedLibrariesResolution bool + verbose bool + onlyUpdateCompilationDatabase bool + verboseInfoFn func(msg string) + verboseWarnFn func(msg string) + verboseStdoutFn func(data []byte) + verboseStderrFn func(data []byte) + importedLibraries libraries.List + librariesResolutionResults map[string]libraryResolutionResult + includeFolders paths.PathList +} + +// NewSketchLibrariesDetector todo +func NewSketchLibrariesDetector( + lm *librariesmanager.LibrariesManager, + libsResolver *librariesresolver.Cpp, + verbose, useCachedLibrariesResolution bool, + onlyUpdateCompilationDatabase bool, + verboseInfoFn func(msg string), + verboseWarnFn func(msg string), + verboseStdoutFn func(data []byte), + verboseStderrFn func(data []byte), +) *SketchLibrariesDetector { + return &SketchLibrariesDetector{ + librariesManager: lm, + librariesResolver: libsResolver, + useCachedLibrariesResolution: useCachedLibrariesResolution, + librariesResolutionResults: map[string]libraryResolutionResult{}, + verbose: verbose, + verboseInfoFn: verboseInfoFn, + verboseWarnFn: verboseWarnFn, + verboseStdoutFn: verboseStdoutFn, + verboseStderrFn: verboseStderrFn, + importedLibraries: libraries.List{}, + includeFolders: paths.PathList{}, + onlyUpdateCompilationDatabase: onlyUpdateCompilationDatabase, + } +} + +// ResolveLibrary todo +func (l *SketchLibrariesDetector) resolveLibrary(header, platformArch string) *libraries.Library { + importedLibraries := l.importedLibraries + candidates := l.librariesResolver.AlternativesFor(header) + + if l.verbose { + l.verboseInfoFn(tr("Alternatives for %[1]s: %[2]s", header, candidates)) + l.verboseInfoFn(fmt.Sprintf("ResolveLibrary(%s)", header)) + l.verboseInfoFn(fmt.Sprintf(" -> %s: %s", tr("candidates"), candidates)) + } + + if len(candidates) == 0 { + return nil + } + + for _, candidate := range candidates { + if importedLibraries.Contains(candidate) { + return nil + } + } + + selected := l.librariesResolver.ResolveFor(header, platformArch) + if alreadyImported := importedLibraries.FindByName(selected.Name); alreadyImported != nil { + // Certain libraries might have the same name but be different. + // This usually happens when the user includes two or more custom libraries that have + // different header name but are stored in a parent folder with identical name, like + // ./libraries1/Lib/lib1.h and ./libraries2/Lib/lib2.h + // Without this check the library resolution would be stuck in a loop. + // This behaviour has been reported in this issue: + // https://github.com/arduino/arduino-cli/issues/973 + if selected == alreadyImported { + selected = alreadyImported + } + } + + candidates.Remove(selected) + l.librariesResolutionResults[header] = libraryResolutionResult{ + Library: selected, + NotUsedLibraries: candidates, + } + + return selected +} + +// ImportedLibraries todo +func (l *SketchLibrariesDetector) ImportedLibraries() libraries.List { + // TODO understand if we have to do a deepcopy + return l.importedLibraries +} + +// AppendImportedLibraries todo should rename this, probably after refactoring the +// container_find_includes command. +func (l *SketchLibrariesDetector) AppendImportedLibraries(library *libraries.Library) { + l.importedLibraries = append(l.importedLibraries, library) +} + +// UseCachedLibrariesResolution todo +func (l *SketchLibrariesDetector) UseCachedLibrariesResolution() bool { + return l.useCachedLibrariesResolution +} + +// PrintUsedAndNotUsedLibraries todo +func (l *SketchLibrariesDetector) PrintUsedAndNotUsedLibraries(sketchError bool) { + // Print this message: + // - as warning, when the sketch didn't compile + // - as info, when verbose is on + // - otherwise, output nothing + if !sketchError && !l.verbose { + return + } + + res := "" + for header, libResResult := range l.librariesResolutionResults { + if len(libResResult.NotUsedLibraries) == 0 { + continue + } + res += fmt.Sprintln(tr(`Multiple libraries were found for "%[1]s"`, header)) + res += fmt.Sprintln(" " + tr("Used: %[1]s", libResResult.Library.InstallDir)) + for _, notUsedLibrary := range libResResult.NotUsedLibraries { + res += fmt.Sprintln(" " + tr("Not used: %[1]s", notUsedLibrary.InstallDir)) + } + } + res = strings.TrimSpace(res) + if sketchError { + l.verboseWarnFn(res) + } else { + l.verboseInfoFn(res) + } + // todo why?? should we remove this? + time.Sleep(100 * time.Millisecond) +} + +func (l *SketchLibrariesDetector) IncludeFolders() paths.PathList { + // TODO should we do a deep copy? + return l.includeFolders +} + +// AppendIncludeFolder todo should rename this, probably after refactoring the +// container_find_includes command. +// Original comment: +// Append the given folder to the include path and match or append it to +// the cache. sourceFilePath and include indicate the source of this +// include (e.g. what #include line in what file it was resolved from) +// and should be the empty string for the default include folders, like +// the core or variant. +func (l *SketchLibrariesDetector) AppendIncludeFolder( + cache *includeCache, + sourceFilePath *paths.Path, + include string, + folder *paths.Path, +) { + l.includeFolders = append(l.includeFolders, folder) + cache.ExpectEntry(sourceFilePath, include, folder) +} + +// FindIncludes todo +func (l *SketchLibrariesDetector) FindIncludes( + buildPath *paths.Path, + buildCorePath *paths.Path, + buildVariantPath *paths.Path, + sketchBuildPath *paths.Path, + sketch *sketch.Sketch, + librariesBuildPath *paths.Path, + buildProperties *properties.Map, + platformArch string, +) error { + err := l.findIncludes(buildPath, buildCorePath, buildVariantPath, sketchBuildPath, sketch, librariesBuildPath, buildProperties, platformArch) + if err != nil && l.onlyUpdateCompilationDatabase { + l.verboseInfoFn( + fmt.Sprintf( + "%s: %s", + tr("An error occurred detecting libraries"), + tr("the compilation database may be incomplete or inaccurate"), + ), + ) + return nil + } + return err +} + +func (l *SketchLibrariesDetector) findIncludes( + buildPath *paths.Path, + buildCorePath *paths.Path, + buildVariantPath *paths.Path, + sketchBuildPath *paths.Path, + sketch *sketch.Sketch, + librariesBuildPath *paths.Path, + buildProperties *properties.Map, + platformArch string, +) error { + librariesResolutionCache := buildPath.Join("libraries.cache") + if l.useCachedLibrariesResolution && librariesResolutionCache.Exist() { + if d, err := librariesResolutionCache.ReadFile(); err != nil { + return err + } else { + includeFolders := l.includeFolders + if err := json.Unmarshal(d, &includeFolders); err != nil { + return err + } + } + if l.verbose { + l.verboseInfoFn("Using cached library discovery: " + librariesResolutionCache.String()) + } + return nil + } + + cachePath := buildPath.Join("includes.cache") + cache := readCache(cachePath) + + l.AppendIncludeFolder(cache, nil, "", buildCorePath) + if buildVariantPath != nil { + l.AppendIncludeFolder(cache, nil, "", buildVariantPath) + } + + sourceFileQueue := &UniqueSourceFileQueue{} + + if !l.useCachedLibrariesResolution { + sketch := sketch + mergedfile, err := MakeSourceFile(sketchBuildPath, librariesBuildPath, sketch, paths.New(sketch.MainFile.Base()+".cpp")) + if err != nil { + return errors.WithStack(err) + } + sourceFileQueue.Push(mergedfile) + + l.queueSourceFilesFromFolder(sketchBuildPath, librariesBuildPath, sourceFileQueue, sketch, sketchBuildPath, false /* recurse */) + srcSubfolderPath := sketchBuildPath.Join("src") + if srcSubfolderPath.IsDir() { + l.queueSourceFilesFromFolder(sketchBuildPath, librariesBuildPath, sourceFileQueue, sketch, srcSubfolderPath, true /* recurse */) + } + + for !sourceFileQueue.Empty() { + err := l.findIncludesUntilDone(cache, sourceFileQueue, buildProperties, sketchBuildPath, librariesBuildPath, platformArch) + if err != nil { + cachePath.Remove() + return errors.WithStack(err) + } + } + + // Finalize the cache + cache.ExpectEnd() + if err := writeCache(cache, cachePath); err != nil { + return errors.WithStack(err) + } + } + + if err := l.failIfImportedLibraryIsWrong(); err != nil { + return errors.WithStack(err) + } + + if d, err := json.Marshal(l.includeFolders); err != nil { + return err + } else if err := librariesResolutionCache.WriteFile(d); err != nil { + return err + } + + return nil +} + +func (l *SketchLibrariesDetector) findIncludesUntilDone( + cache *includeCache, + sourceFileQueue *UniqueSourceFileQueue, + buildProperties *properties.Map, + sketchBuildPath *paths.Path, + librariesBuildPath *paths.Path, + platformArch string, +) error { + sourceFile := sourceFileQueue.Pop() + sourcePath := sourceFile.SourcePath() + targetFilePath := paths.NullPath() + depPath := sourceFile.DepfilePath() + objPath := sourceFile.ObjectPath() + + // TODO: This should perhaps also compare against the + // include.cache file timestamp. Now, it only checks if the file + // changed after the object file was generated, but if it + // changed between generating the cache and the object file, + // this could show the file as unchanged when it really is + // changed. Changing files during a build isn't really + // supported, but any problems from it should at least be + // resolved when doing another build, which is not currently the + // case. + // TODO: This reads the dependency file, but the actual building + // does it again. Should the result be somehow cached? Perhaps + // remove the object file if it is found to be stale? + unchanged, err := utils.ObjFileIsUpToDate(sourcePath, objPath, depPath) + if err != nil { + return errors.WithStack(err) + } + + first := true + for { + cache.ExpectFile(sourcePath) + + // Libraries may require the "utility" directory to be added to the include + // search path, but only for the source code of the library, so we temporary + // copy the current search path list and add the library' utility directory + // if needed. + includeFolders := l.includeFolders + if extraInclude := sourceFile.ExtraIncludePath(); extraInclude != nil { + includeFolders = append(includeFolders, extraInclude) + } + + var preprocErr error + var preprocStderr []byte + + var missingIncludeH string + if unchanged && cache.valid { + missingIncludeH = cache.Next().Include + if first && l.verbose { + l.verboseInfoFn(tr("Using cached library dependencies for file: %[1]s", sourcePath)) + } + } else { + var preprocStdout []byte + preprocStdout, preprocStderr, preprocErr = preprocessor.GCC(sourcePath, targetFilePath, includeFolders, buildProperties) + if l.verbose { + l.verboseStdoutFn(preprocStdout) + } + // Unwrap error and see if it is an ExitError. + if preprocErr == nil { + // Preprocessor successful, done + missingIncludeH = "" + } else if _, isExitErr := errors.Cause(preprocErr).(*exec.ExitError); !isExitErr || preprocStderr == nil { + // Ignore ExitErrors (e.g. gcc returning non-zero status), but bail out on other errors + return errors.WithStack(preprocErr) + } else { + missingIncludeH = IncludesFinderWithRegExp(string(preprocStderr)) + if missingIncludeH == "" && l.verbose { + l.verboseInfoFn(tr("Error while detecting libraries included by %[1]s", sourcePath)) + } + } + } + + if missingIncludeH == "" { + // No missing includes found, we're done + cache.ExpectEntry(sourcePath, "", nil) + return nil + } + + library := l.resolveLibrary(missingIncludeH, platformArch) + if library == nil { + // Library could not be resolved, show error + if preprocErr == nil || preprocStderr == nil { + // Filename came from cache, so run preprocessor to obtain error to show + var preprocStdout []byte + preprocStdout, preprocStderr, preprocErr = preprocessor.GCC(sourcePath, targetFilePath, includeFolders, buildProperties) + if l.verbose { + l.verboseStdoutFn(preprocStdout) + } + if preprocErr == nil { + // If there is a missing #include in the cache, but running + // gcc does not reproduce that, there is something wrong. + // Returning an error here will cause the cache to be + // deleted, so hopefully the next compilation will succeed. + return errors.New(tr("Internal error in cache")) + } + } + l.verboseStderrFn(preprocStderr) + return errors.WithStack(preprocErr) + } + + // Add this library to the list of libraries, the + // include path and queue its source files for further + // include scanning + l.AppendImportedLibraries(library) + l.AppendIncludeFolder(cache, sourcePath, missingIncludeH, library.SourceDir) + + if library.Precompiled && library.PrecompiledWithSources { + // Fully precompiled libraries should have no dependencies to avoid ABI breakage + if l.verbose { + l.verboseInfoFn(tr("Skipping dependencies detection for precompiled library %[1]s", library.Name)) + } + } else { + for _, sourceDir := range library.SourceDirs() { + l.queueSourceFilesFromFolder(sketchBuildPath, librariesBuildPath, sourceFileQueue, library, sourceDir.Dir, sourceDir.Recurse) + } + } + first = false + } +} + +func (l *SketchLibrariesDetector) queueSourceFilesFromFolder( + sketchBuildPath *paths.Path, + librariesBuildPath *paths.Path, + sourceFileQueue *UniqueSourceFileQueue, + origin interface{}, + folder *paths.Path, + recurse bool, +) error { + sourceFileExtensions := []string{} + for k := range globals.SourceFilesValidExtensions { + sourceFileExtensions = append(sourceFileExtensions, k) + } + filePaths, err := utils.FindFilesInFolder(folder, recurse, sourceFileExtensions...) + if err != nil { + return errors.WithStack(err) + } + + for _, filePath := range filePaths { + sourceFile, err := MakeSourceFile(sketchBuildPath, librariesBuildPath, origin, filePath) + if err != nil { + return errors.WithStack(err) + } + sourceFileQueue.Push(sourceFile) + } + + return nil +} + +func (l *SketchLibrariesDetector) failIfImportedLibraryIsWrong() error { + if len(l.importedLibraries) == 0 { + return nil + } + + for _, library := range l.importedLibraries { + if !library.IsLegacy { + if library.InstallDir.Join("arch").IsDir() { + return errors.New(tr("%[1]s folder is no longer supported! See %[2]s for more information", "'arch'", "http://goo.gl/gfFJzU")) + } + for _, propName := range libraries.MandatoryProperties { + if !library.Properties.ContainsKey(propName) { + return errors.New(tr("Missing '%[1]s' from library in %[2]s", propName, library.InstallDir)) + } + } + if library.Layout == libraries.RecursiveLayout { + if library.UtilityDir != nil { + return errors.New(tr("Library can't use both '%[1]s' and '%[2]s' folders. Double check in '%[3]s'.", "src", "utility", library.InstallDir)) + } + } + } + } + + return nil +} + +var INCLUDE_REGEXP = regexp.MustCompile("(?ms)^\\s*#[ \t]*include\\s*[<\"](\\S+)[\">]") + +func IncludesFinderWithRegExp(source string) string { + match := INCLUDE_REGEXP.FindStringSubmatch(source) + if match != nil { + return strings.TrimSpace(match[1]) + } + return findIncludeForOldCompilers(source) +} + +func findIncludeForOldCompilers(source string) string { + lines := strings.Split(source, "\n") + for _, line := range lines { + splittedLine := strings.Split(line, ":") + for i := range splittedLine { + if strings.Contains(splittedLine[i], "fatal error") { + return strings.TrimSpace(splittedLine[i+1]) + } + } + } + return "" +} + +type SourceFile struct { + // Path to the source file within the sketch/library root folder + relativePath *paths.Path + + // ExtraIncludePath contains an extra include path that must be + // used to compile this source file. + // This is mainly used for source files that comes from old-style libraries + // (Arduino IDE <1.5) requiring an extra include path to the "utility" folder. + extraIncludePath *paths.Path + + // The source root for the given origin, where its source files + // can be found. Prepending this to SourceFile.RelativePath will give + // the full path to that source file. + sourceRoot *paths.Path + + // The build root for the given origin, where build products will + // be placed. Any directories inside SourceFile.RelativePath will be + // appended here. + buildRoot *paths.Path +} + +func (f *SourceFile) Equals(g *SourceFile) bool { + return f.relativePath.EqualsTo(g.relativePath) && + f.buildRoot.EqualsTo(g.buildRoot) && + f.sourceRoot.EqualsTo(g.sourceRoot) +} + +// Create a SourceFile containing the given source file path within the +// given origin. The given path can be absolute, or relative within the +// origin's root source folder +func MakeSourceFile( + sketchBuildPath *paths.Path, + librariesBuildPath *paths.Path, + origin interface{}, + path *paths.Path, +) (*SourceFile, error) { + res := &SourceFile{} + + switch o := origin.(type) { + case *sketch.Sketch: + res.buildRoot = sketchBuildPath + res.sourceRoot = sketchBuildPath + case *libraries.Library: + res.buildRoot = librariesBuildPath.Join(o.DirName) + res.sourceRoot = o.SourceDir + res.extraIncludePath = o.UtilityDir + default: + panic("Unexpected origin for SourceFile: " + fmt.Sprint(origin)) + } + + if path.IsAbs() { + var err error + path, err = res.sourceRoot.RelTo(path) + if err != nil { + return nil, err + } + } + res.relativePath = path + return res, nil +} + +func (f *SourceFile) ExtraIncludePath() *paths.Path { + return f.extraIncludePath +} + +func (f *SourceFile) SourcePath() *paths.Path { + return f.sourceRoot.JoinPath(f.relativePath) +} + +func (f *SourceFile) ObjectPath() *paths.Path { + return f.buildRoot.Join(f.relativePath.String() + ".o") +} + +func (f *SourceFile) DepfilePath() *paths.Path { + return f.buildRoot.Join(f.relativePath.String() + ".d") +} + +// LibrariesLoader todo +func LibrariesLoader( + useCachedLibrariesResolution bool, + librariesManager *librariesmanager.LibrariesManager, + builtInLibrariesDirs *paths.Path, libraryDirs, otherLibrariesDirs paths.PathList, + actualPlatform, targetPlatform *cores.PlatformRelease, +) (*librariesmanager.LibrariesManager, *librariesresolver.Cpp, []byte, error) { + verboseOut := &bytes.Buffer{} + lm := librariesManager + if useCachedLibrariesResolution { + // Since we are using the cached libraries resolution + // the library manager is not needed. + lm = librariesmanager.NewLibraryManager(nil, nil) + } + if librariesManager == nil { + lm = librariesmanager.NewLibraryManager(nil, nil) + + builtInLibrariesFolders := builtInLibrariesDirs + if builtInLibrariesFolders != nil { + if err := builtInLibrariesFolders.ToAbs(); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + lm.AddLibrariesDir(builtInLibrariesFolders, libraries.IDEBuiltIn) + } + + if actualPlatform != targetPlatform { + lm.AddPlatformReleaseLibrariesDir(actualPlatform, libraries.ReferencedPlatformBuiltIn) + } + lm.AddPlatformReleaseLibrariesDir(targetPlatform, libraries.PlatformBuiltIn) + + librariesFolders := otherLibrariesDirs + if err := librariesFolders.ToAbs(); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + for _, folder := range librariesFolders { + lm.AddLibrariesDir(folder, libraries.User) + } + + for _, status := range lm.RescanLibraries() { + // With the refactoring of the initialization step of the CLI we changed how + // errors are returned when loading platforms and libraries, that meant returning a list of + // errors instead of a single one to enhance the experience for the user. + // I have no intention right now to start a refactoring of the legacy package too, so + // here's this shitty solution for now. + // When we're gonna refactor the legacy package this will be gone. + verboseOut.Write([]byte(status.Message())) + } + + for _, dir := range libraryDirs { + // Libraries specified this way have top priority + if err := lm.LoadLibraryFromDir(dir, libraries.Unmanaged); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + } + } + + resolver := librariesresolver.NewCppResolver() + if err := resolver.ScanIDEBuiltinLibraries(lm); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + if err := resolver.ScanUserAndUnmanagedLibraries(lm); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + if err := resolver.ScanPlatformLibraries(lm, targetPlatform); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + if actualPlatform != targetPlatform { + if err := resolver.ScanPlatformLibraries(lm, actualPlatform); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + } + return lm, resolver, verboseOut.Bytes(), nil +} + +func (l *SketchLibrariesDetector) OnlyUpdateCompilationDatabase() bool { + return l.onlyUpdateCompilationDatabase +} + +type includeCacheEntry struct { + Sourcefile *paths.Path + Include string + Includepath *paths.Path +} + +func (entry *includeCacheEntry) String() string { + return fmt.Sprintf("SourceFile: %s; Include: %s; IncludePath: %s", + entry.Sourcefile, entry.Include, entry.Includepath) +} + +func (entry *includeCacheEntry) Equals(other *includeCacheEntry) bool { + return entry.String() == other.String() +} + +type includeCache struct { + // Are the cache contents valid so far? + valid bool + // Index into entries of the next entry to be processed. Unused + // when the cache is invalid. + next int + entries []*includeCacheEntry +} + +// Return the next cache entry. Should only be called when the cache is +// valid and a next entry is available (the latter can be checked with +// ExpectFile). Does not advance the cache. +func (cache *includeCache) Next() *includeCacheEntry { + return cache.entries[cache.next] +} + +// Check that the next cache entry is about the given file. If it is +// not, or no entry is available, the cache is invalidated. Does not +// advance the cache. +func (cache *includeCache) ExpectFile(sourcefile *paths.Path) { + if cache.valid && (cache.next >= len(cache.entries) || !cache.Next().Sourcefile.EqualsTo(sourcefile)) { + cache.valid = false + cache.entries = cache.entries[:cache.next] + } +} + +// Check that the next entry matches the given values. If so, advance +// the cache. If not, the cache is invalidated. If the cache is +// invalidated, or was already invalid, an entry with the given values +// is appended. +func (cache *includeCache) ExpectEntry(sourcefile *paths.Path, include string, librarypath *paths.Path) { + entry := &includeCacheEntry{Sourcefile: sourcefile, Include: include, Includepath: librarypath} + if cache.valid { + if cache.next < len(cache.entries) && cache.Next().Equals(entry) { + cache.next++ + } else { + cache.valid = false + cache.entries = cache.entries[:cache.next] + } + } + + if !cache.valid { + cache.entries = append(cache.entries, entry) + } +} + +// Check that the cache is completely consumed. If not, the cache is +// invalidated. +func (cache *includeCache) ExpectEnd() { + if cache.valid && cache.next < len(cache.entries) { + cache.valid = false + cache.entries = cache.entries[:cache.next] + } +} + +// Read the cache from the given file +func readCache(path *paths.Path) *includeCache { + bytes, err := path.ReadFile() + if err != nil { + // Return an empty, invalid cache + return &includeCache{} + } + result := &includeCache{} + err = json.Unmarshal(bytes, &result.entries) + if err != nil { + // Return an empty, invalid cache + return &includeCache{} + } + result.valid = true + return result +} + +// Write the given cache to the given file if it is invalidated. If the +// cache is still valid, just update the timestamps of the file. +func writeCache(cache *includeCache, path *paths.Path) error { + // If the cache was still valid all the way, just touch its file + // (in case any source file changed without influencing the + // includes). If it was invalidated, overwrite the cache with + // the new contents. + if cache.valid { + path.Chtimes(time.Now(), time.Now()) + } else { + bytes, err := json.MarshalIndent(cache.entries, "", " ") + if err != nil { + return errors.WithStack(err) + } + err = path.WriteFile(bytes) + if err != nil { + return errors.WithStack(err) + } + } + return nil +} + +type UniqueSourceFileQueue []*SourceFile + +func (queue *UniqueSourceFileQueue) Push(value *SourceFile) { + if !queue.Contains(value) { + *queue = append(*queue, value) + } +} + +func (queue UniqueSourceFileQueue) Contains(target *SourceFile) bool { + return slices.ContainsFunc(queue, target.Equals) +} + +func (queue *UniqueSourceFileQueue) Pop() *SourceFile { + old := *queue + x := old[0] + *queue = old[1:] + return x +} + +func (queue UniqueSourceFileQueue) Empty() bool { + return len(queue) == 0 +} diff --git a/arduino/builder/libraries.go b/arduino/builder/libraries.go deleted file mode 100644 index 2f8219c5241..00000000000 --- a/arduino/builder/libraries.go +++ /dev/null @@ -1,240 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "bytes" - "fmt" - "strings" - "time" - - "github.com/arduino/arduino-cli/arduino/cores" - "github.com/arduino/arduino-cli/arduino/libraries" - "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" - "github.com/arduino/arduino-cli/arduino/libraries/librariesresolver" - "github.com/arduino/go-paths-helper" - "github.com/pkg/errors" -) - -type libraryResolutionResult struct { - Library *libraries.Library - NotUsedLibraries []*libraries.Library -} - -// SketchLibrariesDetector todo -type SketchLibrariesDetector struct { - librariesManager *librariesmanager.LibrariesManager - librariesResolver *librariesresolver.Cpp - useCachedLibrariesResolution bool - verbose bool - verboseInfoFn func(msg string) - verboseWarnFn func(msg string) - importedLibraries libraries.List - librariesResolutionResults map[string]libraryResolutionResult -} - -// NewSketchLibrariesDetector todo -func NewSketchLibrariesDetector( - lm *librariesmanager.LibrariesManager, - libsResolver *librariesresolver.Cpp, - verbose, useCachedLibrariesResolution bool, - verboseInfoFn func(msg string), - verboseWarnFn func(msg string), -) *SketchLibrariesDetector { - return &SketchLibrariesDetector{ - librariesManager: lm, - librariesResolver: libsResolver, - useCachedLibrariesResolution: useCachedLibrariesResolution, - librariesResolutionResults: map[string]libraryResolutionResult{}, - verbose: verbose, - verboseInfoFn: verboseInfoFn, - verboseWarnFn: verboseWarnFn, - importedLibraries: libraries.List{}, - } -} - -// ResolveLibrary todo -func (l *SketchLibrariesDetector) ResolveLibrary(header, platformArch string) *libraries.Library { - importedLibraries := l.importedLibraries - candidates := l.librariesResolver.AlternativesFor(header) - - if l.verbose { - l.verboseInfoFn(tr("Alternatives for %[1]s: %[2]s", header, candidates)) - l.verboseInfoFn(fmt.Sprintf("ResolveLibrary(%s)", header)) - l.verboseInfoFn(fmt.Sprintf(" -> %s: %s", tr("candidates"), candidates)) - } - - if len(candidates) == 0 { - return nil - } - - for _, candidate := range candidates { - if importedLibraries.Contains(candidate) { - return nil - } - } - - selected := l.librariesResolver.ResolveFor(header, platformArch) - if alreadyImported := importedLibraries.FindByName(selected.Name); alreadyImported != nil { - // Certain libraries might have the same name but be different. - // This usually happens when the user includes two or more custom libraries that have - // different header name but are stored in a parent folder with identical name, like - // ./libraries1/Lib/lib1.h and ./libraries2/Lib/lib2.h - // Without this check the library resolution would be stuck in a loop. - // This behaviour has been reported in this issue: - // https://github.com/arduino/arduino-cli/issues/973 - if selected == alreadyImported { - selected = alreadyImported - } - } - - candidates.Remove(selected) - l.librariesResolutionResults[header] = libraryResolutionResult{ - Library: selected, - NotUsedLibraries: candidates, - } - - return selected -} - -// ImportedLibraries todo -func (l *SketchLibrariesDetector) ImportedLibraries() libraries.List { - // TODO understand if we have to do a deepcopy - return l.importedLibraries -} - -// AppendImportedLibraries todo should rename this, probably after refactoring the -// container_find_includes command. -func (l *SketchLibrariesDetector) AppendImportedLibraries(library *libraries.Library) { - l.importedLibraries = append(l.importedLibraries, library) -} - -// UseCachedLibrariesResolution todo -func (l *SketchLibrariesDetector) UseCachedLibrariesResolution() bool { - return l.useCachedLibrariesResolution -} - -// PrintUsedAndNotUsedLibraries todo -func (l *SketchLibrariesDetector) PrintUsedAndNotUsedLibraries(sketchError bool) { - // Print this message: - // - as warning, when the sketch didn't compile - // - as info, when verbose is on - // - otherwise, output nothing - if !sketchError && !l.verbose { - return - } - - res := "" - for header, libResResult := range l.librariesResolutionResults { - if len(libResResult.NotUsedLibraries) == 0 { - continue - } - res += fmt.Sprintln(tr(`Multiple libraries were found for "%[1]s"`, header)) - res += fmt.Sprintln(" " + tr("Used: %[1]s", libResResult.Library.InstallDir)) - for _, notUsedLibrary := range libResResult.NotUsedLibraries { - res += fmt.Sprintln(" " + tr("Not used: %[1]s", notUsedLibrary.InstallDir)) - } - } - res = strings.TrimSpace(res) - if sketchError { - l.verboseWarnFn(res) - } else { - l.verboseInfoFn(res) - } - // todo why?? should we remove this? - time.Sleep(100 * time.Millisecond) -} - -// AppendIncludeFolder todo should rename this, probably after refactoring the -// container_find_includes command. -//func (l *SketchLibrariesDetector) AppendIncludeFolder(ctx *types.Context, cache *includeCache, sourceFilePath *paths.Path, include string, folder *paths.Path) { -// ctx.IncludeFolders = append(ctx.IncludeFolders, folder) -// cache.ExpectEntry(sourceFilePath, include, folder) -//} - -// LibrariesLoader todo -func LibrariesLoader( - useCachedLibrariesResolution bool, - librariesManager *librariesmanager.LibrariesManager, - builtInLibrariesDirs *paths.Path, libraryDirs, otherLibrariesDirs paths.PathList, - actualPlatform, targetPlatform *cores.PlatformRelease, -) (*librariesmanager.LibrariesManager, *librariesresolver.Cpp, []byte, error) { - verboseOut := &bytes.Buffer{} - lm := librariesManager - if useCachedLibrariesResolution { - // Since we are using the cached libraries resolution - // the library manager is not needed. - lm = librariesmanager.NewLibraryManager(nil, nil) - } - if librariesManager == nil { - lm = librariesmanager.NewLibraryManager(nil, nil) - - builtInLibrariesFolders := builtInLibrariesDirs - if builtInLibrariesFolders != nil { - if err := builtInLibrariesFolders.ToAbs(); err != nil { - return nil, nil, nil, errors.WithStack(err) - } - lm.AddLibrariesDir(builtInLibrariesFolders, libraries.IDEBuiltIn) - } - - if actualPlatform != targetPlatform { - lm.AddPlatformReleaseLibrariesDir(actualPlatform, libraries.ReferencedPlatformBuiltIn) - } - lm.AddPlatformReleaseLibrariesDir(targetPlatform, libraries.PlatformBuiltIn) - - librariesFolders := otherLibrariesDirs - if err := librariesFolders.ToAbs(); err != nil { - return nil, nil, nil, errors.WithStack(err) - } - for _, folder := range librariesFolders { - lm.AddLibrariesDir(folder, libraries.User) - } - - for _, status := range lm.RescanLibraries() { - // With the refactoring of the initialization step of the CLI we changed how - // errors are returned when loading platforms and libraries, that meant returning a list of - // errors instead of a single one to enhance the experience for the user. - // I have no intention right now to start a refactoring of the legacy package too, so - // here's this shitty solution for now. - // When we're gonna refactor the legacy package this will be gone. - verboseOut.Write([]byte(status.Message())) - } - - for _, dir := range libraryDirs { - // Libraries specified this way have top priority - if err := lm.LoadLibraryFromDir(dir, libraries.Unmanaged); err != nil { - return nil, nil, nil, errors.WithStack(err) - } - } - } - - resolver := librariesresolver.NewCppResolver() - if err := resolver.ScanIDEBuiltinLibraries(lm); err != nil { - return nil, nil, nil, errors.WithStack(err) - } - if err := resolver.ScanUserAndUnmanagedLibraries(lm); err != nil { - return nil, nil, nil, errors.WithStack(err) - } - if err := resolver.ScanPlatformLibraries(lm, targetPlatform); err != nil { - return nil, nil, nil, errors.WithStack(err) - } - if actualPlatform != targetPlatform { - if err := resolver.ScanPlatformLibraries(lm, actualPlatform); err != nil { - return nil, nil, nil, errors.WithStack(err) - } - } - return lm, resolver, verboseOut.Bytes(), nil -} diff --git a/arduino/builder/preprocessor/arduino_preprocessor.go b/arduino/builder/preprocessor/arduino_preprocessor.go index 8629ec211ba..a81d4bff612 100644 --- a/arduino/builder/preprocessor/arduino_preprocessor.go +++ b/arduino/builder/preprocessor/arduino_preprocessor.go @@ -21,9 +21,9 @@ import ( "path/filepath" "runtime" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/executils" - "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" diff --git a/arduino/builder/preprocessor/gcc.go b/arduino/builder/preprocessor/gcc.go index 476c1f2f5b4..9f63eaa93df 100644 --- a/arduino/builder/preprocessor/gcc.go +++ b/arduino/builder/preprocessor/gcc.go @@ -22,7 +22,7 @@ import ( "github.com/arduino/arduino-cli/executils" f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/legacy/builder/utils" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" diff --git a/arduino/builder/utils/utils.go b/arduino/builder/utils/utils.go new file mode 100644 index 00000000000..64c1a54dbdd --- /dev/null +++ b/arduino/builder/utils/utils.go @@ -0,0 +1,185 @@ +package utils + +import ( + "os" + "strings" + "unicode" + + f "github.com/arduino/arduino-cli/internal/algorithms" + "github.com/arduino/go-paths-helper" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/text/runes" + "golang.org/x/text/transform" + "golang.org/x/text/unicode/norm" +) + +func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool, error) { + logrus.Debugf("Checking previous results for %v (result = %v, dep = %v)", sourceFile, objectFile, dependencyFile) + if objectFile == nil || dependencyFile == nil { + logrus.Debugf("Not found: nil") + return false, nil + } + + sourceFile = sourceFile.Clean() + sourceFileStat, err := sourceFile.Stat() + if err != nil { + return false, errors.WithStack(err) + } + + objectFile = objectFile.Clean() + objectFileStat, err := objectFile.Stat() + if err != nil { + if os.IsNotExist(err) { + logrus.Debugf("Not found: %v", objectFile) + return false, nil + } else { + return false, errors.WithStack(err) + } + } + + dependencyFile = dependencyFile.Clean() + dependencyFileStat, err := dependencyFile.Stat() + if err != nil { + if os.IsNotExist(err) { + logrus.Debugf("Not found: %v", dependencyFile) + return false, nil + } else { + return false, errors.WithStack(err) + } + } + + if sourceFileStat.ModTime().After(objectFileStat.ModTime()) { + logrus.Debugf("%v newer than %v", sourceFile, objectFile) + return false, nil + } + if sourceFileStat.ModTime().After(dependencyFileStat.ModTime()) { + logrus.Debugf("%v newer than %v", sourceFile, dependencyFile) + return false, nil + } + + rows, err := dependencyFile.ReadFileAsLines() + if err != nil { + return false, errors.WithStack(err) + } + + rows = f.Map(rows, removeEndingBackSlash) + rows = f.Map(rows, strings.TrimSpace) + rows = f.Map(rows, unescapeDep) + rows = f.Filter(rows, f.NotEquals("")) + + if len(rows) == 0 { + return true, nil + } + + firstRow := rows[0] + if !strings.HasSuffix(firstRow, ":") { + logrus.Debugf("No colon in first line of depfile") + return false, nil + } + objFileInDepFile := firstRow[:len(firstRow)-1] + if objFileInDepFile != objectFile.String() { + logrus.Debugf("Depfile is about different file: %v", objFileInDepFile) + return false, nil + } + + // The first line of the depfile contains the path to the object file to generate. + // The second line of the depfile contains the path to the source file. + // All subsequent lines contain the header files necessary to compile the object file. + + // If we don't do this check it might happen that trying to compile a source file + // that has the same name but a different path wouldn't recreate the object file. + if sourceFile.String() != strings.Trim(rows[1], " ") { + return false, nil + } + + rows = rows[1:] + for _, row := range rows { + depStat, err := os.Stat(row) + if err != nil && !os.IsNotExist(err) { + // There is probably a parsing error of the dep file + // Ignore the error and trigger a full rebuild anyway + logrus.WithError(err).Debugf("Failed to read: %v", row) + return false, nil + } + if os.IsNotExist(err) { + logrus.Debugf("Not found: %v", row) + return false, nil + } + if depStat.ModTime().After(objectFileStat.ModTime()) { + logrus.Debugf("%v newer than %v", row, objectFile) + return false, nil + } + } + + return true, nil +} + +func removeEndingBackSlash(s string) string { + return strings.TrimSuffix(s, "\\") +} + +func unescapeDep(s string) string { + s = strings.ReplaceAll(s, "\\ ", " ") + s = strings.ReplaceAll(s, "\\\t", "\t") + s = strings.ReplaceAll(s, "\\#", "#") + s = strings.ReplaceAll(s, "$$", "$") + s = strings.ReplaceAll(s, "\\\\", "\\") + return s +} + +// Normalizes an UTF8 byte slice +// TODO: use it more often troughout all the project (maybe on logger interface?) +func NormalizeUTF8(buf []byte) []byte { + t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC) + result, _, _ := transform.Bytes(t, buf) + return result +} + +var SOURCE_CONTROL_FOLDERS = map[string]bool{"CVS": true, "RCS": true, ".git": true, ".github": true, ".svn": true, ".hg": true, ".bzr": true, ".vscode": true, ".settings": true, ".pioenvs": true, ".piolibdeps": true} + +// FilterOutSCCS is a ReadDirFilter that excludes known VSC or project files +func FilterOutSCCS(file *paths.Path) bool { + return !SOURCE_CONTROL_FOLDERS[file.Base()] +} + +// FilterReadableFiles is a ReadDirFilter that accepts only readable files +func FilterReadableFiles(file *paths.Path) bool { + // See if the file is readable by opening it + f, err := file.Open() + if err != nil { + return false + } + f.Close() + return true +} + +// FilterOutHiddenFiles is a ReadDirFilter that exclude files with a "." prefix in their name +var FilterOutHiddenFiles = paths.FilterOutPrefixes(".") + +func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (paths.PathList, error) { + fileFilter := paths.AndFilter( + FilterOutHiddenFiles, + FilterOutSCCS, + paths.FilterOutDirectories(), + FilterReadableFiles, + ) + if len(extensions) > 0 { + fileFilter = paths.AndFilter( + paths.FilterSuffixes(extensions...), + fileFilter, + ) + } + if recurse { + dirFilter := paths.AndFilter( + FilterOutHiddenFiles, + FilterOutSCCS, + ) + return dir.ReadDirRecursiveFiltered(dirFilter, fileFilter) + } + return dir.ReadDir(fileFilter) +} + +func WrapWithHyphenI(value string) string { + return "\"-I" + value + "\"" +} diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 96f30a83f39..1daa6f31714 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -24,6 +24,7 @@ import ( "github.com/arduino/arduino-cli/arduino" bldr "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/detector" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" "github.com/arduino/arduino-cli/arduino/sketch" @@ -254,7 +255,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream libsManager = lm } useCachedLibrariesResolution := req.GetSkipLibrariesDiscovery() - libsManager, libsResolver, verboseOut, err := bldr.LibrariesLoader( + libsManager, libsResolver, verboseOut, err := detector.LibrariesLoader( useCachedLibrariesResolution, libsManager, builderCtx.BuiltInLibrariesDirs, builderCtx.LibraryDirs, builderCtx.OtherLibrariesDirs, builderCtx.ActualPlatform, builderCtx.TargetPlatform, @@ -267,12 +268,15 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx.Warn(string(verboseOut)) } - builderCtx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( + builderCtx.SketchLibrariesDetector = detector.NewSketchLibrariesDetector( libsManager, libsResolver, builderCtx.Verbose, useCachedLibrariesResolution, + req.GetCreateCompilationDatabaseOnly(), func(msg string) { builderCtx.Info(msg) }, func(msg string) { builderCtx.Warn(msg) }, + func(data []byte) { builderCtx.WriteStdout(data) }, + func(data []byte) { builderCtx.WriteStderr(data) }, ) defer func() { diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index 8a8c8253d26..c9336d24528 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -52,7 +52,7 @@ func (s *Builder) Run(ctx *types.Context) error { }), utils.LogIfVerbose(false, tr("Detecting libraries used...")), - &ContainerFindIncludes{}, + findIncludes(ctx), &WarnAboutArchIncompatibleLibraries{}, @@ -144,7 +144,7 @@ func PreprocessSketch(ctx *types.Context) error { preprocessorImpl = preprocessor.PreprocessSketchWithArduinoPreprocessor } normalOutput, verboseOutput, err := preprocessorImpl( - ctx.Sketch, ctx.BuildPath, ctx.IncludeFolders, ctx.LineOffset, + ctx.Sketch, ctx.BuildPath, ctx.SketchLibrariesDetector.IncludeFolders(), ctx.LineOffset, ctx.BuildProperties, ctx.OnlyUpdateCompilationDatabase) if ctx.Verbose { ctx.WriteStdout(verboseOutput) @@ -172,7 +172,7 @@ func (s *Preprocess) Run(ctx *types.Context) error { return _err }), - &ContainerFindIncludes{}, + findIncludes(ctx), &WarnAboutArchIncompatibleLibraries{}, @@ -220,3 +220,19 @@ func RunPreprocess(ctx *types.Context) error { command := Preprocess{} return command.Run(ctx) } + +func findIncludes(ctx *types.Context) types.BareCommand { + return types.BareCommand(func(ctx *types.Context) error { + ctx.SketchLibrariesDetector.FindIncludes( + ctx.BuildPath, + ctx.BuildProperties.GetPath("build.core.path"), + ctx.BuildProperties.GetPath("build.variant.path"), + ctx.SketchBuildPath, + ctx.Sketch, + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.TargetPlatform.Platform.Architecture, + ) + return nil + }) +} diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go index f7cd0348c55..a827e496dac 100644 --- a/legacy/builder/builder_utils/utils.go +++ b/legacy/builder/builder_utils/utils.go @@ -24,16 +24,15 @@ import ( "strings" "sync" + bUtils "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/globals" "github.com/arduino/arduino-cli/i18n" - f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" - "github.com/sirupsen/logrus" ) var tr = i18n.Tr @@ -48,7 +47,7 @@ func DirContentIsOlderThan(dir *paths.Path, target *paths.Path, extensions ...st } targetModTime := targetStat.ModTime() - files, err := utils.FindFilesInFolder(dir, true, extensions...) + files, err := bUtils.FindFilesInFolder(dir, true, extensions...) if err != nil { return false, err } @@ -78,7 +77,7 @@ func compileFiles(ctx *types.Context, sourceDir *paths.Path, recurse bool, build validExtensions = append(validExtensions, ext) } - sources, err := utils.FindFilesInFolder(sourceDir, recurse, validExtensions...) + sources, err := bUtils.FindFilesInFolder(sourceDir, recurse, validExtensions...) if err != nil { return nil, err } @@ -169,7 +168,7 @@ func compileFileWithRecipe(ctx *types.Context, sourcePath *paths.Path, source *p return nil, errors.WithStack(err) } - objIsUpToDate, err := ObjFileIsUpToDate(source, objectFile, depsFile) + objIsUpToDate, err := bUtils.ObjFileIsUpToDate(source, objectFile, depsFile) if err != nil { return nil, errors.WithStack(err) } @@ -205,120 +204,6 @@ func compileFileWithRecipe(ctx *types.Context, sourcePath *paths.Path, source *p return objectFile, nil } -func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool, error) { - logrus.Debugf("Checking previous results for %v (result = %v, dep = %v)", sourceFile, objectFile, dependencyFile) - if objectFile == nil || dependencyFile == nil { - logrus.Debugf("Not found: nil") - return false, nil - } - - sourceFile = sourceFile.Clean() - sourceFileStat, err := sourceFile.Stat() - if err != nil { - return false, errors.WithStack(err) - } - - objectFile = objectFile.Clean() - objectFileStat, err := objectFile.Stat() - if err != nil { - if os.IsNotExist(err) { - logrus.Debugf("Not found: %v", objectFile) - return false, nil - } else { - return false, errors.WithStack(err) - } - } - - dependencyFile = dependencyFile.Clean() - dependencyFileStat, err := dependencyFile.Stat() - if err != nil { - if os.IsNotExist(err) { - logrus.Debugf("Not found: %v", dependencyFile) - return false, nil - } else { - return false, errors.WithStack(err) - } - } - - if sourceFileStat.ModTime().After(objectFileStat.ModTime()) { - logrus.Debugf("%v newer than %v", sourceFile, objectFile) - return false, nil - } - if sourceFileStat.ModTime().After(dependencyFileStat.ModTime()) { - logrus.Debugf("%v newer than %v", sourceFile, dependencyFile) - return false, nil - } - - rows, err := dependencyFile.ReadFileAsLines() - if err != nil { - return false, errors.WithStack(err) - } - - rows = f.Map(rows, removeEndingBackSlash) - rows = f.Map(rows, strings.TrimSpace) - rows = f.Map(rows, unescapeDep) - rows = f.Filter(rows, f.NotEquals("")) - - if len(rows) == 0 { - return true, nil - } - - firstRow := rows[0] - if !strings.HasSuffix(firstRow, ":") { - logrus.Debugf("No colon in first line of depfile") - return false, nil - } - objFileInDepFile := firstRow[:len(firstRow)-1] - if objFileInDepFile != objectFile.String() { - logrus.Debugf("Depfile is about different file: %v", objFileInDepFile) - return false, nil - } - - // The first line of the depfile contains the path to the object file to generate. - // The second line of the depfile contains the path to the source file. - // All subsequent lines contain the header files necessary to compile the object file. - - // If we don't do this check it might happen that trying to compile a source file - // that has the same name but a different path wouldn't recreate the object file. - if sourceFile.String() != strings.Trim(rows[1], " ") { - return false, nil - } - - rows = rows[1:] - for _, row := range rows { - depStat, err := os.Stat(row) - if err != nil && !os.IsNotExist(err) { - // There is probably a parsing error of the dep file - // Ignore the error and trigger a full rebuild anyway - logrus.WithError(err).Debugf("Failed to read: %v", row) - return false, nil - } - if os.IsNotExist(err) { - logrus.Debugf("Not found: %v", row) - return false, nil - } - if depStat.ModTime().After(objectFileStat.ModTime()) { - logrus.Debugf("%v newer than %v", row, objectFile) - return false, nil - } - } - - return true, nil -} - -func unescapeDep(s string) string { - s = strings.Replace(s, "\\ ", " ", -1) - s = strings.Replace(s, "\\\t", "\t", -1) - s = strings.Replace(s, "\\#", "#", -1) - s = strings.Replace(s, "$$", "$", -1) - s = strings.Replace(s, "\\\\", "\\", -1) - return s -} - -func removeEndingBackSlash(s string) string { - return strings.TrimSuffix(s, "\\") -} - func ArchiveCompiledFiles(ctx *types.Context, buildPath *paths.Path, archiveFile *paths.Path, objectFilesToArchive paths.PathList, buildProperties *properties.Map) (*paths.Path, error) { archiveFilePath := buildPath.JoinPath(archiveFile) diff --git a/legacy/builder/container_find_includes.go b/legacy/builder/container_find_includes.go deleted file mode 100644 index b90c25d46d5..00000000000 --- a/legacy/builder/container_find_includes.go +++ /dev/null @@ -1,474 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -/* - -Include detection - -This code is responsible for figuring out what libraries the current -sketch needs an populates both Context.ImportedLibraries with a list of -Library objects, and Context.IncludeFolders with a list of folders to -put on the include path. - -Simply put, every #include in a source file pulls in the library that -provides that source file. This includes source files in the selected -libraries, so libraries can recursively include other libraries as well. - -To implement this, the gcc preprocessor is used. A queue is created -containing, at first, the source files in the sketch. Each of the files -in the queue is processed in turn by running the preprocessor on it. If -the preprocessor provides an error, the output is examined to see if the -error is a missing header file originating from a #include directive. - -The filename is extracted from that #include directive, and a library is -found that provides it. If multiple libraries provide the same file, one -is slected (how this selection works is not described here, see the -ResolveLibrary function for that). The library selected in this way is -added to the include path through Context.IncludeFolders and the list of -libraries to include in the link through Context.ImportedLibraries. - -Furthermore, all of the library source files are added to the queue, to -be processed as well. When the preprocessor completes without showing an -#include error, processing of the file is complete and it advances to -the next. When no library can be found for a included filename, an error -is shown and the process is aborted. - -Caching - -Since this process is fairly slow (requiring at least one invocation of -the preprocessor per source file), its results are cached. - -Just caching the complete result (i.e. the resulting list of imported -libraries) seems obvious, but such a cache is hard to invalidate. Making -a list of all the source and header files used to create the list and -check if any of them changed is probably feasible, but this would also -require caching the full list of libraries to invalidate the cache when -the include to library resolution might have a different result. Another -downside of a complete cache is that any changes requires re-running -everything, even if no includes were actually changed. - -Instead, caching happens by keeping a sort of "journal" of the steps in -the include detection, essentially tracing each file processed and each -include path entry added. The cache is used by retracing these steps: -The include detection process is executed normally, except that instead -of running the preprocessor, the include filenames are (when possible) -read from the cache. Then, the include file to library resolution is -again executed normally. The results are checked against the cache and -as long as the results match, the cache is considered valid. - -When a source file (or any of the files it includes, as indicated by the -.d file) is changed, the preprocessor is executed as normal for the -file, ignoring any includes from the cache. This does not, however, -invalidate the cache: If the results from the preprocessor match the -entries in the cache, the cache remains valid and can again be used for -the next (unchanged) file. - -The cache file uses the JSON format and contains a list of entries. Each -entry represents a discovered library and contains: - - Sourcefile: The source file that the include was found in - - Include: The included filename found - - Includepath: The addition to the include path - -There are also some special entries: - - When adding the initial include path entries, such as for the core - and variant paths. These are not discovered, so the Sourcefile and - Include fields will be empty. - - When a file contains no (more) missing includes, an entry with an - empty Include and IncludePath is generated. - -*/ - -package builder - -import ( - "encoding/json" - "fmt" - "os/exec" - "time" - - "github.com/arduino/arduino-cli/arduino/builder/preprocessor" - "github.com/arduino/arduino-cli/arduino/globals" - "github.com/arduino/arduino-cli/arduino/libraries" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" - "github.com/arduino/go-paths-helper" - "github.com/pkg/errors" -) - -type ContainerFindIncludes struct{} - -func (s *ContainerFindIncludes) Run(ctx *types.Context) error { - err := s.findIncludes(ctx) - if err != nil && ctx.OnlyUpdateCompilationDatabase { - ctx.Info( - fmt.Sprintf("%s: %s", - tr("An error occurred detecting libraries"), - tr("the compilation database may be incomplete or inaccurate"))) - return nil - } - return err -} - -func (s *ContainerFindIncludes) findIncludes(ctx *types.Context) error { - librariesResolutionCache := ctx.BuildPath.Join("libraries.cache") - if ctx.SketchLibrariesDetector.UseCachedLibrariesResolution() && librariesResolutionCache.Exist() { - if d, err := librariesResolutionCache.ReadFile(); err != nil { - return err - } else if err := json.Unmarshal(d, &ctx.IncludeFolders); err != nil { - return err - } - if ctx.Verbose { - ctx.Info("Using cached library discovery: " + librariesResolutionCache.String()) - } - return nil - } - - cachePath := ctx.BuildPath.Join("includes.cache") - cache := readCache(cachePath) - - appendIncludeFolder(ctx, cache, nil, "", ctx.BuildProperties.GetPath("build.core.path")) - if ctx.BuildProperties.Get("build.variant.path") != "" { - appendIncludeFolder(ctx, cache, nil, "", ctx.BuildProperties.GetPath("build.variant.path")) - } - - sourceFileQueue := &types.UniqueSourceFileQueue{} - - if !ctx.SketchLibrariesDetector.UseCachedLibrariesResolution() { - sketch := ctx.Sketch - mergedfile, err := types.MakeSourceFile(ctx, sketch, paths.New(sketch.MainFile.Base()+".cpp")) - if err != nil { - return errors.WithStack(err) - } - sourceFileQueue.Push(mergedfile) - - queueSourceFilesFromFolder(ctx, sourceFileQueue, sketch, ctx.SketchBuildPath, false /* recurse */) - srcSubfolderPath := ctx.SketchBuildPath.Join("src") - if srcSubfolderPath.IsDir() { - queueSourceFilesFromFolder(ctx, sourceFileQueue, sketch, srcSubfolderPath, true /* recurse */) - } - - for !sourceFileQueue.Empty() { - err := findIncludesUntilDone(ctx, cache, sourceFileQueue) - if err != nil { - cachePath.Remove() - return errors.WithStack(err) - } - } - - // Finalize the cache - cache.ExpectEnd() - if err := writeCache(cache, cachePath); err != nil { - return errors.WithStack(err) - } - } - - if err := failIfImportedLibraryIsWrong(ctx); err != nil { - return errors.WithStack(err) - } - - if d, err := json.Marshal(ctx.IncludeFolders); err != nil { - return err - } else if err := librariesResolutionCache.WriteFile(d); err != nil { - return err - } - - return nil -} - -// Append the given folder to the include path and match or append it to -// the cache. sourceFilePath and include indicate the source of this -// include (e.g. what #include line in what file it was resolved from) -// and should be the empty string for the default include folders, like -// the core or variant. -func appendIncludeFolder(ctx *types.Context, cache *includeCache, sourceFilePath *paths.Path, include string, folder *paths.Path) { - ctx.IncludeFolders = append(ctx.IncludeFolders, folder) - cache.ExpectEntry(sourceFilePath, include, folder) -} - -type includeCacheEntry struct { - Sourcefile *paths.Path - Include string - Includepath *paths.Path -} - -func (entry *includeCacheEntry) String() string { - return fmt.Sprintf("SourceFile: %s; Include: %s; IncludePath: %s", - entry.Sourcefile, entry.Include, entry.Includepath) -} - -func (entry *includeCacheEntry) Equals(other *includeCacheEntry) bool { - return entry.String() == other.String() -} - -type includeCache struct { - // Are the cache contents valid so far? - valid bool - // Index into entries of the next entry to be processed. Unused - // when the cache is invalid. - next int - entries []*includeCacheEntry -} - -// Return the next cache entry. Should only be called when the cache is -// valid and a next entry is available (the latter can be checked with -// ExpectFile). Does not advance the cache. -func (cache *includeCache) Next() *includeCacheEntry { - return cache.entries[cache.next] -} - -// Check that the next cache entry is about the given file. If it is -// not, or no entry is available, the cache is invalidated. Does not -// advance the cache. -func (cache *includeCache) ExpectFile(sourcefile *paths.Path) { - if cache.valid && (cache.next >= len(cache.entries) || !cache.Next().Sourcefile.EqualsTo(sourcefile)) { - cache.valid = false - cache.entries = cache.entries[:cache.next] - } -} - -// Check that the next entry matches the given values. If so, advance -// the cache. If not, the cache is invalidated. If the cache is -// invalidated, or was already invalid, an entry with the given values -// is appended. -func (cache *includeCache) ExpectEntry(sourcefile *paths.Path, include string, librarypath *paths.Path) { - entry := &includeCacheEntry{Sourcefile: sourcefile, Include: include, Includepath: librarypath} - if cache.valid { - if cache.next < len(cache.entries) && cache.Next().Equals(entry) { - cache.next++ - } else { - cache.valid = false - cache.entries = cache.entries[:cache.next] - } - } - - if !cache.valid { - cache.entries = append(cache.entries, entry) - } -} - -// Check that the cache is completely consumed. If not, the cache is -// invalidated. -func (cache *includeCache) ExpectEnd() { - if cache.valid && cache.next < len(cache.entries) { - cache.valid = false - cache.entries = cache.entries[:cache.next] - } -} - -// Read the cache from the given file -func readCache(path *paths.Path) *includeCache { - bytes, err := path.ReadFile() - if err != nil { - // Return an empty, invalid cache - return &includeCache{} - } - result := &includeCache{} - err = json.Unmarshal(bytes, &result.entries) - if err != nil { - // Return an empty, invalid cache - return &includeCache{} - } - result.valid = true - return result -} - -// Write the given cache to the given file if it is invalidated. If the -// cache is still valid, just update the timestamps of the file. -func writeCache(cache *includeCache, path *paths.Path) error { - // If the cache was still valid all the way, just touch its file - // (in case any source file changed without influencing the - // includes). If it was invalidated, overwrite the cache with - // the new contents. - if cache.valid { - path.Chtimes(time.Now(), time.Now()) - } else { - bytes, err := json.MarshalIndent(cache.entries, "", " ") - if err != nil { - return errors.WithStack(err) - } - err = path.WriteFile(bytes) - if err != nil { - return errors.WithStack(err) - } - } - return nil -} - -func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFileQueue *types.UniqueSourceFileQueue) error { - sourceFile := sourceFileQueue.Pop() - sourcePath := sourceFile.SourcePath() - targetFilePath := paths.NullPath() - depPath := sourceFile.DepfilePath() - objPath := sourceFile.ObjectPath() - - // TODO: This should perhaps also compare against the - // include.cache file timestamp. Now, it only checks if the file - // changed after the object file was generated, but if it - // changed between generating the cache and the object file, - // this could show the file as unchanged when it really is - // changed. Changing files during a build isn't really - // supported, but any problems from it should at least be - // resolved when doing another build, which is not currently the - // case. - // TODO: This reads the dependency file, but the actual building - // does it again. Should the result be somehow cached? Perhaps - // remove the object file if it is found to be stale? - unchanged, err := builder_utils.ObjFileIsUpToDate(sourcePath, objPath, depPath) - if err != nil { - return errors.WithStack(err) - } - - first := true - for { - cache.ExpectFile(sourcePath) - - // Libraries may require the "utility" directory to be added to the include - // search path, but only for the source code of the library, so we temporary - // copy the current search path list and add the library' utility directory - // if needed. - includeFolders := ctx.IncludeFolders - if extraInclude := sourceFile.ExtraIncludePath(); extraInclude != nil { - includeFolders = append(includeFolders, extraInclude) - } - - var preprocErr error - var preprocStderr []byte - - var missingIncludeH string - if unchanged && cache.valid { - missingIncludeH = cache.Next().Include - if first && ctx.Verbose { - ctx.Info(tr("Using cached library dependencies for file: %[1]s", sourcePath)) - } - } else { - var preprocStdout []byte - preprocStdout, preprocStderr, preprocErr = preprocessor.GCC(sourcePath, targetFilePath, includeFolders, ctx.BuildProperties) - if ctx.Verbose { - ctx.WriteStdout(preprocStdout) - } - // Unwrap error and see if it is an ExitError. - if preprocErr == nil { - // Preprocessor successful, done - missingIncludeH = "" - } else if _, isExitErr := errors.Cause(preprocErr).(*exec.ExitError); !isExitErr || preprocStderr == nil { - // Ignore ExitErrors (e.g. gcc returning non-zero status), but bail out on other errors - return errors.WithStack(preprocErr) - } else { - missingIncludeH = IncludesFinderWithRegExp(string(preprocStderr)) - if missingIncludeH == "" && ctx.Verbose { - ctx.Info(tr("Error while detecting libraries included by %[1]s", sourcePath)) - } - } - } - - if missingIncludeH == "" { - // No missing includes found, we're done - cache.ExpectEntry(sourcePath, "", nil) - return nil - } - - library := ResolveLibrary(ctx, missingIncludeH) - if library == nil { - // Library could not be resolved, show error - if preprocErr == nil || preprocStderr == nil { - // Filename came from cache, so run preprocessor to obtain error to show - var preprocStdout []byte - preprocStdout, preprocStderr, preprocErr = preprocessor.GCC(sourcePath, targetFilePath, includeFolders, ctx.BuildProperties) - if ctx.Verbose { - ctx.WriteStdout(preprocStdout) - } - if preprocErr == nil { - // If there is a missing #include in the cache, but running - // gcc does not reproduce that, there is something wrong. - // Returning an error here will cause the cache to be - // deleted, so hopefully the next compilation will succeed. - return errors.New(tr("Internal error in cache")) - } - } - ctx.WriteStderr(preprocStderr) - return errors.WithStack(preprocErr) - } - - // Add this library to the list of libraries, the - // include path and queue its source files for further - // include scanning - ctx.SketchLibrariesDetector.AppendImportedLibraries(library) - appendIncludeFolder(ctx, cache, sourcePath, missingIncludeH, library.SourceDir) - - if library.Precompiled && library.PrecompiledWithSources { - // Fully precompiled libraries should have no dependencies to avoid ABI breakage - if ctx.Verbose { - ctx.Info(tr("Skipping dependencies detection for precompiled library %[1]s", library.Name)) - } - } else { - for _, sourceDir := range library.SourceDirs() { - queueSourceFilesFromFolder(ctx, sourceFileQueue, library, sourceDir.Dir, sourceDir.Recurse) - } - } - first = false - } -} - -func queueSourceFilesFromFolder(ctx *types.Context, sourceFileQueue *types.UniqueSourceFileQueue, origin interface{}, folder *paths.Path, recurse bool) error { - sourceFileExtensions := []string{} - for k := range globals.SourceFilesValidExtensions { - sourceFileExtensions = append(sourceFileExtensions, k) - } - filePaths, err := utils.FindFilesInFolder(folder, recurse, sourceFileExtensions...) - if err != nil { - return errors.WithStack(err) - } - - for _, filePath := range filePaths { - sourceFile, err := types.MakeSourceFile(ctx, origin, filePath) - if err != nil { - return errors.WithStack(err) - } - sourceFileQueue.Push(sourceFile) - } - - return nil -} - -func ResolveLibrary(ctx *types.Context, header string) *libraries.Library { - return ctx.SketchLibrariesDetector.ResolveLibrary(header, ctx.TargetPlatform.Platform.Architecture) -} - -func failIfImportedLibraryIsWrong(ctx *types.Context) error { - if len(ctx.SketchLibrariesDetector.ImportedLibraries()) == 0 { - return nil - } - - for _, library := range ctx.SketchLibrariesDetector.ImportedLibraries() { - if !library.IsLegacy { - if library.InstallDir.Join("arch").IsDir() { - return errors.New(tr("%[1]s folder is no longer supported! See %[2]s for more information", "'arch'", "http://goo.gl/gfFJzU")) - } - for _, propName := range libraries.MandatoryProperties { - if !library.Properties.ContainsKey(propName) { - return errors.New(tr("Missing '%[1]s' from library in %[2]s", propName, library.InstallDir)) - } - } - if library.Layout == libraries.RecursiveLayout { - if library.UtilityDir != nil { - return errors.New(tr("Library can't use both '%[1]s' and '%[2]s' folders. Double check in '%[3]s'.", "src", "utility", library.InstallDir)) - } - } - } - } - - return nil -} diff --git a/legacy/builder/create_cmake_rule.go b/legacy/builder/create_cmake_rule.go index 37b60e92fac..45f68ef9ed4 100644 --- a/legacy/builder/create_cmake_rule.go +++ b/legacy/builder/create_cmake_rule.go @@ -26,11 +26,11 @@ import ( properties "github.com/arduino/go-properties-orderedmap" "golang.org/x/exp/slices" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/globals" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" ) type ExportProjectCMake struct { diff --git a/legacy/builder/includes_finder_with_regexp.go b/legacy/builder/includes_finder_with_regexp.go deleted file mode 100644 index 03017c67048..00000000000 --- a/legacy/builder/includes_finder_with_regexp.go +++ /dev/null @@ -1,44 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "regexp" - "strings" -) - -var INCLUDE_REGEXP = regexp.MustCompile("(?ms)^\\s*#[ \t]*include\\s*[<\"](\\S+)[\">]") - -func IncludesFinderWithRegExp(source string) string { - match := INCLUDE_REGEXP.FindStringSubmatch(source) - if match != nil { - return strings.TrimSpace(match[1]) - } - return findIncludeForOldCompilers(source) -} - -func findIncludeForOldCompilers(source string) string { - lines := strings.Split(source, "\n") - for _, line := range lines { - splittedLine := strings.Split(line, ":") - for i := range splittedLine { - if strings.Contains(splittedLine[i], "fatal error") { - return strings.TrimSpace(splittedLine[i+1]) - } - } - } - return "" -} diff --git a/legacy/builder/phases/core_builder.go b/legacy/builder/phases/core_builder.go index 5c94573f349..455588f7444 100644 --- a/legacy/builder/phases/core_builder.go +++ b/legacy/builder/phases/core_builder.go @@ -28,7 +28,7 @@ import ( "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" diff --git a/legacy/builder/phases/libraries_builder.go b/legacy/builder/phases/libraries_builder.go index 3d28fbf5520..88106c9a9de 100644 --- a/legacy/builder/phases/libraries_builder.go +++ b/legacy/builder/phases/libraries_builder.go @@ -18,12 +18,12 @@ package phases import ( "strings" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/libraries" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" @@ -37,7 +37,8 @@ type LibrariesBuilder struct{} func (s *LibrariesBuilder) Run(ctx *types.Context) error { librariesBuildPath := ctx.LibrariesBuildPath buildProperties := ctx.BuildProperties - includes := f.Map(ctx.IncludeFolders.AsStrings(), utils.WrapWithHyphenI) + includesFolders := ctx.SketchLibrariesDetector.IncludeFolders() + includes := f.Map(includesFolders.AsStrings(), utils.WrapWithHyphenI) libs := ctx.SketchLibrariesDetector.ImportedLibraries() if err := librariesBuildPath.MkdirAll(); err != nil { diff --git a/legacy/builder/phases/sketch_builder.go b/legacy/builder/phases/sketch_builder.go index 300cb6836af..398ba87de8b 100644 --- a/legacy/builder/phases/sketch_builder.go +++ b/legacy/builder/phases/sketch_builder.go @@ -16,10 +16,10 @@ package phases import ( + "github.com/arduino/arduino-cli/arduino/builder/utils" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/pkg/errors" ) @@ -28,7 +28,8 @@ type SketchBuilder struct{} func (s *SketchBuilder) Run(ctx *types.Context) error { sketchBuildPath := ctx.SketchBuildPath buildProperties := ctx.BuildProperties - includes := f.Map(ctx.IncludeFolders.AsStrings(), utils.WrapWithHyphenI) + includesFolders := ctx.SketchLibrariesDetector.IncludeFolders() + includes := f.Map(includesFolders.AsStrings(), utils.WrapWithHyphenI) if err := sketchBuildPath.MkdirAll(); err != nil { return errors.WithStack(err) diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index d4a6d4901a5..0f32c02e3c8 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -136,8 +136,11 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat lm, libsResolver, ctx.Verbose, false, + false, func(msg string) { ctx.Info(msg) }, func(msg string) { ctx.Warn(msg) }, + func(data []byte) { ctx.WriteStdout(data) }, + func(data []byte) { ctx.WriteStderr(data) }, ) } diff --git a/legacy/builder/test/unused_compiled_libraries_remover_test.go b/legacy/builder/test/unused_compiled_libraries_remover_test.go index 5c04b1cee10..ab49dc9e26f 100644 --- a/legacy/builder/test/unused_compiled_libraries_remover_test.go +++ b/legacy/builder/test/unused_compiled_libraries_remover_test.go @@ -38,7 +38,7 @@ func TestUnusedCompiledLibrariesRemover(t *testing.T) { ctx := &types.Context{} ctx.LibrariesBuildPath = temp ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( - nil, nil, false, false, nil, nil, + nil, nil, false, false, false, nil, nil, nil, nil, ) ctx.SketchLibrariesDetector.AppendImportedLibraries(&libraries.Library{Name: "Bridge"}) @@ -61,7 +61,7 @@ func TestUnusedCompiledLibrariesRemoverLibDoesNotExist(t *testing.T) { ctx := &types.Context{} ctx.LibrariesBuildPath = paths.TempDir().Join("test") ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( - nil, nil, false, false, nil, nil, + nil, nil, false, false, false, nil, nil, nil, nil, ) ctx.SketchLibrariesDetector.AppendImportedLibraries(&libraries.Library{Name: "Bridge"}) @@ -81,7 +81,7 @@ func TestUnusedCompiledLibrariesRemoverNoUsedLibraries(t *testing.T) { ctx := &types.Context{} ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( - nil, nil, false, false, nil, nil, + nil, nil, false, false, false, nil, nil, nil, nil, ) ctx.LibrariesBuildPath = temp diff --git a/legacy/builder/types/accessories.go b/legacy/builder/types/accessories.go deleted file mode 100644 index b4de7a1a18a..00000000000 --- a/legacy/builder/types/accessories.go +++ /dev/null @@ -1,41 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package types - -import "golang.org/x/exp/slices" - -type UniqueSourceFileQueue []*SourceFile - -func (queue *UniqueSourceFileQueue) Push(value *SourceFile) { - if !queue.Contains(value) { - *queue = append(*queue, value) - } -} - -func (queue UniqueSourceFileQueue) Contains(target *SourceFile) bool { - return slices.ContainsFunc(queue, target.Equals) -} - -func (queue *UniqueSourceFileQueue) Pop() *SourceFile { - old := *queue - x := old[0] - *queue = old[1:] - return x -} - -func (queue UniqueSourceFileQueue) Empty() bool { - return len(queue) == 0 -} diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index a78530387e9..a6570bf3268 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -23,6 +23,7 @@ import ( "sync" "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/detector" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" "github.com/arduino/arduino-cli/arduino/sketch" @@ -62,7 +63,7 @@ func (p *ProgressStruct) CompleteStep() { // Context structure type Context struct { Builder *builder.Builder - SketchLibrariesDetector *builder.SketchLibrariesDetector + SketchLibrariesDetector *detector.SketchLibrariesDetector // Build options HardwareDirs paths.PathList @@ -100,9 +101,6 @@ type Context struct { Sketch *sketch.Sketch WarningsLevel string - // Libraries handling - IncludeFolders paths.PathList - // C++ Parsing LineOffset int diff --git a/legacy/builder/types/types.go b/legacy/builder/types/types.go index d84864eecfe..8678fd2fe51 100644 --- a/legacy/builder/types/types.go +++ b/legacy/builder/types/types.go @@ -15,86 +15,6 @@ package types -import ( - "fmt" - - "github.com/arduino/arduino-cli/arduino/libraries" - "github.com/arduino/arduino-cli/arduino/sketch" - paths "github.com/arduino/go-paths-helper" -) - -type SourceFile struct { - // Path to the source file within the sketch/library root folder - relativePath *paths.Path - - // ExtraIncludePath contains an extra include path that must be - // used to compile this source file. - // This is mainly used for source files that comes from old-style libraries - // (Arduino IDE <1.5) requiring an extra include path to the "utility" folder. - extraIncludePath *paths.Path - - // The source root for the given origin, where its source files - // can be found. Prepending this to SourceFile.RelativePath will give - // the full path to that source file. - sourceRoot *paths.Path - - // The build root for the given origin, where build products will - // be placed. Any directories inside SourceFile.RelativePath will be - // appended here. - buildRoot *paths.Path -} - -func (f *SourceFile) Equals(g *SourceFile) bool { - return f.relativePath.EqualsTo(g.relativePath) && - f.buildRoot.EqualsTo(g.buildRoot) && - f.sourceRoot.EqualsTo(g.sourceRoot) -} - -// Create a SourceFile containing the given source file path within the -// given origin. The given path can be absolute, or relative within the -// origin's root source folder -func MakeSourceFile(ctx *Context, origin interface{}, path *paths.Path) (*SourceFile, error) { - res := &SourceFile{} - - switch o := origin.(type) { - case *sketch.Sketch: - res.buildRoot = ctx.SketchBuildPath - res.sourceRoot = ctx.SketchBuildPath - case *libraries.Library: - res.buildRoot = ctx.LibrariesBuildPath.Join(o.DirName) - res.sourceRoot = o.SourceDir - res.extraIncludePath = o.UtilityDir - default: - panic("Unexpected origin for SourceFile: " + fmt.Sprint(origin)) - } - - if path.IsAbs() { - var err error - path, err = res.sourceRoot.RelTo(path) - if err != nil { - return nil, err - } - } - res.relativePath = path - return res, nil -} - -func (f *SourceFile) ExtraIncludePath() *paths.Path { - return f.extraIncludePath -} - -func (f *SourceFile) SourcePath() *paths.Path { - return f.sourceRoot.JoinPath(f.relativePath) -} - -func (f *SourceFile) ObjectPath() *paths.Path { - return f.buildRoot.Join(f.relativePath.String() + ".o") -} - -func (f *SourceFile) DepfilePath() *paths.Path { - return f.buildRoot.Join(f.relativePath.String() + ".d") -} - type Command interface { Run(ctx *Context) error } diff --git a/legacy/builder/utils/utils.go b/legacy/builder/utils/utils.go index 102b007883f..1cddbd3bb48 100644 --- a/legacy/builder/utils/utils.go +++ b/legacy/builder/utils/utils.go @@ -20,42 +20,12 @@ import ( "os" "os/exec" "strings" - "unicode" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/types" - paths "github.com/arduino/go-paths-helper" "github.com/pkg/errors" - "golang.org/x/text/runes" - "golang.org/x/text/transform" - "golang.org/x/text/unicode/norm" ) -var SOURCE_CONTROL_FOLDERS = map[string]bool{"CVS": true, "RCS": true, ".git": true, ".github": true, ".svn": true, ".hg": true, ".bzr": true, ".vscode": true, ".settings": true, ".pioenvs": true, ".piolibdeps": true} - -// FilterOutHiddenFiles is a ReadDirFilter that exclude files with a "." prefix in their name -var FilterOutHiddenFiles = paths.FilterOutPrefixes(".") - -// FilterOutSCCS is a ReadDirFilter that excludes known VSC or project files -func FilterOutSCCS(file *paths.Path) bool { - return !SOURCE_CONTROL_FOLDERS[file.Base()] -} - -// FilterReadableFiles is a ReadDirFilter that accepts only readable files -func FilterReadableFiles(file *paths.Path) bool { - // See if the file is readable by opening it - f, err := file.Open() - if err != nil { - return false - } - f.Close() - return true -} - -func WrapWithHyphenI(value string) string { - return "\"-I" + value + "\"" -} - func printableArgument(arg string) string { if strings.ContainsAny(arg, "\"\\ \t") { arg = strings.Replace(arg, "\\", "\\\\", -1) @@ -126,29 +96,6 @@ func ExecCommand(ctx *types.Context, command *exec.Cmd, stdout int, stderr int) return outbytes, errbytes, errors.WithStack(err) } -func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (paths.PathList, error) { - fileFilter := paths.AndFilter( - FilterOutHiddenFiles, - FilterOutSCCS, - paths.FilterOutDirectories(), - FilterReadableFiles, - ) - if len(extensions) > 0 { - fileFilter = paths.AndFilter( - paths.FilterSuffixes(extensions...), - fileFilter, - ) - } - if recurse { - dirFilter := paths.AndFilter( - FilterOutHiddenFiles, - FilterOutSCCS, - ) - return dir.ReadDirRecursiveFiltered(dirFilter, fileFilter) - } - return dir.ReadDir(fileFilter) -} - type loggerAction struct { onlyIfVerbose bool warn bool @@ -169,11 +116,3 @@ func (l *loggerAction) Run(ctx *types.Context) error { func LogIfVerbose(warn bool, msg string) types.Command { return &loggerAction{onlyIfVerbose: true, warn: warn, msg: msg} } - -// Normalizes an UTF8 byte slice -// TODO: use it more often troughout all the project (maybe on logger interface?) -func NormalizeUTF8(buf []byte) []byte { - t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC) - result, _, _ := transform.Bytes(t, buf) - return result -} From ea37d5477e9f68776e855ff71cedd2fae4e0718a Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Mon, 4 Sep 2023 17:29:42 +0200 Subject: [PATCH 27/29] refactoring the cmd.Exec in favour of executils --- arduino/builder/compilation_database.go | 8 ++--- arduino/builder/compilation_database_test.go | 5 +-- arduino/builder/cpp/cpp.go | 4 +++ arduino/builder/preprocessor/gcc.go | 4 +-- arduino/builder/utils/utils.go | 3 -- executils/process.go | 10 ++++++ legacy/builder/builder_utils/utils.go | 26 +++++++++------ legacy/builder/create_cmake_rule.go | 4 +-- legacy/builder/phases/core_builder.go | 4 +-- legacy/builder/phases/libraries_builder.go | 12 +++---- legacy/builder/phases/linker.go | 6 ++-- legacy/builder/phases/sizer.go | 4 +-- legacy/builder/phases/sketch_builder.go | 4 +-- legacy/builder/recipe_runner.go | 4 +-- legacy/builder/utils/utils.go | 33 +++++++------------- 15 files changed, 70 insertions(+), 61 deletions(-) diff --git a/arduino/builder/compilation_database.go b/arduino/builder/compilation_database.go index 9d82cb88382..6a7e2e475dd 100644 --- a/arduino/builder/compilation_database.go +++ b/arduino/builder/compilation_database.go @@ -19,8 +19,8 @@ import ( "encoding/json" "fmt" "os" - "os/exec" + "github.com/arduino/arduino-cli/executils" "github.com/arduino/go-paths-helper" ) @@ -68,8 +68,8 @@ func (db *CompilationDatabase) SaveToFile() { } // Add adds a new CompilationDatabase entry -func (db *CompilationDatabase) Add(target *paths.Path, command *exec.Cmd) { - commandDir := command.Dir +func (db *CompilationDatabase) Add(target *paths.Path, command *executils.Process) { + commandDir := command.GetDir() if commandDir == "" { // This mimics what Cmd.Run also does: Use Dir if specified, // current directory otherwise @@ -82,7 +82,7 @@ func (db *CompilationDatabase) Add(target *paths.Path, command *exec.Cmd) { entry := CompilationCommand{ Directory: commandDir, - Arguments: command.Args, + Arguments: command.GetArgs(), File: target.String(), } diff --git a/arduino/builder/compilation_database_test.go b/arduino/builder/compilation_database_test.go index c7f5bc3003d..8a715533617 100644 --- a/arduino/builder/compilation_database_test.go +++ b/arduino/builder/compilation_database_test.go @@ -16,9 +16,9 @@ package builder import ( - "os/exec" "testing" + "github.com/arduino/arduino-cli/executils" "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" ) @@ -28,7 +28,8 @@ func TestCompilationDatabase(t *testing.T) { require.NoError(t, err) defer tmpfile.Remove() - cmd := exec.Command("gcc", "arg1", "arg2") + cmd, err := executils.NewProcess(nil, "gcc", "arg1", "arg2") + require.NoError(t, err) db := NewCompilationDatabase(tmpfile) db.Add(paths.New("test"), cmd) db.SaveToFile() diff --git a/arduino/builder/cpp/cpp.go b/arduino/builder/cpp/cpp.go index 71c2b696702..4eaa4fafcd4 100644 --- a/arduino/builder/cpp/cpp.go +++ b/arduino/builder/cpp/cpp.go @@ -106,3 +106,7 @@ func ParseString(line string) (string, string, bool) { i += width } } + +func WrapWithHyphenI(value string) string { + return "\"-I" + value + "\"" +} diff --git a/arduino/builder/preprocessor/gcc.go b/arduino/builder/preprocessor/gcc.go index 9f63eaa93df..2029f08b972 100644 --- a/arduino/builder/preprocessor/gcc.go +++ b/arduino/builder/preprocessor/gcc.go @@ -20,9 +20,9 @@ import ( "fmt" "strings" + "github.com/arduino/arduino-cli/arduino/builder/cpp" "github.com/arduino/arduino-cli/executils" f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" @@ -38,7 +38,7 @@ func GCC(sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths. gccBuildProperties.SetPath("source_file", sourceFilePath) gccBuildProperties.SetPath("preprocessed_file_path", targetFilePath) - includesStrings := f.Map(includes.AsStrings(), utils.WrapWithHyphenI) + includesStrings := f.Map(includes.AsStrings(), cpp.WrapWithHyphenI) gccBuildProperties.Set("includes", strings.Join(includesStrings, " ")) const gccPreprocRecipeProperty = "recipe.preproc.macros" diff --git a/arduino/builder/utils/utils.go b/arduino/builder/utils/utils.go index 64c1a54dbdd..b1ee7b0bac8 100644 --- a/arduino/builder/utils/utils.go +++ b/arduino/builder/utils/utils.go @@ -180,6 +180,3 @@ func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (pat return dir.ReadDir(fileFilter) } -func WrapWithHyphenI(value string) string { - return "\"-I" + value + "\"" -} diff --git a/executils/process.go b/executils/process.go index 7e0ac135089..137d2f86bc3 100644 --- a/executils/process.go +++ b/executils/process.go @@ -137,6 +137,11 @@ func (p *Process) SetDir(dir string) { p.cmd.Dir = dir } +// GetDir gets the working directory of the command. +func (p *Process) GetDir() string { + return p.cmd.Dir +} + // SetDirFromPath sets the working directory of the command. If path is nil, Run // runs the command in the calling process's current directory. func (p *Process) SetDirFromPath(path *paths.Path) { @@ -187,3 +192,8 @@ func (p *Process) RunAndCaptureOutput(ctx context.Context) ([]byte, []byte, erro err := p.RunWithinContext(ctx) return stdout.Bytes(), stderr.Bytes(), err } + +// GetArgs returns the command arguments +func (p *Process) GetArgs() []string { + return p.cmd.Args +} diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go index a827e496dac..1d53860b163 100644 --- a/legacy/builder/builder_utils/utils.go +++ b/legacy/builder/builder_utils/utils.go @@ -18,7 +18,6 @@ package builder_utils import ( "fmt" "os" - "os/exec" "path/filepath" "runtime" "strings" @@ -26,6 +25,7 @@ import ( bUtils "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/globals" + "github.com/arduino/arduino-cli/executils" "github.com/arduino/arduino-cli/i18n" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" @@ -173,7 +173,7 @@ func compileFileWithRecipe(ctx *types.Context, sourcePath *paths.Path, source *p return nil, errors.WithStack(err) } - command, err := PrepareCommandForRecipe(properties, recipe, false, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, err := PrepareCommandForRecipe(properties, recipe, false) if err != nil { return nil, errors.WithStack(err) } @@ -244,7 +244,7 @@ func ArchiveCompiledFiles(ctx *types.Context, buildPath *paths.Path, archiveFile properties.SetPath(constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH, archiveFilePath) properties.SetPath(constants.BUILD_PROPERTIES_OBJECT_FILE, objectFile) - command, err := PrepareCommandForRecipe(properties, constants.RECIPE_AR_PATTERN, false, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, err := PrepareCommandForRecipe(properties, constants.RECIPE_AR_PATTERN, false) if err != nil { return nil, errors.WithStack(err) } @@ -260,7 +260,7 @@ func ArchiveCompiledFiles(ctx *types.Context, buildPath *paths.Path, archiveFile const COMMANDLINE_LIMIT = 30000 -func PrepareCommandForRecipe(buildProperties *properties.Map, recipe string, removeUnsetProperties bool, toolEnv []string) (*exec.Cmd, error) { +func PrepareCommandForRecipe(buildProperties *properties.Map, recipe string, removeUnsetProperties bool) (*executils.Process, error) { pattern := buildProperties.Get(recipe) if pattern == "" { return nil, errors.Errorf(tr("%[1]s pattern is missing"), recipe) @@ -275,24 +275,30 @@ func PrepareCommandForRecipe(buildProperties *properties.Map, recipe string, rem if err != nil { return nil, errors.WithStack(err) } - command := exec.Command(parts[0], parts[1:]...) - command.Env = append(os.Environ(), toolEnv...) // if the overall commandline is too long for the platform // try reducing the length by making the filenames relative // and changing working directory to build.path + var relativePath string if len(commandLine) > COMMANDLINE_LIMIT { - relativePath := buildProperties.Get("build.path") - for i, arg := range command.Args { + relativePath = buildProperties.Get("build.path") + for i, arg := range parts { if _, err := os.Stat(arg); os.IsNotExist(err) { continue } rel, err := filepath.Rel(relativePath, arg) if err == nil && !strings.Contains(rel, "..") && len(rel) < len(arg) { - command.Args[i] = rel + parts[i] = rel } } - command.Dir = relativePath + } + + command, err := executils.NewProcess(nil, parts...) + if err != nil { + return nil, errors.WithStack(err) + } + if relativePath != "" { + command.SetDir(relativePath) } return command, nil diff --git a/legacy/builder/create_cmake_rule.go b/legacy/builder/create_cmake_rule.go index 45f68ef9ed4..6d6b67ae1d6 100644 --- a/legacy/builder/create_cmake_rule.go +++ b/legacy/builder/create_cmake_rule.go @@ -367,9 +367,9 @@ func extractCompileFlags(ctx *types.Context, recipe string, defines, dynamicLibs return target } - command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, recipe, true, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, recipe, true) - for _, arg := range command.Args { + for _, arg := range command.GetArgs() { if strings.HasPrefix(arg, "-D") { *defines = appendIfNotPresent(*defines, arg) continue diff --git a/legacy/builder/phases/core_builder.go b/legacy/builder/phases/core_builder.go index 455588f7444..0b629365eba 100644 --- a/legacy/builder/phases/core_builder.go +++ b/legacy/builder/phases/core_builder.go @@ -22,13 +22,13 @@ import ( "os" "strings" + "github.com/arduino/arduino-cli/arduino/builder/cpp" "github.com/arduino/arduino-cli/buildcache" "github.com/arduino/arduino-cli/i18n" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" @@ -80,7 +80,7 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path if variantFolder != nil && variantFolder.IsDir() { includes = append(includes, variantFolder.String()) } - includes = f.Map(includes, utils.WrapWithHyphenI) + includes = f.Map(includes, cpp.WrapWithHyphenI) var err error diff --git a/legacy/builder/phases/libraries_builder.go b/legacy/builder/phases/libraries_builder.go index 88106c9a9de..832bc9b0f8f 100644 --- a/legacy/builder/phases/libraries_builder.go +++ b/legacy/builder/phases/libraries_builder.go @@ -18,7 +18,7 @@ package phases import ( "strings" - "github.com/arduino/arduino-cli/arduino/builder/utils" + "github.com/arduino/arduino-cli/arduino/builder/cpp" "github.com/arduino/arduino-cli/arduino/libraries" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" @@ -38,7 +38,7 @@ func (s *LibrariesBuilder) Run(ctx *types.Context) error { librariesBuildPath := ctx.LibrariesBuildPath buildProperties := ctx.BuildProperties includesFolders := ctx.SketchLibrariesDetector.IncludeFolders() - includes := f.Map(includesFolders.AsStrings(), utils.WrapWithHyphenI) + includes := f.Map(includesFolders.AsStrings(), cpp.WrapWithHyphenI) libs := ctx.SketchLibrariesDetector.ImportedLibraries() if err := librariesBuildPath.MkdirAll(); err != nil { @@ -67,9 +67,9 @@ func findExpectedPrecompiledLibFolder(ctx *types.Context, library *libraries.Lib // Add fpu specifications if they exist // To do so, resolve recipe.cpp.o.pattern, // search for -mfpu=xxx -mfloat-abi=yyy and add to a subfolder - command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, "recipe.cpp.o.pattern", true, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, "recipe.cpp.o.pattern", true) fpuSpecs := "" - for _, el := range strings.Split(command.String(), " ") { + for _, el := range command.GetArgs() { if strings.Contains(el, FPU_CFLAG) { toAdd := strings.Split(el, "=") if len(toAdd) > 1 { @@ -78,7 +78,7 @@ func findExpectedPrecompiledLibFolder(ctx *types.Context, library *libraries.Lib } } } - for _, el := range strings.Split(command.String(), " ") { + for _, el := range command.GetArgs() { if strings.Contains(el, FLOAT_ABI_CFLAG) { toAdd := strings.Split(el, "=") if len(toAdd) > 1 { @@ -201,7 +201,7 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p } } else { if library.UtilityDir != nil { - includes = append(includes, utils.WrapWithHyphenI(library.UtilityDir.String())) + includes = append(includes, cpp.WrapWithHyphenI(library.UtilityDir.String())) } libObjectFiles, err := builder_utils.CompileFiles(ctx, library.SourceDir, libraryBuildPath, buildProperties, includes) if err != nil { diff --git a/legacy/builder/phases/linker.go b/legacy/builder/phases/linker.go index cc7c293199d..1f324576586 100644 --- a/legacy/builder/phases/linker.go +++ b/legacy/builder/phases/linker.go @@ -93,7 +93,7 @@ func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths properties.SetPath("archive_file_path", archive) properties.SetPath("object_file", object) - command, err := builder_utils.PrepareCommandForRecipe(properties, constants.RECIPE_AR_PATTERN, false, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, err := builder_utils.PrepareCommandForRecipe(properties, constants.RECIPE_AR_PATTERN, false) if err != nil { return errors.WithStack(err) } @@ -114,10 +114,10 @@ func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths properties.Set(constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH, coreArchiveFilePath.String()) properties.Set("object_files", objectFileList) - command, err := builder_utils.PrepareCommandForRecipe(properties, constants.RECIPE_C_COMBINE_PATTERN, false, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, err := builder_utils.PrepareCommandForRecipe(properties, constants.RECIPE_C_COMBINE_PATTERN, false) if err != nil { return err - } +} _, _, err = utils.ExecCommand(ctx, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */) return err diff --git a/legacy/builder/phases/sizer.go b/legacy/builder/phases/sizer.go index d466aca2b06..c064fbe717b 100644 --- a/legacy/builder/phases/sizer.go +++ b/legacy/builder/phases/sizer.go @@ -50,7 +50,7 @@ func (s *Sizer) Run(ctx *types.Context) error { } func checkSizeAdvanced(ctx *types.Context, properties *properties.Map) error { - command, err := builder_utils.PrepareCommandForRecipe(properties, "recipe.advanced_size.pattern", false, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, err := builder_utils.PrepareCommandForRecipe(properties, "recipe.advanced_size.pattern", false) if err != nil { return errors.New(tr("Error while determining sketch size: %s", err)) } @@ -179,7 +179,7 @@ func checkSize(ctx *types.Context, buildProperties *properties.Map) error { } func execSizeRecipe(ctx *types.Context, properties *properties.Map) (textSize int, dataSize int, eepromSize int, resErr error) { - command, err := builder_utils.PrepareCommandForRecipe(properties, "recipe.size.pattern", false, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, err := builder_utils.PrepareCommandForRecipe(properties, "recipe.size.pattern", false) if err != nil { resErr = fmt.Errorf(tr("Error while determining sketch size: %s"), err) return diff --git a/legacy/builder/phases/sketch_builder.go b/legacy/builder/phases/sketch_builder.go index 398ba87de8b..51057a21835 100644 --- a/legacy/builder/phases/sketch_builder.go +++ b/legacy/builder/phases/sketch_builder.go @@ -16,7 +16,7 @@ package phases import ( - "github.com/arduino/arduino-cli/arduino/builder/utils" + "github.com/arduino/arduino-cli/arduino/builder/cpp" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/types" @@ -29,7 +29,7 @@ func (s *SketchBuilder) Run(ctx *types.Context) error { sketchBuildPath := ctx.SketchBuildPath buildProperties := ctx.BuildProperties includesFolders := ctx.SketchLibrariesDetector.IncludeFolders() - includes := f.Map(includesFolders.AsStrings(), utils.WrapWithHyphenI) + includes := f.Map(includesFolders.AsStrings(), cpp.WrapWithHyphenI) if err := sketchBuildPath.MkdirAll(); err != nil { return errors.WithStack(err) diff --git a/legacy/builder/recipe_runner.go b/legacy/builder/recipe_runner.go index a6b552b2c6d..01257267287 100644 --- a/legacy/builder/recipe_runner.go +++ b/legacy/builder/recipe_runner.go @@ -44,14 +44,14 @@ func (s *RecipeByPrefixSuffixRunner) Run(ctx *types.Context) error { for _, recipe := range recipes { logrus.Debugf(fmt.Sprintf("Running recipe: %s", recipe)) - command, err := builder_utils.PrepareCommandForRecipe(properties, recipe, false, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + command, err := builder_utils.PrepareCommandForRecipe(properties, recipe, false) if err != nil { return errors.WithStack(err) } if ctx.OnlyUpdateCompilationDatabase && s.SkipIfOnlyUpdatingCompilationDatabase { if ctx.Verbose { - ctx.Info(tr("Skipping: %[1]s", strings.Join(command.Args, " "))) + ctx.Info(tr("Skipping: %[1]s", strings.Join(command.GetArgs(), " "))) } return nil } diff --git a/legacy/builder/utils/utils.go b/legacy/builder/utils/utils.go index 1cddbd3bb48..ec34d1f2c3b 100644 --- a/legacy/builder/utils/utils.go +++ b/legacy/builder/utils/utils.go @@ -18,9 +18,9 @@ package utils import ( "bytes" "os" - "os/exec" "strings" + "github.com/arduino/arduino-cli/executils" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/pkg/errors" @@ -51,30 +51,30 @@ const ( Capture = 3 // Capture into buffer ) -func ExecCommand(ctx *types.Context, command *exec.Cmd, stdout int, stderr int) ([]byte, []byte, error) { +func ExecCommand(ctx *types.Context, command *executils.Process, stdout int, stderr int) ([]byte, []byte, error) { if ctx.Verbose { - ctx.Info(PrintableCommand(command.Args)) + ctx.Info(PrintableCommand(command.GetArgs())) } + stdoutBuffer := &bytes.Buffer{} if stdout == Capture { - buffer := &bytes.Buffer{} - command.Stdout = buffer + command.RedirectStdoutTo(stdoutBuffer) } else if stdout == Show || (stdout == ShowIfVerbose && ctx.Verbose) { if ctx.Stdout != nil { - command.Stdout = ctx.Stdout + command.RedirectStdoutTo(ctx.Stdout) } else { - command.Stdout = os.Stdout + command.RedirectStdoutTo(os.Stdout) } } + stderrBuffer := &bytes.Buffer{} if stderr == Capture { - buffer := &bytes.Buffer{} - command.Stderr = buffer + command.RedirectStderrTo(stderrBuffer) } else if stderr == Show || (stderr == ShowIfVerbose && ctx.Verbose) { if ctx.Stderr != nil { - command.Stderr = ctx.Stderr + command.RedirectStderrTo(ctx.Stderr) } else { - command.Stderr = os.Stderr + command.RedirectStderrTo(os.Stderr) } } @@ -84,16 +84,7 @@ func ExecCommand(ctx *types.Context, command *exec.Cmd, stdout int, stderr int) } err = command.Wait() - - var outbytes, errbytes []byte - if buf, ok := command.Stdout.(*bytes.Buffer); ok { - outbytes = buf.Bytes() - } - if buf, ok := command.Stderr.(*bytes.Buffer); ok { - errbytes = buf.Bytes() - } - - return outbytes, errbytes, errors.WithStack(err) + return stdoutBuffer.Bytes(), stderrBuffer.Bytes(), errors.WithStack(err) } type loggerAction struct { From 417e951d987f47e72685bb926b9fe93f7950d860 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Tue, 5 Sep 2023 09:51:22 +0200 Subject: [PATCH 28/29] use detector FindIncludes in tests --- legacy/builder/builder.go | 3 +-- legacy/builder/test/builder_test.go | 5 +++-- legacy/builder/test/builder_utils_test.go | 16 ++++++++-------- .../test/includes_finder_with_regexp_test.go | 14 +++++++------- legacy/builder/test/libraries_loader_test.go | 10 +++++----- .../unused_compiled_libraries_remover_test.go | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index c9336d24528..ff517a5a2d4 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -223,7 +223,7 @@ func RunPreprocess(ctx *types.Context) error { func findIncludes(ctx *types.Context) types.BareCommand { return types.BareCommand(func(ctx *types.Context) error { - ctx.SketchLibrariesDetector.FindIncludes( + return ctx.SketchLibrariesDetector.FindIncludes( ctx.BuildPath, ctx.BuildProperties.GetPath("build.core.path"), ctx.BuildProperties.GetPath("build.variant.path"), @@ -233,6 +233,5 @@ func findIncludes(ctx *types.Context) types.BareCommand { ctx.BuildProperties, ctx.TargetPlatform.Platform.Architecture, ) - return nil }) } diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index 0f32c02e3c8..696bcfad957 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -22,6 +22,7 @@ import ( "time" bldr "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/detector" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/legacy/builder" @@ -125,14 +126,14 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat } if !stepToSkip[skipLibraries] { - lm, libsResolver, _, err := bldr.LibrariesLoader( + lm, libsResolver, _, err := detector.LibrariesLoader( false, nil, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) NoError(t, err) - ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( + ctx.SketchLibrariesDetector = detector.NewSketchLibrariesDetector( lm, libsResolver, ctx.Verbose, false, diff --git a/legacy/builder/test/builder_utils_test.go b/legacy/builder/test/builder_utils_test.go index b20e7e2555a..f032eabeddc 100644 --- a/legacy/builder/test/builder_utils_test.go +++ b/legacy/builder/test/builder_utils_test.go @@ -20,7 +20,7 @@ import ( "testing" "time" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" + "github.com/arduino/arduino-cli/arduino/builder/utils" paths "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" ) @@ -42,7 +42,7 @@ func TestObjFileIsUpToDateObjMissing(t *testing.T) { sourceFile := tempFile(t, "source") defer sourceFile.RemoveAll() - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, nil, nil) + upToDate, err := utils.ObjFileIsUpToDate(sourceFile, nil, nil) NoError(t, err) require.False(t, upToDate) } @@ -54,7 +54,7 @@ func TestObjFileIsUpToDateDepMissing(t *testing.T) { objFile := tempFile(t, "obj") defer objFile.RemoveAll() - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, nil) + upToDate, err := utils.ObjFileIsUpToDate(sourceFile, objFile, nil) NoError(t, err) require.False(t, upToDate) } @@ -70,7 +70,7 @@ func TestObjFileIsUpToDateObjOlder(t *testing.T) { sourceFile := tempFile(t, "source") defer sourceFile.RemoveAll() - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) NoError(t, err) require.False(t, upToDate) } @@ -86,7 +86,7 @@ func TestObjFileIsUpToDateObjNewer(t *testing.T) { depFile := tempFile(t, "dep") defer depFile.RemoveAll() - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) NoError(t, err) require.True(t, upToDate) } @@ -110,7 +110,7 @@ func TestObjFileIsUpToDateDepIsNewer(t *testing.T) { data := objFile.String() + ": \\\n\t" + sourceFile.String() + " \\\n\t" + headerFile.String() depFile.WriteFile([]byte(data)) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) NoError(t, err) require.False(t, upToDate) } @@ -132,7 +132,7 @@ func TestObjFileIsUpToDateDepIsOlder(t *testing.T) { res := objFile.String() + ": \\\n\t" + sourceFile.String() + " \\\n\t" + headerFile.String() depFile.WriteFile([]byte(res)) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) NoError(t, err) require.True(t, upToDate) } @@ -156,7 +156,7 @@ func TestObjFileIsUpToDateDepIsWrong(t *testing.T) { res := sourceFile.String() + ": \\\n\t" + sourceFile.String() + " \\\n\t" + headerFile.String() depFile.WriteFile([]byte(res)) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) NoError(t, err) require.False(t, upToDate) } diff --git a/legacy/builder/test/includes_finder_with_regexp_test.go b/legacy/builder/test/includes_finder_with_regexp_test.go index 3a55415aa88..16dd07da48f 100644 --- a/legacy/builder/test/includes_finder_with_regexp_test.go +++ b/legacy/builder/test/includes_finder_with_regexp_test.go @@ -18,7 +18,7 @@ package test import ( "testing" - "github.com/arduino/arduino-cli/legacy/builder" + "github.com/arduino/arduino-cli/arduino/builder/detector" "github.com/stretchr/testify/require" ) @@ -27,13 +27,13 @@ func TestIncludesFinderWithRegExp(t *testing.T) { "#include \n" + "^\n" + "compilation terminated." - include := builder.IncludesFinderWithRegExp(output) + include := detector.IncludesFinderWithRegExp(output) require.Equal(t, "SPI.h", include) } func TestIncludesFinderWithRegExpEmptyOutput(t *testing.T) { - include := builder.IncludesFinderWithRegExp("") + include := detector.IncludesFinderWithRegExp("") require.Equal(t, "", include) } @@ -43,7 +43,7 @@ func TestIncludesFinderWithRegExpPaddedIncludes(t *testing.T) { " # include \n" + " ^\n" + "compilation terminated.\n" - include := builder.IncludesFinderWithRegExp(output) + include := detector.IncludesFinderWithRegExp(output) require.Equal(t, "Wire.h", include) } @@ -53,7 +53,7 @@ func TestIncludesFinderWithRegExpPaddedIncludes2(t *testing.T) { " #\t\t\tinclude \n" + " ^\n" + "compilation terminated.\n" - include := builder.IncludesFinderWithRegExp(output) + include := detector.IncludesFinderWithRegExp(output) require.Equal(t, "Wire.h", include) } @@ -62,7 +62,7 @@ func TestIncludesFinderWithRegExpPaddedIncludes3(t *testing.T) { output := "/some/path/sketch.ino:1:33: fatal error: SPI.h: No such file or directory\n" + "compilation terminated.\n" - include := builder.IncludesFinderWithRegExp(output) + include := detector.IncludesFinderWithRegExp(output) require.Equal(t, "SPI.h", include) } @@ -71,7 +71,7 @@ func TestIncludesFinderWithRegExpPaddedIncludes4(t *testing.T) { output := "In file included from /tmp/arduino_modified_sketch_815412/binouts.ino:52:0:\n" + "/tmp/arduino_build_static/sketch/regtable.h:31:22: fatal error: register.h: No such file or directory\n" - include := builder.IncludesFinderWithRegExp(output) + include := detector.IncludesFinderWithRegExp(output) require.Equal(t, "register.h", include) } diff --git a/legacy/builder/test/libraries_loader_test.go b/legacy/builder/test/libraries_loader_test.go index dc4707455ff..e4e1cacfa59 100644 --- a/legacy/builder/test/libraries_loader_test.go +++ b/legacy/builder/test/libraries_loader_test.go @@ -20,8 +20,8 @@ import ( "sort" "testing" + "github.com/arduino/arduino-cli/arduino/builder/detector" "github.com/arduino/arduino-cli/arduino/libraries" - "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" @@ -47,7 +47,7 @@ func TestLoadLibrariesAVR(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, nil, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - lm, libsResolver, _, err := builder.LibrariesLoader( + lm, libsResolver, _, err := detector.LibrariesLoader( false, nil, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, @@ -151,7 +151,7 @@ func TestLoadLibrariesSAM(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, nil, "arduino:sam:arduino_due_x_dbg") defer cleanUpBuilderTestContext(t, ctx) - lm, libsResolver, _, err := builder.LibrariesLoader( + lm, libsResolver, _, err := detector.LibrariesLoader( false, nil, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, @@ -228,7 +228,7 @@ func TestLoadLibrariesAVRNoDuplicateLibrariesFolders(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, nil, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) - lm, _, _, err := builder.LibrariesLoader( + lm, _, _, err := detector.LibrariesLoader( false, nil, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, @@ -251,7 +251,7 @@ func TestLoadLibrariesMyAVRPlatform(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, nil, "my_avr_platform:avr:custom_yun") defer cleanUpBuilderTestContext(t, ctx) - lm, _, _, err := builder.LibrariesLoader( + lm, _, _, err := detector.LibrariesLoader( false, nil, ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, diff --git a/legacy/builder/test/unused_compiled_libraries_remover_test.go b/legacy/builder/test/unused_compiled_libraries_remover_test.go index ab49dc9e26f..24001e674ab 100644 --- a/legacy/builder/test/unused_compiled_libraries_remover_test.go +++ b/legacy/builder/test/unused_compiled_libraries_remover_test.go @@ -18,7 +18,7 @@ package test import ( "testing" - bldr "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/detector" "github.com/arduino/arduino-cli/arduino/libraries" "github.com/arduino/arduino-cli/legacy/builder" "github.com/arduino/arduino-cli/legacy/builder/types" @@ -37,7 +37,7 @@ func TestUnusedCompiledLibrariesRemover(t *testing.T) { ctx := &types.Context{} ctx.LibrariesBuildPath = temp - ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( + ctx.SketchLibrariesDetector = detector.NewSketchLibrariesDetector( nil, nil, false, false, false, nil, nil, nil, nil, ) ctx.SketchLibrariesDetector.AppendImportedLibraries(&libraries.Library{Name: "Bridge"}) @@ -60,7 +60,7 @@ func TestUnusedCompiledLibrariesRemover(t *testing.T) { func TestUnusedCompiledLibrariesRemoverLibDoesNotExist(t *testing.T) { ctx := &types.Context{} ctx.LibrariesBuildPath = paths.TempDir().Join("test") - ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( + ctx.SketchLibrariesDetector = detector.NewSketchLibrariesDetector( nil, nil, false, false, false, nil, nil, nil, nil, ) ctx.SketchLibrariesDetector.AppendImportedLibraries(&libraries.Library{Name: "Bridge"}) @@ -80,7 +80,7 @@ func TestUnusedCompiledLibrariesRemoverNoUsedLibraries(t *testing.T) { NoError(t, temp.Join("dummy_file").WriteFile([]byte{})) ctx := &types.Context{} - ctx.SketchLibrariesDetector = bldr.NewSketchLibrariesDetector( + ctx.SketchLibrariesDetector = detector.NewSketchLibrariesDetector( nil, nil, false, false, false, nil, nil, nil, nil, ) ctx.LibrariesBuildPath = temp From bf0410979a165127c1e1b8d71d36583a36cbf163 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Tue, 5 Sep 2023 09:59:20 +0200 Subject: [PATCH 29/29] add comments and make some plubic methods private --- arduino/builder/cpp/cpp.go | 1 + arduino/builder/detector/detector.go | 103 ++++++++++++++------------- arduino/builder/utils/utils.go | 23 +++--- 3 files changed, 64 insertions(+), 63 deletions(-) diff --git a/arduino/builder/cpp/cpp.go b/arduino/builder/cpp/cpp.go index 4eaa4fafcd4..53123434fe1 100644 --- a/arduino/builder/cpp/cpp.go +++ b/arduino/builder/cpp/cpp.go @@ -107,6 +107,7 @@ func ParseString(line string) (string, string, bool) { } } +// WrapWithHyphenI fixdoc func WrapWithHyphenI(value string) string { return "\"-I" + value + "\"" } diff --git a/arduino/builder/detector/detector.go b/arduino/builder/detector/detector.go index aa00a02e718..dc64b0319e5 100644 --- a/arduino/builder/detector/detector.go +++ b/arduino/builder/detector/detector.go @@ -146,11 +146,6 @@ func (l *SketchLibrariesDetector) AppendImportedLibraries(library *libraries.Lib l.importedLibraries = append(l.importedLibraries, library) } -// UseCachedLibrariesResolution todo -func (l *SketchLibrariesDetector) UseCachedLibrariesResolution() bool { - return l.useCachedLibrariesResolution -} - // PrintUsedAndNotUsedLibraries todo func (l *SketchLibrariesDetector) PrintUsedAndNotUsedLibraries(sketchError bool) { // Print this message: @@ -182,12 +177,13 @@ func (l *SketchLibrariesDetector) PrintUsedAndNotUsedLibraries(sketchError bool) time.Sleep(100 * time.Millisecond) } +// IncludeFolders fixdoc func (l *SketchLibrariesDetector) IncludeFolders() paths.PathList { // TODO should we do a deep copy? return l.includeFolders } -// AppendIncludeFolder todo should rename this, probably after refactoring the +// appendIncludeFolder todo should rename this, probably after refactoring the // container_find_includes command. // Original comment: // Append the given folder to the include path and match or append it to @@ -195,7 +191,7 @@ func (l *SketchLibrariesDetector) IncludeFolders() paths.PathList { // include (e.g. what #include line in what file it was resolved from) // and should be the empty string for the default include folders, like // the core or variant. -func (l *SketchLibrariesDetector) AppendIncludeFolder( +func (l *SketchLibrariesDetector) appendIncludeFolder( cache *includeCache, sourceFilePath *paths.Path, include string, @@ -242,13 +238,13 @@ func (l *SketchLibrariesDetector) findIncludes( ) error { librariesResolutionCache := buildPath.Join("libraries.cache") if l.useCachedLibrariesResolution && librariesResolutionCache.Exist() { - if d, err := librariesResolutionCache.ReadFile(); err != nil { + d, err := librariesResolutionCache.ReadFile() + if err != nil { + return err + } + includeFolders := l.includeFolders + if err := json.Unmarshal(d, &includeFolders); err != nil { return err - } else { - includeFolders := l.includeFolders - if err := json.Unmarshal(d, &includeFolders); err != nil { - return err - } } if l.verbose { l.verboseInfoFn("Using cached library discovery: " + librariesResolutionCache.String()) @@ -259,20 +255,20 @@ func (l *SketchLibrariesDetector) findIncludes( cachePath := buildPath.Join("includes.cache") cache := readCache(cachePath) - l.AppendIncludeFolder(cache, nil, "", buildCorePath) + l.appendIncludeFolder(cache, nil, "", buildCorePath) if buildVariantPath != nil { - l.AppendIncludeFolder(cache, nil, "", buildVariantPath) + l.appendIncludeFolder(cache, nil, "", buildVariantPath) } - sourceFileQueue := &UniqueSourceFileQueue{} + sourceFileQueue := &uniqueSourceFileQueue{} if !l.useCachedLibrariesResolution { sketch := sketch - mergedfile, err := MakeSourceFile(sketchBuildPath, librariesBuildPath, sketch, paths.New(sketch.MainFile.Base()+".cpp")) + mergedfile, err := makeSourceFile(sketchBuildPath, librariesBuildPath, sketch, paths.New(sketch.MainFile.Base()+".cpp")) if err != nil { return errors.WithStack(err) } - sourceFileQueue.Push(mergedfile) + sourceFileQueue.push(mergedfile) l.queueSourceFilesFromFolder(sketchBuildPath, librariesBuildPath, sourceFileQueue, sketch, sketchBuildPath, false /* recurse */) srcSubfolderPath := sketchBuildPath.Join("src") @@ -280,7 +276,7 @@ func (l *SketchLibrariesDetector) findIncludes( l.queueSourceFilesFromFolder(sketchBuildPath, librariesBuildPath, sourceFileQueue, sketch, srcSubfolderPath, true /* recurse */) } - for !sourceFileQueue.Empty() { + for !sourceFileQueue.empty() { err := l.findIncludesUntilDone(cache, sourceFileQueue, buildProperties, sketchBuildPath, librariesBuildPath, platformArch) if err != nil { cachePath.Remove() @@ -310,13 +306,13 @@ func (l *SketchLibrariesDetector) findIncludes( func (l *SketchLibrariesDetector) findIncludesUntilDone( cache *includeCache, - sourceFileQueue *UniqueSourceFileQueue, + sourceFileQueue *uniqueSourceFileQueue, buildProperties *properties.Map, sketchBuildPath *paths.Path, librariesBuildPath *paths.Path, platformArch string, ) error { - sourceFile := sourceFileQueue.Pop() + sourceFile := sourceFileQueue.pop() sourcePath := sourceFile.SourcePath() targetFilePath := paths.NullPath() depPath := sourceFile.DepfilePath() @@ -414,7 +410,7 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone( // include path and queue its source files for further // include scanning l.AppendImportedLibraries(library) - l.AppendIncludeFolder(cache, sourcePath, missingIncludeH, library.SourceDir) + l.appendIncludeFolder(cache, sourcePath, missingIncludeH, library.SourceDir) if library.Precompiled && library.PrecompiledWithSources { // Fully precompiled libraries should have no dependencies to avoid ABI breakage @@ -433,7 +429,7 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone( func (l *SketchLibrariesDetector) queueSourceFilesFromFolder( sketchBuildPath *paths.Path, librariesBuildPath *paths.Path, - sourceFileQueue *UniqueSourceFileQueue, + sourceFileQueue *uniqueSourceFileQueue, origin interface{}, folder *paths.Path, recurse bool, @@ -448,11 +444,11 @@ func (l *SketchLibrariesDetector) queueSourceFilesFromFolder( } for _, filePath := range filePaths { - sourceFile, err := MakeSourceFile(sketchBuildPath, librariesBuildPath, origin, filePath) + sourceFile, err := makeSourceFile(sketchBuildPath, librariesBuildPath, origin, filePath) if err != nil { return errors.WithStack(err) } - sourceFileQueue.Push(sourceFile) + sourceFileQueue.push(sourceFile) } return nil @@ -484,10 +480,12 @@ func (l *SketchLibrariesDetector) failIfImportedLibraryIsWrong() error { return nil } -var INCLUDE_REGEXP = regexp.MustCompile("(?ms)^\\s*#[ \t]*include\\s*[<\"](\\S+)[\">]") +// includeRegexp fixdoc +var includeRegexp = regexp.MustCompile("(?ms)^\\s*#[ \t]*include\\s*[<\"](\\S+)[\">]") +// IncludesFinderWithRegExp fixdoc func IncludesFinderWithRegExp(source string) string { - match := INCLUDE_REGEXP.FindStringSubmatch(source) + match := includeRegexp.FindStringSubmatch(source) if match != nil { return strings.TrimSpace(match[1]) } @@ -507,7 +505,7 @@ func findIncludeForOldCompilers(source string) string { return "" } -type SourceFile struct { +type sourceFile struct { // Path to the source file within the sketch/library root folder relativePath *paths.Path @@ -528,22 +526,23 @@ type SourceFile struct { buildRoot *paths.Path } -func (f *SourceFile) Equals(g *SourceFile) bool { +// Equals fixdoc +func (f *sourceFile) Equals(g *sourceFile) bool { return f.relativePath.EqualsTo(g.relativePath) && f.buildRoot.EqualsTo(g.buildRoot) && f.sourceRoot.EqualsTo(g.sourceRoot) } -// Create a SourceFile containing the given source file path within the +// makeSourceFile containing the given source file path within the // given origin. The given path can be absolute, or relative within the // origin's root source folder -func MakeSourceFile( +func makeSourceFile( sketchBuildPath *paths.Path, librariesBuildPath *paths.Path, origin interface{}, path *paths.Path, -) (*SourceFile, error) { - res := &SourceFile{} +) (*sourceFile, error) { + res := &sourceFile{} switch o := origin.(type) { case *sketch.Sketch: @@ -568,19 +567,23 @@ func MakeSourceFile( return res, nil } -func (f *SourceFile) ExtraIncludePath() *paths.Path { +// ExtraIncludePath fixdoc +func (f *sourceFile) ExtraIncludePath() *paths.Path { return f.extraIncludePath } -func (f *SourceFile) SourcePath() *paths.Path { +// SourcePath fixdoc +func (f *sourceFile) SourcePath() *paths.Path { return f.sourceRoot.JoinPath(f.relativePath) } -func (f *SourceFile) ObjectPath() *paths.Path { +// ObjectPath fixdoc +func (f *sourceFile) ObjectPath() *paths.Path { return f.buildRoot.Join(f.relativePath.String() + ".o") } -func (f *SourceFile) DepfilePath() *paths.Path { +// DepfilePath fixdoc +func (f *sourceFile) DepfilePath() *paths.Path { return f.buildRoot.Join(f.relativePath.String() + ".d") } @@ -658,21 +661,19 @@ func LibrariesLoader( return lm, resolver, verboseOut.Bytes(), nil } -func (l *SketchLibrariesDetector) OnlyUpdateCompilationDatabase() bool { - return l.onlyUpdateCompilationDatabase -} - type includeCacheEntry struct { Sourcefile *paths.Path Include string Includepath *paths.Path } +// String fixdoc func (entry *includeCacheEntry) String() string { return fmt.Sprintf("SourceFile: %s; Include: %s; IncludePath: %s", entry.Sourcefile, entry.Include, entry.Includepath) } +// Equals fixdoc func (entry *includeCacheEntry) Equals(other *includeCacheEntry) bool { return entry.String() == other.String() } @@ -686,14 +687,14 @@ type includeCache struct { entries []*includeCacheEntry } -// Return the next cache entry. Should only be called when the cache is +// Next Return the next cache entry. Should only be called when the cache is // valid and a next entry is available (the latter can be checked with // ExpectFile). Does not advance the cache. func (cache *includeCache) Next() *includeCacheEntry { return cache.entries[cache.next] } -// Check that the next cache entry is about the given file. If it is +// ExpectFile check that the next cache entry is about the given file. If it is // not, or no entry is available, the cache is invalidated. Does not // advance the cache. func (cache *includeCache) ExpectFile(sourcefile *paths.Path) { @@ -703,7 +704,7 @@ func (cache *includeCache) ExpectFile(sourcefile *paths.Path) { } } -// Check that the next entry matches the given values. If so, advance +// ExpectEntry check that the next entry matches the given values. If so, advance // the cache. If not, the cache is invalidated. If the cache is // invalidated, or was already invalid, an entry with the given values // is appended. @@ -723,7 +724,7 @@ func (cache *includeCache) ExpectEntry(sourcefile *paths.Path, include string, l } } -// Check that the cache is completely consumed. If not, the cache is +// ExpectEnd check that the cache is completely consumed. If not, the cache is // invalidated. func (cache *includeCache) ExpectEnd() { if cache.valid && cache.next < len(cache.entries) { @@ -771,25 +772,25 @@ func writeCache(cache *includeCache, path *paths.Path) error { return nil } -type UniqueSourceFileQueue []*SourceFile +type uniqueSourceFileQueue []*sourceFile -func (queue *UniqueSourceFileQueue) Push(value *SourceFile) { - if !queue.Contains(value) { +func (queue *uniqueSourceFileQueue) push(value *sourceFile) { + if !queue.contains(value) { *queue = append(*queue, value) } } -func (queue UniqueSourceFileQueue) Contains(target *SourceFile) bool { +func (queue uniqueSourceFileQueue) contains(target *sourceFile) bool { return slices.ContainsFunc(queue, target.Equals) } -func (queue *UniqueSourceFileQueue) Pop() *SourceFile { +func (queue *uniqueSourceFileQueue) pop() *sourceFile { old := *queue x := old[0] *queue = old[1:] return x } -func (queue UniqueSourceFileQueue) Empty() bool { +func (queue uniqueSourceFileQueue) empty() bool { return len(queue) == 0 } diff --git a/arduino/builder/utils/utils.go b/arduino/builder/utils/utils.go index b1ee7b0bac8..ae64dacb47c 100644 --- a/arduino/builder/utils/utils.go +++ b/arduino/builder/utils/utils.go @@ -14,6 +14,7 @@ import ( "golang.org/x/text/unicode/norm" ) +// ObjFileIsUpToDate fixdoc func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool, error) { logrus.Debugf("Checking previous results for %v (result = %v, dep = %v)", sourceFile, objectFile, dependencyFile) if objectFile == nil || dependencyFile == nil { @@ -33,9 +34,8 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool if os.IsNotExist(err) { logrus.Debugf("Not found: %v", objectFile) return false, nil - } else { - return false, errors.WithStack(err) } + return false, errors.WithStack(err) } dependencyFile = dependencyFile.Clean() @@ -44,9 +44,8 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool if os.IsNotExist(err) { logrus.Debugf("Not found: %v", dependencyFile) return false, nil - } else { - return false, errors.WithStack(err) } + return false, errors.WithStack(err) } if sourceFileStat.ModTime().After(objectFileStat.ModTime()) { @@ -128,7 +127,7 @@ func unescapeDep(s string) string { return s } -// Normalizes an UTF8 byte slice +// NormalizeUTF8 byte slice // TODO: use it more often troughout all the project (maybe on logger interface?) func NormalizeUTF8(buf []byte) []byte { t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC) @@ -136,11 +135,11 @@ func NormalizeUTF8(buf []byte) []byte { return result } -var SOURCE_CONTROL_FOLDERS = map[string]bool{"CVS": true, "RCS": true, ".git": true, ".github": true, ".svn": true, ".hg": true, ".bzr": true, ".vscode": true, ".settings": true, ".pioenvs": true, ".piolibdeps": true} +var sourceControlFolders = map[string]bool{"CVS": true, "RCS": true, ".git": true, ".github": true, ".svn": true, ".hg": true, ".bzr": true, ".vscode": true, ".settings": true, ".pioenvs": true, ".piolibdeps": true} // FilterOutSCCS is a ReadDirFilter that excludes known VSC or project files func FilterOutSCCS(file *paths.Path) bool { - return !SOURCE_CONTROL_FOLDERS[file.Base()] + return !sourceControlFolders[file.Base()] } // FilterReadableFiles is a ReadDirFilter that accepts only readable files @@ -154,12 +153,13 @@ func FilterReadableFiles(file *paths.Path) bool { return true } -// FilterOutHiddenFiles is a ReadDirFilter that exclude files with a "." prefix in their name -var FilterOutHiddenFiles = paths.FilterOutPrefixes(".") +// filterOutHiddenFiles is a ReadDirFilter that exclude files with a "." prefix in their name +var filterOutHiddenFiles = paths.FilterOutPrefixes(".") +// FindFilesInFolder fixdoc func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (paths.PathList, error) { fileFilter := paths.AndFilter( - FilterOutHiddenFiles, + filterOutHiddenFiles, FilterOutSCCS, paths.FilterOutDirectories(), FilterReadableFiles, @@ -172,11 +172,10 @@ func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (pat } if recurse { dirFilter := paths.AndFilter( - FilterOutHiddenFiles, + filterOutHiddenFiles, FilterOutSCCS, ) return dir.ReadDirRecursiveFiltered(dirFilter, fileFilter) } return dir.ReadDir(fileFilter) } -