From 3b232adee180e62de981f76f6e82d2b5d8e4ef39 Mon Sep 17 00:00:00 2001 From: umbynos Date: Fri, 28 Jan 2022 15:56:46 +0100 Subject: [PATCH 1/3] add README.md generation in the `sketch-dist` folder The generated README.md contains commands that helps on how to replicate the build environment --- README.md | 26 +++++++++++++++++++++----- cmd/compile.go | 35 ++++++++++++++++++++++++++++++----- 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 5ff2858..c135a8f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # arduino-cslt `arduino-cslt` is a convenient wrapper of [arduino-cli](https://github.com/arduino/arduino-cli), it compiles Arduino sketches outputting a precompiled library under `sketch-dist/` folder created in the current working directory. -It generates a json file in the `extras/` folder that contains information regarding libraries and core to use in order to build the sketch. The result is achieved by parsing the verbose output of `arduino-cli` and by using [GNU ar](https://sourceware.org/binutils/docs/binutils/ar.html) to generate an archive of the object files. +It generates a README.md file that contains information regarding libraries and core to use in order to build the sketch. The result is achieved by parsing the verbose output of `arduino-cli` and by using [GNU ar](https://sourceware.org/binutils/docs/binutils/ar.html) to generate an archive of the object files. ## Prequisites In order to run this tool you have to install first the [Arduino CLI](https://github.com/arduino/arduino-cli) and have `arduino-cli` binary in your `$PATH`, otherwise `arduino-cslt` won't work. @@ -15,7 +15,7 @@ In order to build `arduino-cslt` just use `task go:build` ## Usage `./arduino-cslt compile -b ` -[![asciicast](https://asciinema.org/a/463342.svg)](https://asciinema.org/a/463342) +[![asciicast](https://asciinema.org/a/465059.svg)](https://asciinema.org/a/465059) For example, running `./arduino-cslt compile -b arduino:samd:mkrwifi1010 sketch/sketch.ino` should produce a library with the following structure, in the current working directory: ``` @@ -28,6 +28,7 @@ sketch-dist/ │ ├── cortex-m0plus │ │ └── libsketch.a │ └── libsketch.h +├── README.md <--contains information regarding libraries and core to install in order to reproduce the original build environment └── sketch └── sketch.ino <-- the actual sketch we are going to compile with the arduino-cli later ``` @@ -48,11 +49,26 @@ INFO[0001] restored sketch/sketch.ino INFO[0001] created sketch-dist/libsketch/library.properties INFO[0001] created sketch-dist/libsketch/src/libsketch.h INFO[0001] created sketch-dist/sketch/sketch.ino +INFO[0003] created sketch-dist/README.md INFO[0001] running: gcc-ar rcs sketch-dist/libsketch/src/cortex-m0plus/libsketch.a /tmp/arduino-sketch-E4D76B1781E9EB73A7B3491CAC68F374/sketch/sketch.ino.cpp.o INFO[0001] created sketch-dist/libsketch/src/cortex-m0plus/libsketch.a INFO[0001] created sketch-dist/libsketch/extras/result.json ``` +The content of `sketch-dist/README.md` included copy-pastable commands to reproduce the build environment: +```markdown +This package contains firmware code loaded in your product. +The firmware contains additional code licensed with LGPL clause; in order to re-compile the entire firmware bundle, please execute the following. + +## Install core and libraries +`arduino-cli core install arduino:samd@1.8.12` +`arduino-cli lib install WiFiNINA@1.8.13` +`arduino-cli lib install SPI@1.0` + +## Compile +`arduino-cli compile -b arduino:samd:mkrwifi1010 sketch-dist/sketch/sketch.ino --library sketch-dist/libsketch` +``` + And the content of `sketch-dist/libsketch/extras/result.json` is: ```json { @@ -80,12 +96,12 @@ And the content of `sketch-dist/libsketch/extras/result.json` is: ``` ## How to compile the precompiled sketch -In order to compile the sketch you have first to install manually the libraries and the core listed in the `sketch-dist//extras/result.json` file. - -You can install a library with [`arduino-cli lib install LIBRARY[@VERSION_NUMBER]`](https://arduino.github.io/arduino-cli/latest/commands/arduino-cli_lib_install/). +In order to compile the sketch you can follow the instructions listed in the `sketch-dist/README.md` file. You can install a core with [`arduino-cli core install PACKAGER:ARCH[@VERSION]`](https://arduino.github.io/arduino-cli/latest/commands/arduino-cli_core_install/). +You can install a library with [`arduino-cli lib install LIBRARY[@VERSION_NUMBER]`](https://arduino.github.io/arduino-cli/latest/commands/arduino-cli_lib_install/). + After completing that operation you can compile it with: `arduino-cli compile -b sketch-dist/sketch/sketch.ino --library sketch-dist/`. diff --git a/cmd/compile.go b/cmd/compile.go index 95552df..2015c70 100644 --- a/cmd/compile.go +++ b/cmd/compile.go @@ -64,10 +64,9 @@ var compileCmd = &cobra.Command{ │ ├── cortex-m0plus │ │ └── libsketch.a │ └── libsketch.h + ├── README.md <--contains information regarding libraries and core to install in order to reproduce the original build environment └── sketch - └── sketch.ino <-- the actual sketch we are going to compile with the arduino-cli later - - The result.json file contains information regarding libraries and core to use in order to reproduce the original build environment`, + └── sketch.ino <-- the actual sketch we can recompile with the arduino-cli later`, Example: os.Args[0] + `compile -b arduino:samd:mkrwifi1010 sketch/sketch.ino`, Args: cobra.ExactArgs(1), // the path of the sketch to build Run: compileSketch, @@ -141,7 +140,7 @@ func compileSketch(cmd *cobra.Command, args []string) { sketchName := strings.TrimSuffix(inoPath.Base(), inoPath.Ext()) // let's create the library corresponding to the precompiled sketch - createLib(sketchName, buildMcu, returnJson, objFilePaths) + createLib(sketchName, buildMcu, fqbn, returnJson, objFilePaths) } // parseCliCompileOutput function takes cmdOutToParse as argument, @@ -274,9 +273,10 @@ func patchSketch(inoPath *paths.Path) (oldSketchContent []byte) { // createLib function will take care of creating the library directory structure and files required, for the precompiled library to be recognized as such. // sketchName is the name of the sketch without the .ino extension. We use this for the name of the lib. // buildMcu is the name of the MCU of the board we have compiled for. The library specifications (https://arduino.github.io/arduino-cli/0.20/library-specification/#precompiled-binaries) requires that the precompiled archive is stored inside a folder with the name of the MCU used during the compile. +// fqbn is required in order to generate the README.md file with instructions. // returnJson is the ResultJson object containing informations regarding core and libraries used during the compile process. // objFilePaths is a paths.PathList containing the paths.Paths to all the sketch related object files produced during the compile phase. -func createLib(sketchName string, buildMcu string, returnJson *ResultJson, objFilePaths *paths.PathList) { +func createLib(sketchName, buildMcu, fqbn string, returnJson *ResultJson, objFilePaths *paths.PathList) { // we are going to leverage the precompiled library infrastructure to make the linking work. // this type of lib, as the type suggest, is already compiled so it only gets linked during the linking phase of a sketch // but we have to create a library folder structure in the current directory: @@ -290,6 +290,7 @@ func createLib(sketchName string, buildMcu string, returnJson *ResultJson, objFi // │ ├── cortex-m0plus // │ │ └── libsketch.a // │ └── libsketch.h + // ├── README.md <--contains information regarding libraries and core to install in order to reproduce the original build environment // └── sketch // └── sketch.ino <-- the actual sketch we are going to compile with the arduino-cli later @@ -373,6 +374,30 @@ void loop() { sketchFilePath := sketchDir.Join(sketchName + ".ino") createFile(sketchFilePath, sketchFile) + // generate the commands to run to successfully reproduce the build environment, they will be used as content for the README.md + var readmeContent []string + readmeContent = append(readmeContent, "`arduino-cli core install "+returnJson.CoreInfo.Id+"@"+returnJson.CoreInfo.Version+"`") + for _, readmeLib := range returnJson.LibsInfo { + readmeContent = append(readmeContent, "`arduino-cli lib install "+readmeLib.Name+"@"+readmeLib.Version+"`") + } + // make the paths relative, absolute paths are too long and are different on the user machine + sketchFileRelPath, _ := sketchFilePath.RelFrom(workingDir) + libRelDir, _ := libDir.RelFrom(workingDir) + readmeCompile := "`arduino-cli compile -b " + fqbn + " " + sketchFileRelPath.String() + " --library " + libRelDir.String() + "`" + + //create the README.md file containig instructions regarding what commands to run in order to have again a working binary + // the README.md contains the following: + readmeMd := `This package contains firmware code loaded in your product. +The firmware contains additional code licensed with LGPL clause; in order to re-compile the entire firmware bundle, please execute the following. + +## Install core and libraries +` + strings.Join(readmeContent, "\n") + "\n" + ` +## Compile +` + readmeCompile + "\n" + + readmeMdPath := rootDir.Join("README.md") + createFile(readmeMdPath, readmeMd) + // run gcc-ar to create an archive containing all the object files except the main.cpp.o (we don't need it because we have created a substitute of it before ⬆️) // we exclude the main.cpp.o because we are going to link the archive libsketch.a against sketchName.ino objFilePaths.FilterOutPrefix("main.cpp") From 622c5f5e55134645bdb9a79cdfd8a5adfbfbabf5 Mon Sep 17 00:00:00 2001 From: umbynos Date: Fri, 28 Jan 2022 17:15:54 +0100 Subject: [PATCH 2/3] apply suggestions from code review and move file creation in functions for better readability --- README.md | 3 +-- cmd/compile.go | 61 +++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index c135a8f..167549d 100644 --- a/README.md +++ b/README.md @@ -62,8 +62,7 @@ The firmware contains additional code licensed with LGPL clause; in order to re- ## Install core and libraries `arduino-cli core install arduino:samd@1.8.12` -`arduino-cli lib install WiFiNINA@1.8.13` -`arduino-cli lib install SPI@1.0` +`arduino-cli lib install WiFiNINA@1.8.13 SPI@1.0` ## Compile `arduino-cli compile -b arduino:samd:mkrwifi1010 sketch-dist/sketch/sketch.ino --library sketch-dist/libsketch` diff --git a/cmd/compile.go b/cmd/compile.go index 2015c70..b7f50d2 100644 --- a/cmd/compile.go +++ b/cmd/compile.go @@ -7,6 +7,7 @@ package cmd import ( "bytes" "encoding/json" + "fmt" "os" "os/exec" "strings" @@ -217,7 +218,7 @@ func getInoSketchPath(argSketchPath string) (inoPath *paths.Path) { return inoPath } -// createMainCpp function, as the name suggests. will create a main.cpp file inside inoPath +// createMainCpp function will create a main.cpp file inside inoPath // we do this because setup() and loop() functions will be replaced inside the ino file, in order to allow the linking afterwards // creating this file is mandatory, we include also Arduino.h because it's a step done by the builder during the building phase, but only for ino files func createMainCpp(inoPath *paths.Path) { @@ -237,7 +238,7 @@ _loop(); createFile(mainCppPath, mainCpp) } -// removeMainCpp function, as the name suggests. will remove a main.cpp file inside inoPath +// removeMainCpp function will remove a main.cpp file inside inoPath // we do this after the compile has been completed, this way we can rerun arduino-cslt again. // If we do not remove this file and run the compile again it will fail because a main.cpp file with the same definitions is already present func removeMainCpp(inoPath *paths.Path) { @@ -328,7 +329,22 @@ func createLib(sketchName, buildMcu, fqbn string, returnJson *ResultJson, objFil // let's create the files - // create a library.properties file in the root dir of the lib + createLibraryPropertiesFile(sketchName, libDir) + + createLibSketchHeaderFile(sketchName, srcDir, returnJson) + + sketchFilePath := createSketchFile(sketchName, sketchDir) + + createReadmeMdFile(sketchFilePath, libDir, workingDir, rootDir, returnJson) + + createArchiveFile(sketchName, objFilePaths, srcDir) + + createResultJsonFile(extraDir, returnJson) +} + +// createLibraryPropertiesFile will create a library.properties file in the libDir, +// the sketchName is required in order to correctly set the name of the "library" +func createLibraryPropertiesFile(sketchName string, libDir *paths.Path) { // the library.properties contains the following: libraryProperties := `name=` + sketchName + ` author=TODO @@ -342,6 +358,15 @@ precompiled=true` libraryPropertyPath := libDir.Join("library.properties") createFile(libraryPropertyPath, libraryProperties) +} + +// createLibSketchHeaderFile will create the libsketch header file, +// the file will be created in the srcDir +// This file has predeclarations of _setup() and _loop() functions declared originally in the main.cpp file (which is not included in the .a archive), +// It is the counterpart of libsketch.a +// we pass resultJson because from there we can extract infos regarding used libs +// sketchName is used to name the file +func createLibSketchHeaderFile(sketchName string, srcDir *paths.Path, returnJson *ResultJson) { // we calculate the #include part to append at the beginning of the header file here with all the libraries used by the original sketch var librariesIncludes []string for _, lib := range returnJson.LibsInfo { @@ -350,9 +375,6 @@ precompiled=true` } } - // create the header file in the src/ dir - // This file has predeclarations of _setup() and _loop() functions declared originally in the main.cpp file (which is not included in the .a archive), - // It is the counterpart of libsketch.a // the libsketch.h contains the following: libsketchHeader := strings.Join(librariesIncludes, "\n") + ` void _setup(); @@ -360,8 +382,12 @@ void _loop();` libsketchFilePath := srcDir.Parent().Join("lib" + sketchName + ".h") createFile(libsketchFilePath, libsketchHeader) +} - // create the sketch file in the sketch-dist dir +// createSketchFile will create the sketch which will be the entrypoint of the compilation with the arduino-cli +// the sketch file will be created in the sketchDir +// the sketchName argument is used to correctly include the right .h file +func createSketchFile(sketchName string, sketchDir *paths.Path) *paths.Path { // This one will include the libsketch.h and basically is the replacement of main.cpp // the sketch.ino contains the following: sketchFile := `#include <` + "lib" + sketchName + `.h> @@ -373,13 +399,20 @@ void loop() { }` sketchFilePath := sketchDir.Join(sketchName + ".ino") createFile(sketchFilePath, sketchFile) + return sketchFilePath +} +// createReadmeMdFile is a helper function that is reposnible for the generation of the README.md file containing informations on how to reproduce the build environment +// it takes the resultJson and some paths.Paths as input to do the required calculations.. The name of the arguments should be sufficient to understand +func createReadmeMdFile(sketchFilePath, libDir, workingDir, rootDir *paths.Path, returnJson *ResultJson) { // generate the commands to run to successfully reproduce the build environment, they will be used as content for the README.md var readmeContent []string readmeContent = append(readmeContent, "`arduino-cli core install "+returnJson.CoreInfo.Id+"@"+returnJson.CoreInfo.Version+"`") - for _, readmeLib := range returnJson.LibsInfo { - readmeContent = append(readmeContent, "`arduino-cli lib install "+readmeLib.Name+"@"+readmeLib.Version+"`") + libs := []string{} + for _, l := range returnJson.LibsInfo { + libs = append(libs, l.Name+"@"+l.Version) } + readmeContent = append(readmeContent, fmt.Sprintf("`arduino-cli lib install %s`", strings.Join(libs, " "))) // make the paths relative, absolute paths are too long and are different on the user machine sketchFileRelPath, _ := sketchFilePath.RelFrom(workingDir) libRelDir, _ := libDir.RelFrom(workingDir) @@ -397,8 +430,10 @@ The firmware contains additional code licensed with LGPL clause; in order to re- readmeMdPath := rootDir.Join("README.md") createFile(readmeMdPath, readmeMd) +} - // run gcc-ar to create an archive containing all the object files except the main.cpp.o (we don't need it because we have created a substitute of it before ⬆️) +// createArchiveFile function will run `gcc-ar` to create an archive containing all the object files except the main.cpp.o (we don't need it because we have created a substitute of it before: sketchfile.ino) +func createArchiveFile(sketchName string, objFilePaths *paths.PathList, srcDir *paths.Path) { // we exclude the main.cpp.o because we are going to link the archive libsketch.a against sketchName.ino objFilePaths.FilterOutPrefix("main.cpp") archivePath := srcDir.Join("lib" + sketchName + ".a") @@ -413,6 +448,10 @@ The firmware contains additional code licensed with LGPL clause; in order to re- } else { logrus.Infof("created %s", archivePath.String()) } +} + +// createResultJsonFile will generate the result.json file and save it in extraDir +func createResultJsonFile(extraDir *paths.Path, returnJson *ResultJson) { // save the result.json in the library extra dir jsonFilePath := extraDir.Join("result.json") if jsonContents, err := json.MarshalIndent(returnJson, "", " "); err != nil { @@ -427,7 +466,7 @@ The firmware contains additional code licensed with LGPL clause; in order to re- // createFile is an helper function useful to create a file, // it takes filePath and fileContent as arguments, // filePath points to the location where to save the file -// fileContent,as the name suggests, include the content of the file +// fileContent include the content of the file func createFile(filePath *paths.Path, fileContent string) { err := os.WriteFile(filePath.String(), []byte(fileContent), 0644) if err != nil { From 7b9831af1b299bd59b072b339d02a952375b39a1 Mon Sep 17 00:00:00 2001 From: Umberto Baldi Date: Mon, 31 Jan 2022 12:47:15 +0100 Subject: [PATCH 3/3] fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 167549d..ad5a19a 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ `arduino-cslt` is a convenient wrapper of [arduino-cli](https://github.com/arduino/arduino-cli), it compiles Arduino sketches outputting a precompiled library under `sketch-dist/` folder created in the current working directory. It generates a README.md file that contains information regarding libraries and core to use in order to build the sketch. The result is achieved by parsing the verbose output of `arduino-cli` and by using [GNU ar](https://sourceware.org/binutils/docs/binutils/ar.html) to generate an archive of the object files. -## Prequisites +## Prerequisites In order to run this tool you have to install first the [Arduino CLI](https://github.com/arduino/arduino-cli) and have `arduino-cli` binary in your `$PATH`, otherwise `arduino-cslt` won't work. Please use a version of the Arduino CLI that has [this](https://github.com/arduino/arduino-cli/pull/1608) change (version > 0.20.2).