Merge pull request #24 from mikusaq/BAF-856/packager-update

Issues consolidation and optimizations

.gitignore

@@ -2,4 +2,5 @@
 go.sum
 install_sysroot
 localInstall
+log
 *.zip

README.md

@@ -1,35 +1,75 @@
-
 # BAP - BringAuto Packager
 
-Build and track C/C++ project dependencies for apps for TODO any Linux distro!
+Build and track C/C++ project dependencies for apps on any Linux distro!
 
-BAM - simple way how to build and maintain our dependencies with almost zero learning curve and out-of-the
-box integration into our workflows.
+BAM provides a simple way to build and maintain dependencies with almost zero learning curve and out-of-the-box integration into your workflows.
 
-## Usage
+## Requirements
 
-bap-builder build and stores all dependencies in the git repository
-(LFS enabled recommended)
+- Docker >= 20.10 (installed according to the official Docker documentation)
+- git >= 2.25
 
-1. create git repository (optionally with LFS)
+Standalone binaries are built for Linux kernel >= 5.10.0-amd64.
 
-```
-mkdir lfsrepo && cd lfsrepo
-git init
-cd ../
-```
+## Build
 
-2. Build all docker images needed for build
+### Requirements for Build
 
-```
-bap-builder build-image --context ./example --name debian11
-```
+- Go >= 1.22
 
-3. Build all packages for given distro
+### Build from Source
 
+Clone the repository and, in the repository root directory, run:
+
+```bash
+go get bringauto/bap-builder
+cd bap-builder
+go build bringauto/bap-builder
 ```
-bap-builder build-package --context ./example --image-name debian11 --output-dir ./lfsrepo --all
-```
+
+## Build Standalone Binaries
+
+To build a complete release package, use the `build.sh` script.
+
+Additional requirements for `build.sh`:
+
+- zip
+- uname
+- sed
+
+## Usage
+
+The `bap-builder` builds and stores all dependencies in the git repository (LFS enabled is recommended).
+
+1. Create a git repository (optionally with LFS):
+
+    ```bash
+    mkdir lfsrepo && cd lfsrepo
+    git init
+    cd ../
+    ```
+
+2. Build all Docker images needed for the build:
+
+    ```bash
+    bap-builder build-image --context ./example --name debian11
+    ```
+
+3. Build all packages for the given distro:
+
+    ```bash
+    bap-builder build-package --context ./example --image-name debian11 --output-dir ./lfsrepo --all
+    ```
+
+*Note: If you do not have `bap-builder` in your system path, you need to use `./bap-builder/bap-builder` instead of `bap-builder`.*
+
+## Documentation
+
+The detailed documentation is available in the `doc` directory.
+
+## Example
+
+In the `example` directory, there is a simple example of how to track dependencies for the `example` project.
 
 ## Motivation
 
@@ -38,7 +78,7 @@ is consistent and work well.
 That's really hard to do because every disto has a different version of the same library. If you want to release
 you app to a new Linux distro you need to test every dependency, solve bugs, maintain your app in time...
 
-There is no simple way how to track, maintain and pack consistent dependency set for C/C++ project without worrying 
+There is no simple way how to track, maintain and pack consistent dependency set for C/C++ project without worrying
 about different version of libraries, tools and the whole environment.
 
 The needed requirement is
@@ -54,6 +94,7 @@ If you want to use one of existing solutions (Connan, Nix, ...) to develop relia
 In other ways the public repository of the system is not authoritative you cannot rely on the package pushed
 to the public repository.\
 If you develop a reliable application you need to have authoritative source of the dependencies
+
 - **You need to create your own instance of the system (or pay to someone to host instance for you)**
 - **You need to create and push your own packages to obtain authoritative source of dependencies.**
 
@@ -70,48 +111,13 @@ BAM solve all these problems! It uses technologies and workflows that are known
 
 You can easily build and track dependencies for your project, download then and use them.
 
-## Requirements
-
-- Docker >= 20.10 (installed by official Docker documentation)
-- git >= 2.25
-
-Standalone binaries are built for Linux kernel >= 5.10.0-amd64
-
-## Build
-
-The project requires go >= 1.18.
-
-```
-go get bringauto/bap-builder
-cd bap-builder
-go build bringauto/bap-builder
-```
-
-## Build standalone binaries
-
-There is a script `build.sh` by that we can build a complete release package.
-
-Additional requirements for `build.sh`:
-
-- zip
-- uname
-- sed
-
 ## FAQ
 
-### Q: I have got a wierd error
-
-Many errors are caused by problem with SSH connection to the Docker container
-or impossibility to start Docker container itself.
-
-In this case
-
-- check if there are no running docker container that 
-are instance of one of the Docker Images in the context directory
-- check if there are no other container that uses port `1122`
+### Q: I am encountering a weird error
 
-## TODO
+Many errors are caused by issues with the SSH connection to the Docker container or the inability to start the Docker container itself.
 
-- SFTP copy and package creation are too slow (~20 minutes for Boost)
-- Refactor `error` handling (use `Log` library, ...) and error messages
+In this case:
 
+- Check if there are no running Docker containers that are instances of one of the Docker images in the context directory.
+- Check if there are no other containers using port `1122`.

bap-builder/CmdArgs.go

@@ -27,14 +27,16 @@ type BuildPackageCmdLineArgs struct {
 	All *bool
 	// Name of the package to build (name of the directory in packages/ dir)
 	Name *string
+	// BuildDeps Build all dependencies of package when building single package
+	BuildDeps *bool
 	// DockerImageName is a name of docker image to which packages will be build.
 	// If empty all docker images from DockerMatrix in config file are used for a given package.
 	// If not empty, only packages which contains DockerImageName in DockerMatrix will be built.
 	// If not empty, packages are built only by toolchain represented by DockerImageName
 	DockerImageName *string
 	// OutputDir relative (to program working dir) ot absolute path where the package will be stored
 	OutputDir *string
-	// Output dir mode
+	// OutputDirMode Output dir mode
 	OutputDirMode *OutputDirMode
 }
 
@@ -87,6 +89,13 @@ func (cmd *CmdLineArgs) InitFlags() {
 			Help:     "Name of the package to build",
 		},
 	)
+	cmd.BuildPackageArgs.BuildDeps = cmd.buildPackageParser.Flag("", "build-deps",
+		&argparse.Options{
+			Required: false,
+			Default:  false,
+			Help:     "Build all dependencies of package when building single package",
+		},
+	)
 	cmd.BuildPackageArgs.OutputDir = cmd.buildPackageParser.String("", "output-dir",
 		&argparse.Options{
 			Required: true,
@@ -134,6 +143,9 @@ func (cmd *CmdLineArgs) ParseArgs(args []string) error {
 
 	cmd.BuildImage = cmd.buildImageParser.Happened()
 	cmd.BuildPackage = cmd.buildPackageParser.Happened()
+	if *cmd.BuildPackageArgs.All && *cmd.BuildPackageArgs.BuildDeps {
+		return fmt.Errorf("all and build-deps flags at the same time")
+	}
 
 	return nil
 }

bap-builder/ContextManager.go

@@ -1,9 +1,9 @@
 package main
 
 import (
+	"bringauto/modules/bringauto_config"
 	"fmt"
 	"io/fs"
-	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
@@ -88,6 +88,68 @@ func (context *ContextManager) GetPackageJsonDefPaths(packageName string) ([]str
 	return packageDefs, nil
 }
 
+// getAllDepsJsonPaths
+// returns all json defintions paths recursively for given package specified by its json definition path
+func (context *ContextManager) getAllDepsJsonPaths(packageJsonPath string, visited map[string]struct{}) ([]string, error) {
+	var config bringauto_config.Config
+	err := config.LoadJSONConfig(packageJsonPath)
+	if err != nil {
+		return []string{}, fmt.Errorf("couldn't load JSON config from %s path - %s", packageJsonPath, err)
+	}
+	visited[packageJsonPath] = struct{}{}
+	var jsonPathListWithDeps []string
+	for _, packageDep := range config.DependsOn {
+		packageDepsJsonPaths, err := context.GetPackageJsonDefPaths(packageDep)
+		if err != nil {
+			return []string{}, fmt.Errorf("couldn't get Json Path of %s package", packageDep)
+		}
+		var depConfig bringauto_config.Config
+		for _, packageDepJsonPath := range packageDepsJsonPaths {
+			_, packageVisited := visited[packageDepJsonPath]
+			if packageVisited {
+				continue
+			}
+			err := depConfig.LoadJSONConfig(packageDepJsonPath)
+			if err != nil {
+				return []string{}, fmt.Errorf("couldn't load JSON config from %s path - %s", packageDepJsonPath, err)
+			}
+			if depConfig.Package.IsDebug != config.Package.IsDebug {
+				continue
+			}
+			jsonPathListWithDeps = append(jsonPathListWithDeps, packageDepJsonPath)
+			jsonPathListWithDepsTmp, err := context.getAllDepsJsonPaths(packageDepJsonPath, visited)
+			if err != nil {
+				return []string{}, err
+			}
+			jsonPathListWithDeps = append(jsonPathListWithDeps, jsonPathListWithDepsTmp...)
+		}
+	}
+
+	return jsonPathListWithDeps, nil
+}
+
+// GetPackageWithDepsJsonDefPaths
+// returns all json definitions paths for given package and all its dependencies json definitions paths recursively
+func (context *ContextManager) GetPackageWithDepsJsonDefPaths(packageName string) ([]string, error) {
+	packageDefs, err := context.GetPackageJsonDefPaths(packageName)
+	if err != nil {
+		return []string{}, fmt.Errorf("cannot get config paths for package '%s' - %s", packageName, err)
+	}
+	var packageDeps []string
+	visitedPackages := make(map[string]struct{})
+	for _, packageDef := range packageDefs {
+		packageDepsTmp, err := context.getAllDepsJsonPaths(packageDef, visitedPackages)
+		if err != nil {
+			return []string{}, err
+		}
+		packageDeps = append(packageDeps, packageDepsTmp...)
+	}
+
+	packageDefs = append(packageDefs, packageDeps...)
+
+	return packageDefs, nil
+}
+
 // GetImageDockerfilePath
 // returns Dockerfile path for the given Image locate in the given context
 func (context *ContextManager) GetImageDockerfilePath(imageName string) (string, error) {
@@ -173,17 +235,17 @@ func getAllFilesInSubdirByRegexp(rootDir string, reg *regexp.Regexp) (map[string
 // get all files from given rootDir which matches given regexp
 func getAllFilesInDirByRegexp(rootDir string, reg *regexp.Regexp) ([]string, error) {
 	var acceptedFileList []string
-	fileList, err := ioutil.ReadDir(rootDir)
+	dirEntryList, err := os.ReadDir(rootDir)
 	if err != nil {
 		return []string{}, fmt.Errorf("cannot list dir %s", rootDir)
 	}
 
-	for _, packagesFileInfos := range fileList {
-		packageNameOk := reg.MatchString(packagesFileInfos.Name())
+	for _, dirEntry := range dirEntryList {
+		packageNameOk := reg.MatchString(dirEntry.Name())
 		if !packageNameOk {
 			continue
 		}
-		acceptedFileList = append(acceptedFileList, path.Join(rootDir, packagesFileInfos.Name()))
+		acceptedFileList = append(acceptedFileList, path.Join(rootDir, dirEntry.Name()))
 	}
 	return acceptedFileList, nil
 }