From cf79c917e72e3dfe3c8879aa64979b891a99df89 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Wed, 17 Jul 2024 15:32:39 +0200 Subject: [PATCH 01/91] Update go and dependencies --- README.md | 2 +- go.mod | 12 ++++++------ go.sum | 10 ++++++++++ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 49acaf3..8110043 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ Standalone binaries are built for Linux kernel >= 5.10.0-amd64 ## Build -The project requires go >= 1.18. +The project requires go >= 1.22. ``` go get bringauto/bap-builder diff --git a/go.mod b/go.mod index c1d54d7..8db8b4f 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module bringauto -go 1.21 +go 1.22 require ( github.com/akamensky/argparse v1.4.0 @@ -8,20 +8,20 @@ require ( github.com/mholt/archiver/v3 v3.5.1 github.com/otiai10/copy v1.14.0 github.com/pkg/sftp v1.13.6 - golang.org/x/crypto v0.19.0 + golang.org/x/crypto v0.25.0 ) require ( github.com/andybalholm/brotli v1.1.0 // indirect github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/klauspost/compress v1.17.6 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/kr/fs v0.1.0 // indirect github.com/nwaples/rardecode v1.1.3 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect - github.com/ulikunitz/xz v0.5.11 // indirect + github.com/ulikunitz/xz v0.5.12 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.17.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index 4835c54..7a81509 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,8 @@ github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6 github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= @@ -84,6 +86,8 @@ github.com/ulikunitz/xz v0.5.9 h1:RsKRIA2MO8x56wkkcd3LbtcE/uMszhb6DpRf+3uwa3I= github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -94,6 +98,8 @@ golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -103,6 +109,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -114,6 +122,8 @@ golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= From 061bfae5470ea01c5668f32d34980a4e3fe0ef63 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Fri, 19 Jul 2024 21:04:32 +0200 Subject: [PATCH 02/91] doc typos fix --- doc/BuildProcess.md | 4 ++-- doc/ContextDirectoryStructure.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/BuildProcess.md b/doc/BuildProcess.md index 7ac7d77..62c7578 100644 --- a/doc/BuildProcess.md +++ b/doc/BuildProcess.md @@ -17,7 +17,7 @@ Each package has a Config file in the JSON format. (Config file details in [Cont - the package configs are stored in linear list - package config are topological sorted by `Dependencies`, -- the packages are build from first item of the list (head of the list) to the las package of the list. +- the packages are build from first item of the list (head of the list) to the last package of the list. During the build the package files installed by installation feature of the CMake are copied to the `install_sysroot` directory located in the working directory of the builder. @@ -36,7 +36,7 @@ Same as for All build except that only configs for given `package_group` are rea **Build phase** -Same as fo All build except that the dependencies are ignored. (so If you have not initialized `install_sysroot` +Same as for All build except that the dependencies are ignored. (so If you have not initialized `install_sysroot` then the package build fail) diff --git a/doc/ContextDirectoryStructure.md b/doc/ContextDirectoryStructure.md index c999842..40fbfb1 100644 --- a/doc/ContextDirectoryStructure.md +++ b/doc/ContextDirectoryStructure.md @@ -27,7 +27,7 @@ The image name is recognized by a name of a directory in the `docker/` directory Docker image built by Dockerfile in directory must be tagged by . -You can use `bam-builder build-image` feature to build docker images instead of directly invoke `docker` command. +You can use `bap-builder build-image` feature to build docker images instead of directly invoke `docker` command. ## package_group_name From 86871952a290c42ff409bca65c26e7750a4b9b75 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Fri, 19 Jul 2024 21:06:39 +0200 Subject: [PATCH 03/91] Add tar functionality, copy file optimized --- modules/bringauto_ssh/SFTP.go | 113 ++++++++++++++++++++++++++++------ modules/bringauto_ssh/Tar.go | 38 ++++++++++++ 2 files changed, 132 insertions(+), 19 deletions(-) create mode 100644 modules/bringauto_ssh/Tar.go diff --git a/modules/bringauto_ssh/SFTP.go b/modules/bringauto_ssh/SFTP.go index fe7ea0e..6034ed6 100644 --- a/modules/bringauto_ssh/SFTP.go +++ b/modules/bringauto_ssh/SFTP.go @@ -4,11 +4,18 @@ import ( "bufio" "fmt" "github.com/pkg/sftp" + "github.com/mholt/archiver/v3" "io" "io/ioutil" "os" "path" "regexp" + "time" +) + +const ( + archiveName string = "install_arch.tar" + archiveNameSep string = string(os.PathSeparator) + archiveName ) type SFTP struct { @@ -26,6 +33,20 @@ type SFTP struct { func (sftpd *SFTP) DownloadDirectory() error { var err error + fmt.Println("Creating tar on remote") + tar := Tar{ + ArchiveName: archiveName, + SourceDir: "/INSTALL", + } + + shellEvaluator := ShellEvaluator{ + Commands: tar.ConstructCMDLine(), + StdOut: os.Stdout, + } + + shellEvaluator.RunOverSSH(*sftpd.SSHCredentials) + fmt.Printf("%s Tar created\n", time.Now()) + sshSession := SSHSession{} err = sshSession.LoginMultipleAttempts(*sftpd.SSHCredentials) if err != nil { @@ -53,14 +74,63 @@ func (sftpd *SFTP) DownloadDirectory() error { return fmt.Errorf("local directory '%s' is not empty", sftpd.EmptyLocalDir) } - err = sftpd.copyRecursive(sftpClient, sftpd.RemoteDir, sftpd.EmptyLocalDir) + localArchivePath := sftpd.EmptyLocalDir + archiveNameSep + + fmt.Printf("%s Copying tar with sftp\n", time.Now()) + + err = sftpd.copyFile(sftpClient, sftpd.RemoteDir + archiveNameSep, localArchivePath) if err != nil { return fmt.Errorf("cannot copy recursive %s", err) } + fmt.Printf("%s File copied. Unarchiving tar.\n", time.Now()) + + tarArchive := archiver.Tar{ + OverwriteExisting: false, + MkdirAll: false, + ImplicitTopLevelFolder: false, + ContinueOnError: true, + } + + tarArchive.Unarchive(localArchivePath, sftpd.EmptyLocalDir) + + fmt.Printf("%s Tar unarchived.\n", time.Now()) + + err = os.Remove(localArchivePath) + if err != nil { + return fmt.Errorf("cannot remove local dir %s", err) + } + + return nil +} + +func (sftpd *SFTP) copyFile(sftpClient *sftp.Client, remoteFile string, localDir string) error { + var err error + _, err = sftpClient.Lstat(remoteFile) + if os.IsNotExist(err) { + return fmt.Errorf("requested remote file %s does not exist", remoteFile) + } + normalizedLocalDir, _ := normalizePath(localDir) + + remotePathStat, err := sftpClient.Lstat(remoteFile) + if err != nil { + return fmt.Errorf("cannot get Lstat if remote %s", remoteFile) + } + sourceFile, err := sftpClient.Open(remoteFile) + if err != nil { + return fmt.Errorf("cannot open file for read - %s,%s", remoteFile, err) + } + destFile, err := os.OpenFile(normalizedLocalDir, os.O_RDWR|os.O_CREATE, remotePathStat.Mode().Perm()) + if err != nil { + return err + } + + copyIOFile(sourceFile, destFile) + return nil } +// deprecated, reason: very slow with big trees, new approach: create tar and and copy single archive with copyFile() func (sftpd *SFTP) copyRecursive(sftpClient *sftp.Client, remoteDir string, localDir string) error { var err error _, err = sftpClient.Lstat(sftpd.RemoteDir) @@ -114,24 +184,7 @@ func (sftpd *SFTP) copyRecursive(sftpClient *sftp.Client, remoteDir string, loca go func() { defer func() { allDone <- true }() - sourceFileBuff := bufio.NewReaderSize(sourceFile, 1024*1024) - destFileBuff := bufio.NewWriterSize(destFile, 1027*1024) - - _, err = io.Copy(destFileBuff, sourceFileBuff) - if err != nil { - panic(fmt.Errorf("cannot copy remote file %s to dest file %s", remotePath, absoluteLocalPath)) - } - - _ = destFileBuff.Flush() - - err = destFile.Close() - if err != nil { - panic(fmt.Errorf("cannot close destFile: %s", err)) - } - err = sourceFile.Close() - if err != nil { - panic(fmt.Errorf("cannot close sourceFile: %s", err)) - } + copyIOFile(sourceFile, destFile) }() } @@ -144,6 +197,28 @@ func (sftpd *SFTP) copyRecursive(sftpClient *sftp.Client, remoteDir string, loca return nil } +func copyIOFile(sourceFile *sftp.File, destFile *os.File) { + sourceFileBuff := bufio.NewReaderSize(sourceFile, 1024*1024) + destFileBuff := bufio.NewWriterSize(destFile, 1027*1024) + + var err error + _, err = io.Copy(destFileBuff, sourceFileBuff) + if err != nil { + panic(fmt.Errorf("cannot copy remote IO files")) + } + + _ = destFileBuff.Flush() + + err = destFile.Close() + if err != nil { + panic(fmt.Errorf("cannot close destFile: %s", err)) + } + err = sourceFile.Close() + if err != nil { + panic(fmt.Errorf("cannot close sourceFile: %s", err)) + } +} + // normalizePath func normalizePath(p string) (string, error) { regexp, regexpErr := regexp.CompilePOSIX("[/]{2,}") diff --git a/modules/bringauto_ssh/Tar.go b/modules/bringauto_ssh/Tar.go new file mode 100644 index 0000000..0a03ae3 --- /dev/null +++ b/modules/bringauto_ssh/Tar.go @@ -0,0 +1,38 @@ +package bringauto_ssh + +import ( + "bringauto/modules/bringauto_prerequisites" + "strings" + "os" +) + +type Tar struct { + ArchiveName string + SourceDir string +} + +func (tar *Tar) FillDefault(*bringauto_prerequisites.Args) error { + tar.ArchiveName = "" + tar.SourceDir = "" + return nil +} + +func (tar *Tar) FillDynamic(*bringauto_prerequisites.Args) error { + return nil +} + +func (tar *Tar) CheckPrerequisites(*bringauto_prerequisites.Args) error { + return nil +} + +func (tar *Tar) ConstructCMDLine() []string { + var cmdLine []string + cmdLine = append(cmdLine, "tar") + cmdLine = append(cmdLine, "cvf") + cmdLine = append(cmdLine, tar.SourceDir + string(os.PathSeparator) + tar.ArchiveName) + cmdLine = append(cmdLine, "-C") + cmdLine = append(cmdLine, tar.SourceDir) + cmdLine = append(cmdLine, ".") + + return []string{strings.Join(cmdLine, " ")} +} \ No newline at end of file From 59c0dc6e66898bbe21f13b9c293cf371f3b463a0 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 22 Jul 2024 10:11:05 +0200 Subject: [PATCH 04/91] Reduce logs --- modules/bringauto_ssh/SFTP.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/bringauto_ssh/SFTP.go b/modules/bringauto_ssh/SFTP.go index 6034ed6..8c5920d 100644 --- a/modules/bringauto_ssh/SFTP.go +++ b/modules/bringauto_ssh/SFTP.go @@ -33,7 +33,6 @@ type SFTP struct { func (sftpd *SFTP) DownloadDirectory() error { var err error - fmt.Println("Creating tar on remote") tar := Tar{ ArchiveName: archiveName, SourceDir: "/INSTALL", @@ -45,7 +44,6 @@ func (sftpd *SFTP) DownloadDirectory() error { } shellEvaluator.RunOverSSH(*sftpd.SSHCredentials) - fmt.Printf("%s Tar created\n", time.Now()) sshSession := SSHSession{} err = sshSession.LoginMultipleAttempts(*sftpd.SSHCredentials) @@ -94,8 +92,6 @@ func (sftpd *SFTP) DownloadDirectory() error { tarArchive.Unarchive(localArchivePath, sftpd.EmptyLocalDir) - fmt.Printf("%s Tar unarchived.\n", time.Now()) - err = os.Remove(localArchivePath) if err != nil { return fmt.Errorf("cannot remove local dir %s", err) From 516e7014aa80e2beb2de04fd825618b8d56eb727 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 22 Jul 2024 10:24:06 +0200 Subject: [PATCH 05/91] Add handling error states --- modules/bringauto_ssh/SFTP.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/bringauto_ssh/SFTP.go b/modules/bringauto_ssh/SFTP.go index 8c5920d..ce1f515 100644 --- a/modules/bringauto_ssh/SFTP.go +++ b/modules/bringauto_ssh/SFTP.go @@ -43,7 +43,10 @@ func (sftpd *SFTP) DownloadDirectory() error { StdOut: os.Stdout, } - shellEvaluator.RunOverSSH(*sftpd.SSHCredentials) + err = shellEvaluator.RunOverSSH(*sftpd.SSHCredentials) + if err != nil { + return fmt.Errorf("cannot archive /INSTALL dir in docker container - %s", err) + } sshSession := SSHSession{} err = sshSession.LoginMultipleAttempts(*sftpd.SSHCredentials) @@ -90,7 +93,10 @@ func (sftpd *SFTP) DownloadDirectory() error { ContinueOnError: true, } - tarArchive.Unarchive(localArchivePath, sftpd.EmptyLocalDir) + err = tarArchive.Unarchive(localArchivePath, sftpd.EmptyLocalDir) + if err != nil { + return fmt.Errorf("cannot unarchive tar archive locally - %s", err) + } err = os.Remove(localArchivePath) if err != nil { From 2b4c651643a98cad09ce6040a737dbf013d167f7 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Sun, 28 Jul 2024 19:24:44 +0200 Subject: [PATCH 06/91] Add custom logging --- .gitignore | 1 + bap-builder/DockerMode.go | 6 +- bap-builder/PackageMode.go | 18 +++-- bap-builder/main.go | 9 ++- modules/bringauto_build/Build.go | 28 ++++++-- modules/bringauto_docker/DockerBuild.go | 6 +- modules/bringauto_docker/DockerImage.go | 10 +-- modules/bringauto_log/GlobalLogger.go | 60 +++++++++++++++++ modules/bringauto_log/Handler.go | 67 +++++++++++++++++++ modules/bringauto_log/Logger.go | 52 ++++++++++++++ modules/bringauto_log/PackageContextLogger.go | 65 ++++++++++++++++++ modules/bringauto_log/PackageLogger.go | 45 +++++++++++++ modules/bringauto_ssh/SFTP.go | 18 +++-- modules/bringauto_ssh/SSHSession.go | 3 +- 14 files changed, 358 insertions(+), 30 deletions(-) create mode 100644 modules/bringauto_log/GlobalLogger.go create mode 100644 modules/bringauto_log/Handler.go create mode 100644 modules/bringauto_log/Logger.go create mode 100644 modules/bringauto_log/PackageContextLogger.go create mode 100644 modules/bringauto_log/PackageLogger.go diff --git a/.gitignore b/.gitignore index d386cc5..d8fb36c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ go.sum install_sysroot localInstall +log *.zip diff --git a/bap-builder/DockerMode.go b/bap-builder/DockerMode.go index f470ddf..d79b6d4 100644 --- a/bap-builder/DockerMode.go +++ b/bap-builder/DockerMode.go @@ -1,9 +1,9 @@ package main import ( + "bringauto/modules/bringauto_log" "bringauto/modules/bringauto_docker" "fmt" - "os" "path" ) @@ -31,9 +31,11 @@ func buildAllDockerImages(contextPath string) error { return err } + logger := bringauto_log.GetLogger() + for imageName, dockerfilePathList := range dockerfileList { if len(dockerfilePathList) != 1 { - fmt.Fprintf(os.Stderr, "Bug: multiple Dockerfile present for same image name %s", imageName) + logger.Warn("Bug: multiple Dockerfile present for same image name %s", imageName) continue } dockerfileDir := path.Dir(dockerfilePathList[0]) diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index c34e819..2a2bab8 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -1,6 +1,7 @@ package main import ( + "bringauto/modules/bringauto_log" "bringauto/modules/bringauto_build" "bringauto/modules/bringauto_config" "bringauto/modules/bringauto_package" @@ -8,8 +9,6 @@ import ( "bringauto/modules/bringauto_repository" "bringauto/modules/bringauto_sysroot" "fmt" - "log" - "os" "strconv" ) @@ -159,6 +158,8 @@ func buildAllPackages(cmdLine *BuildPackageCmdLineArgs, contextPath string) erro depsList := buildDepList{} configList := depsList.TopologicalSort(defsList) + logger := bringauto_log.GetLogger() + count := int32(0) for _, config := range configList { buildConfigs := config.GetBuildStructure(*cmdLine.DockerImageName) @@ -166,14 +167,14 @@ func buildAllPackages(cmdLine *BuildPackageCmdLineArgs, contextPath string) erro continue } count++ - log.Printf("Build %s\n", buildConfigs[0].Package.CreatePackageName()) + logger.Info("Build %s", buildConfigs[0].Package.CreatePackageName()) err = buildAndCopyPackage(cmdLine, &buildConfigs) if err != nil { - panic(fmt.Errorf("cannot build package '%s' - %s", config.Package.Name, err)) + logger.Fatal("cannot build package '%s' - %s", config.Package.Name, err) } } if count == 0 { - log.Printf("Nothing to build. Did you enter correct image name?") + logger.Warn("Nothing to build. Did you enter correct image name?") } return nil @@ -192,18 +193,21 @@ func buildSinglePackage(cmdLine *BuildPackageCmdLineArgs, contextPath string) er return err } + logger := bringauto_log.GetLogger() + for _, packageJsonDef := range packageJsonDefsList { var config bringauto_config.Config err = config.LoadJSONConfig(packageJsonDef) if err != nil { - fmt.Fprintf(os.Stderr, "package '%s' JSON config def problem - %s\n", packageName, err) + logger.Warn("package '%s' JSON config def problem - %s\n", packageName, err) continue } buildConfigs := config.GetBuildStructure(*cmdLine.DockerImageName) + logger.Info("Build %s", buildConfigs[0].Package.CreatePackageName()) err = buildAndCopyPackage(cmdLine, &buildConfigs) if err != nil { - fmt.Fprintf(os.Stderr, "cannot build package '%s' - %s\n", packageName, err) + logger.Error("cannot build package '%s' - %s\n", packageName, err) continue } } diff --git a/bap-builder/main.go b/bap-builder/main.go index 9466d6d..a8627e0 100644 --- a/bap-builder/main.go +++ b/bap-builder/main.go @@ -1,13 +1,16 @@ package main import ( - "fmt" + "bringauto/modules/bringauto_log" + "bringauto/modules/bringauto_prerequisites" "os" + "time" ) func main() { var err error var args CmdLineArgs + logger := bringauto_prerequisites.CreateAndInitialize[bringauto_log.GlobalLogger](time.Now(), "./log") args.InitFlags() err = args.ParseArgs(os.Args) @@ -18,7 +21,7 @@ func main() { if args.BuildImage { err = BuildDockerImage(&args.BuildImagesArgs, *args.Context) if err != nil { - fmt.Fprintln(os.Stderr, err) + logger.Error(err.Error()) return } return @@ -27,7 +30,7 @@ func main() { if args.BuildPackage { err = BuildPackage(&args.BuildPackageArgs, *args.Context) if err != nil { - fmt.Fprintln(os.Stderr, err) + logger.Error(err.Error()) return } return diff --git a/modules/bringauto_build/Build.go b/modules/bringauto_build/Build.go index 10c220d..09fd3ae 100644 --- a/modules/bringauto_build/Build.go +++ b/modules/bringauto_build/Build.go @@ -1,6 +1,7 @@ package bringauto_build import ( + "bringauto/modules/bringauto_log" "bringauto/modules/bringauto_docker" "bringauto/modules/bringauto_git" "bringauto/modules/bringauto_package" @@ -8,7 +9,6 @@ import ( "bringauto/modules/bringauto_ssh" "bringauto/modules/bringauto_sysroot" "fmt" - "log" "os" "path/filepath" ) @@ -70,6 +70,8 @@ func (build *Build) CheckPrerequisites(*bringauto_prerequisites.Args) error { // s func (build *Build) RunBuild() error { var err error + logger := bringauto_log.GetLogger() + packLogger := logger.CreatePackageLogger(build.Package.CreatePackageName()) err = build.CheckPrerequisites(nil) if err != nil { @@ -109,9 +111,17 @@ func (build *Build) RunBuild() error { }, } + buildChainLogger := packLogger.CreatePackageContextLogger(bringauto_log.BuildChainContext) + file, err := buildChainLogger.GetFile() + + if err != nil { + logger.Error("Failed to open file - %s", err) + return err + } + shellEvaluator := bringauto_ssh.ShellEvaluator{ Commands: buildChain.GenerateCommands(), - StdOut: os.Stdout, + StdOut: file, } err = bringauto_prerequisites.Initialize(build.Docker) @@ -130,11 +140,11 @@ func (build *Build) RunBuild() error { var err error err = dockerStop.Stop() if err != nil { - fmt.Fprintf(os.Stderr, "cannot stop container: %s\n", err) + logger.Error("cannot stop container: %s\n", err) } err = dockerRm.RemoveContainer() if err != nil { - fmt.Fprintf(os.Stderr, "cannot remove container: %s\n", err) + logger.Error("cannot remove container: %s\n", err) } }() @@ -143,7 +153,9 @@ func (build *Build) RunBuild() error { return err } - err = build.downloadInstalledFiles() + file.Close() + + err = build.downloadInstalledFiles(packLogger) return err } @@ -154,7 +166,8 @@ func (build *Build) SetSysroot(sysroot *bringauto_sysroot.Sysroot) { func (build *Build) GetLocalInstallDirPath() string { workingDir, err := os.Getwd() if err != nil { - log.Fatalf("cannot call Getwd - %s", err) + logger := bringauto_log.GetLogger() + logger.Fatal("cannot call Getwd - %s", err) } copyBaseDir := filepath.Join(workingDir, localInstallDirNameConst) return copyBaseDir @@ -173,7 +186,7 @@ func (build *Build) CleanUp() error { return nil } -func (build *Build) downloadInstalledFiles() error { +func (build *Build) downloadInstalledFiles(packageLogger *bringauto_log.PackageLogger) error { var err error copyDir := build.GetLocalInstallDirPath() @@ -188,6 +201,7 @@ func (build *Build) downloadInstalledFiles() error { RemoteDir: dockerInstallDirConst, EmptyLocalDir: copyDir, SSHCredentials: build.SSHCredentials, + PackageLogger : packageLogger, } err = sftpClient.DownloadDirectory() return err diff --git a/modules/bringauto_docker/DockerBuild.go b/modules/bringauto_docker/DockerBuild.go index ce7eaa1..e812ec1 100644 --- a/modules/bringauto_docker/DockerBuild.go +++ b/modules/bringauto_docker/DockerBuild.go @@ -1,9 +1,9 @@ package bringauto_docker import ( + "bringauto/modules/bringauto_log" "bytes" "fmt" - "log" "os/exec" ) @@ -21,7 +21,9 @@ func (dockerBuild *DockerBuild) Build() error { if dockerBuild.DockerfileDir == "" { return fmt.Errorf("DockerBuild - DockerfileDir is empty") } - log.Printf("Build Docker Image: %s", dockerBuild.Tag) + logger := bringauto_log.GetLogger() + logger.Info("Build Docker Image: %s", dockerBuild.Tag) + var ok, _, err = dockerBuild.prepareAndRun(prepareBuildArgs) if !ok { return fmt.Errorf("DockerBuild build error - %s", err) diff --git a/modules/bringauto_docker/DockerImage.go b/modules/bringauto_docker/DockerImage.go index 2d7db07..f6ffd0e 100644 --- a/modules/bringauto_docker/DockerImage.go +++ b/modules/bringauto_docker/DockerImage.go @@ -1,10 +1,10 @@ package bringauto_docker import ( + "bringauto/modules/bringauto_log" "bringauto/modules/bringauto_process" "bytes" "fmt" - "log" "regexp" "strings" ) @@ -14,13 +14,15 @@ type DockerImage Docker func (dockerImage *DockerImage) ImageExists() bool { var err error output, err := dockerImage.runDockerImageCommand([]string{"ls"}) + + logger := bringauto_log.GetLogger() if err != nil { - log.Printf("cannot start docker process: %s", err) + logger.Error("cannot start docker process: %s", err) return false } reg, err := regexp.CompilePOSIX("^(?P[0-9a-zA-Z]+)\\s+(?P[^ ]+)") if err != nil { - log.Printf("cannot compile regexp: %s", err) + logger.Error("cannot compile regexp: %s", err) } outputLines := strings.Split(output, "\n") @@ -30,7 +32,7 @@ func (dockerImage *DockerImage) ImageExists() bool { for _, line := range containersInfoLines { dockerImageLine := reg.FindStringSubmatch(line) if dockerImageLine == nil { - log.Fatalf("Bad imageLine from docker images connect - %s", line) + logger.Fatal("Bad imageLine from docker images connect - %s", line) } imageName := dockerImageLine[imageNameIndex] if imageName == dockerImage.ImageName { diff --git a/modules/bringauto_log/GlobalLogger.go b/modules/bringauto_log/GlobalLogger.go new file mode 100644 index 0000000..6425cf9 --- /dev/null +++ b/modules/bringauto_log/GlobalLogger.go @@ -0,0 +1,60 @@ +package bringauto_log + +import ( + "bringauto/modules/bringauto_prerequisites" + "os" + "time" + "strconv" +) + +var globalLoggerSingleton *GlobalLogger + +func GetLogger() *GlobalLogger { + if globalLoggerSingleton == nil { + globalLoggerSingleton = bringauto_prerequisites.CreateAndInitialize[GlobalLogger]() + globalLoggerSingleton.Warn("Global logger was not initialized. Printing to console.") + } + return globalLoggerSingleton +} + +type GlobalLogger struct { + Logger +} + +type globalLoggerInitArgs struct { + Timestamp time.Time + LogDirPath string +} + +func (logger *GlobalLogger) FillDefault(*bringauto_prerequisites.Args) error { + logger.slogger = getDefaultLogger(os.Stdout) + logger.timestamp = time.Time{} + logger.logDirPath = "" + return nil +} + +func (logger *GlobalLogger) FillDynamic(args *bringauto_prerequisites.Args) error { + if !bringauto_prerequisites.IsEmpty(args) { + var argsStruct globalLoggerInitArgs + bringauto_prerequisites.GetArgs(args, &argsStruct) + logger.timestamp = argsStruct.Timestamp + logger.logDirPath = argsStruct.LogDirPath + "/" + logger.getTimestampString() + } + return nil +} + +func (logger *GlobalLogger) getTimestampString() string { + return strconv.FormatInt(logger.timestamp.Unix(), 10) +} + +func (logger *GlobalLogger) CheckPrerequisites(*bringauto_prerequisites.Args) error { + globalLoggerSingleton = logger + return nil +} + +func (logger *GlobalLogger) CreatePackageLogger(packageName string) *PackageLogger { + packageContextLogger := bringauto_prerequisites.CreateAndInitialize[PackageLogger]( + logger.timestamp, logger.logDirPath, packageName, + ) + return packageContextLogger +} diff --git a/modules/bringauto_log/Handler.go b/modules/bringauto_log/Handler.go new file mode 100644 index 0000000..92ca661 --- /dev/null +++ b/modules/bringauto_log/Handler.go @@ -0,0 +1,67 @@ +package bringauto_log + +import ( + "log/slog" + "context" + "io" + "fmt" + "strconv" +) + +const ( + red = 31 + blue = 34 + orange = 93 + white = 97 +) + +type Handler struct { + writer io.Writer +} + +func NewHandler(writer io.Writer) *Handler { + handler := &Handler{writer: writer} + + return handler +} + +func (handler *Handler) Enabled(_ context.Context, _ slog.Level) bool { + return true +} + +func colorizeLevel(level slog.Level) string { + var colorCode int + switch level.String() { + case "INFO" : + colorCode = white + case "WARN" : + colorCode = orange + case "ERROR" : + colorCode = red + } + + return fmt.Sprintf("\033[%sm%s\033[0m", strconv.Itoa(colorCode), level.String()) +} + +func (handler *Handler) Handle(_ context.Context, r slog.Record) error { + buf := make([]byte, 0, 1024) + formated := r.Time.Format("2006-01-02 15:04:05") + if !r.Time.IsZero() { + buf = fmt.Append(buf, formated) + } + + buf = fmt.Appendf(buf, " %s", colorizeLevel(r.Level)) + + buf = fmt.Appendf(buf, " %s\n", r.Message) + + _, err := handler.writer.Write(buf) + return err +} + +func (handler *Handler) WithGroup(name string) slog.Handler { + return handler +} + +func (handler *Handler) WithAttrs(attrs []slog.Attr) slog.Handler { + return handler +} diff --git a/modules/bringauto_log/Logger.go b/modules/bringauto_log/Logger.go new file mode 100644 index 0000000..7bdf91e --- /dev/null +++ b/modules/bringauto_log/Logger.go @@ -0,0 +1,52 @@ +package bringauto_log + +import ( + "io" + "log/slog" + "os" + "time" + "fmt" +) + +type Logger struct { + slogger *slog.Logger + timestamp time.Time + logDirPath string +} + +func getDefaultLogger(writer io.Writer) *slog.Logger { + return slog.New(NewHandler(writer)) +} + +func (logger *Logger) Info(msg string, args ...any) { + if len(args) == 0 { + logger.slogger.Info(msg) + } else { + logger.slogger.Info(fmt.Sprintf(msg, args)) + } +} + +func (logger *Logger) Warn(msg string, args ...any) { + if len(args) == 0 { + logger.slogger.Warn(msg) + } else { + logger.slogger.Warn(fmt.Sprintf(msg, args)) + } +} + +func (logger *Logger) Error(msg string, args ...any) { + if len(args) == 0 { + logger.slogger.Error(msg) + } else { + logger.slogger.Error(fmt.Sprintf(msg, args)) + } +} + +func (logger *Logger) Fatal(msg string, args ...any) { + if len(args) == 0 { + logger.slogger.Error(msg) + } else { + logger.slogger.Error(fmt.Sprintf(msg, args)) + } + os.Exit(1) +} diff --git a/modules/bringauto_log/PackageContextLogger.go b/modules/bringauto_log/PackageContextLogger.go new file mode 100644 index 0000000..9392fa7 --- /dev/null +++ b/modules/bringauto_log/PackageContextLogger.go @@ -0,0 +1,65 @@ +package bringauto_log + +import ( + "bringauto/modules/bringauto_prerequisites" + "os" + "time" +) + +const ( + BuildChainContext = "build_chain" + TarContext = "tar" +) + +type PackageContextLogger struct { + Logger + logFileName string +} + +type packageContextLoggerInitArgs struct { + Timestamp time.Time + LogDirPath string + LogContext string +} + +func (logger *PackageContextLogger) FillDefault(*bringauto_prerequisites.Args) error { + logger.slogger = getDefaultLogger(os.Stdout) + logger.timestamp = time.Time{} + logger.logDirPath = "" + logger.logFileName = "" + return nil +} + +func (logger *PackageContextLogger) FillDynamic(args *bringauto_prerequisites.Args) error { + if !bringauto_prerequisites.IsEmpty(args) { + var argsStruct packageContextLoggerInitArgs + bringauto_prerequisites.GetArgs(args, &argsStruct) + logger.timestamp = argsStruct.Timestamp + logger.logDirPath = argsStruct.LogDirPath + logger.logFileName = argsStruct.LogContext + ".txt" + } + return nil +} + +func (logger *PackageContextLogger) initLogDir() error { + _, err := os.Stat(logger.logDirPath) + if os.IsNotExist(err) { + err = os.MkdirAll(logger.logDirPath, 0700) + if err != nil { + logger.Error("Failed to create log directory - %s", err) + return err + } + } + return nil +} + +func (logger *PackageContextLogger) CheckPrerequisites(*bringauto_prerequisites.Args) error { + if logger.logDirPath != "" { + logger.initLogDir() + } + return nil +} + +func (logger *PackageContextLogger) GetFile() (*os.File, error) { + return os.OpenFile(logger.logDirPath + "/" + logger.logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) +} diff --git a/modules/bringauto_log/PackageLogger.go b/modules/bringauto_log/PackageLogger.go new file mode 100644 index 0000000..d1c6dc3 --- /dev/null +++ b/modules/bringauto_log/PackageLogger.go @@ -0,0 +1,45 @@ +package bringauto_log + +import ( + "bringauto/modules/bringauto_prerequisites" + "os" + "time" +) + +type PackageLogger struct { + Logger +} + +type packageLoggerInitArgs struct { + Timestamp time.Time + LogDirPath string + PackageName string +} + +func (logger *PackageLogger) FillDefault(*bringauto_prerequisites.Args) error { + logger.slogger = getDefaultLogger(os.Stdout) + logger.timestamp = time.Time{} + logger.logDirPath = "" + return nil +} + +func (logger *PackageLogger) FillDynamic(args *bringauto_prerequisites.Args) error { + if !bringauto_prerequisites.IsEmpty(args) { + var argsStruct packageLoggerInitArgs + bringauto_prerequisites.GetArgs(args, &argsStruct) + logger.timestamp = argsStruct.Timestamp + logger.logDirPath = argsStruct.LogDirPath + "/" + argsStruct.PackageName + } + return nil +} + +func (logger *PackageLogger) CheckPrerequisites(*bringauto_prerequisites.Args) error { + return nil +} + +func (logger *PackageLogger) CreatePackageContextLogger(logContext string) *PackageContextLogger { + packageContextLogger := bringauto_prerequisites.CreateAndInitialize[PackageContextLogger]( + logger.timestamp, logger.logDirPath, logContext, + ) + return packageContextLogger +} diff --git a/modules/bringauto_ssh/SFTP.go b/modules/bringauto_ssh/SFTP.go index ce1f515..47e44d0 100644 --- a/modules/bringauto_ssh/SFTP.go +++ b/modules/bringauto_ssh/SFTP.go @@ -1,6 +1,7 @@ package bringauto_ssh import ( + "bringauto/modules/bringauto_log" "bufio" "fmt" "github.com/pkg/sftp" @@ -10,7 +11,6 @@ import ( "os" "path" "regexp" - "time" ) const ( @@ -24,6 +24,7 @@ type SFTP struct { // Empty, existing local directory where the RemoteDir will be copy EmptyLocalDir string SSHCredentials *SSHCredentials + PackageLogger *bringauto_log.PackageLogger } // DownloadDirectory @@ -38,12 +39,20 @@ func (sftpd *SFTP) DownloadDirectory() error { SourceDir: "/INSTALL", } + tarLogger := sftpd.PackageLogger.CreatePackageContextLogger(bringauto_log.TarContext) + file, err := tarLogger.GetFile() + + if err != nil { + return err + } + shellEvaluator := ShellEvaluator{ Commands: tar.ConstructCMDLine(), - StdOut: os.Stdout, + StdOut: file, } err = shellEvaluator.RunOverSSH(*sftpd.SSHCredentials) + file.Close() if err != nil { return fmt.Errorf("cannot archive /INSTALL dir in docker container - %s", err) } @@ -77,14 +86,15 @@ func (sftpd *SFTP) DownloadDirectory() error { localArchivePath := sftpd.EmptyLocalDir + archiveNameSep - fmt.Printf("%s Copying tar with sftp\n", time.Now()) + logger := bringauto_log.GetLogger() + logger.Info("Copying tar with sftp") err = sftpd.copyFile(sftpClient, sftpd.RemoteDir + archiveNameSep, localArchivePath) if err != nil { return fmt.Errorf("cannot copy recursive %s", err) } - fmt.Printf("%s File copied. Unarchiving tar.\n", time.Now()) + logger.Info("File copied. Unarchiving tar.") tarArchive := archiver.Tar{ OverwriteExisting: false, diff --git a/modules/bringauto_ssh/SSHSession.go b/modules/bringauto_ssh/SSHSession.go index 3b3b5ea..1b0ce76 100644 --- a/modules/bringauto_ssh/SSHSession.go +++ b/modules/bringauto_ssh/SSHSession.go @@ -1,6 +1,7 @@ package bringauto_ssh import ( + "bringauto/modules/bringauto_log" "bringauto/modules/bringauto_prerequisites" "fmt" "golang.org/x/crypto/ssh" @@ -69,7 +70,7 @@ func (session *SSHSession) LoginMultipleAttempts(credentials SSHCredentials) err } else { numberOfAttempts += 1 time.Sleep(waitingInSecondsBeforeRetryConst * time.Second) - fmt.Println("Cannot login over ssh. Again...") + bringauto_log.GetLogger().Info("Cannot login over ssh. Again...") } } return nil From 831c0a9f68bd61e9f09cf879501f12ffeead725b Mon Sep 17 00:00:00 2001 From: Mikulas Date: Sun, 28 Jul 2024 19:36:02 +0200 Subject: [PATCH 07/91] Replace deprecated function from ioutil package --- modules/bringauto_ssh/SFTP.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/bringauto_ssh/SFTP.go b/modules/bringauto_ssh/SFTP.go index 47e44d0..0c3bf14 100644 --- a/modules/bringauto_ssh/SFTP.go +++ b/modules/bringauto_ssh/SFTP.go @@ -7,7 +7,6 @@ import ( "github.com/pkg/sftp" "github.com/mholt/archiver/v3" "io" - "io/ioutil" "os" "path" "regexp" @@ -78,7 +77,7 @@ func (sftpd *SFTP) DownloadDirectory() error { return fmt.Errorf("EmptyLocalDir '%s' does not exist", sftpd.EmptyLocalDir) } - localPathDirContent, _ := ioutil.ReadDir(sftpd.EmptyLocalDir) + localPathDirContent, _ := os.ReadDir(sftpd.EmptyLocalDir) localPathDirIsNotEmpty := len(localPathDirContent) != 0 if localPathDirIsNotEmpty { return fmt.Errorf("local directory '%s' is not empty", sftpd.EmptyLocalDir) From 43e1b069f6611ce33d4ec0f0b658a7ba5c5b4f92 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Sun, 28 Jul 2024 19:43:59 +0200 Subject: [PATCH 08/91] Remove debug logging --- modules/bringauto_ssh/SFTP.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/modules/bringauto_ssh/SFTP.go b/modules/bringauto_ssh/SFTP.go index 0c3bf14..e5af97c 100644 --- a/modules/bringauto_ssh/SFTP.go +++ b/modules/bringauto_ssh/SFTP.go @@ -85,16 +85,11 @@ func (sftpd *SFTP) DownloadDirectory() error { localArchivePath := sftpd.EmptyLocalDir + archiveNameSep - logger := bringauto_log.GetLogger() - logger.Info("Copying tar with sftp") - err = sftpd.copyFile(sftpClient, sftpd.RemoteDir + archiveNameSep, localArchivePath) if err != nil { return fmt.Errorf("cannot copy recursive %s", err) } - logger.Info("File copied. Unarchiving tar.") - tarArchive := archiver.Tar{ OverwriteExisting: false, MkdirAll: false, From 834e77d283e051d3590ca9f6467b7ffe64b20fcd Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 29 Jul 2024 14:15:51 +0200 Subject: [PATCH 09/91] Reduce abstraction, refactor --- modules/bringauto_build/Build.go | 28 +++++++----- modules/bringauto_log/GlobalLogger.go | 6 +-- modules/bringauto_log/PackageContextLogger.go | 3 +- modules/bringauto_log/PackageLogger.go | 45 ------------------- modules/bringauto_ssh/SFTP.go | 14 ++---- 5 files changed, 26 insertions(+), 70 deletions(-) delete mode 100644 modules/bringauto_log/PackageLogger.go diff --git a/modules/bringauto_build/Build.go b/modules/bringauto_build/Build.go index 09fd3ae..6931110 100644 --- a/modules/bringauto_build/Build.go +++ b/modules/bringauto_build/Build.go @@ -1,9 +1,9 @@ package bringauto_build import ( - "bringauto/modules/bringauto_log" "bringauto/modules/bringauto_docker" "bringauto/modules/bringauto_git" + "bringauto/modules/bringauto_log" "bringauto/modules/bringauto_package" "bringauto/modules/bringauto_prerequisites" "bringauto/modules/bringauto_ssh" @@ -70,8 +70,6 @@ func (build *Build) CheckPrerequisites(*bringauto_prerequisites.Args) error { // s func (build *Build) RunBuild() error { var err error - logger := bringauto_log.GetLogger() - packLogger := logger.CreatePackageLogger(build.Package.CreatePackageName()) err = build.CheckPrerequisites(nil) if err != nil { @@ -111,14 +109,17 @@ func (build *Build) RunBuild() error { }, } - buildChainLogger := packLogger.CreatePackageContextLogger(bringauto_log.BuildChainContext) - file, err := buildChainLogger.GetFile() + logger := bringauto_log.GetLogger() + packBuildChainLogger := logger.CreatePackageContextLogger(build.Package.Name, bringauto_log.BuildChainContext) + file, err := packBuildChainLogger.GetFile() if err != nil { logger.Error("Failed to open file - %s", err) return err } + defer file.Close() + shellEvaluator := bringauto_ssh.ShellEvaluator{ Commands: buildChain.GenerateCommands(), StdOut: file, @@ -153,9 +154,7 @@ func (build *Build) RunBuild() error { return err } - file.Close() - - err = build.downloadInstalledFiles(packLogger) + err = build.downloadInstalledFiles() return err } @@ -186,7 +185,7 @@ func (build *Build) CleanUp() error { return nil } -func (build *Build) downloadInstalledFiles(packageLogger *bringauto_log.PackageLogger) error { +func (build *Build) downloadInstalledFiles() error { var err error copyDir := build.GetLocalInstallDirPath() @@ -197,11 +196,20 @@ func (build *Build) downloadInstalledFiles(packageLogger *bringauto_log.PackageL } } + packTarLogger := bringauto_log.GetLogger().CreatePackageContextLogger(build.Package.Name, bringauto_log.TarContext) + logFile, err := packTarLogger.GetFile() + + if err != nil { + return fmt.Errorf("failed to open file - %s", err) + } + + defer logFile.Close() + sftpClient := bringauto_ssh.SFTP{ RemoteDir: dockerInstallDirConst, EmptyLocalDir: copyDir, SSHCredentials: build.SSHCredentials, - PackageLogger : packageLogger, + LogWriter: logFile, } err = sftpClient.DownloadDirectory() return err diff --git a/modules/bringauto_log/GlobalLogger.go b/modules/bringauto_log/GlobalLogger.go index 6425cf9..f4a064f 100644 --- a/modules/bringauto_log/GlobalLogger.go +++ b/modules/bringauto_log/GlobalLogger.go @@ -52,9 +52,9 @@ func (logger *GlobalLogger) CheckPrerequisites(*bringauto_prerequisites.Args) er return nil } -func (logger *GlobalLogger) CreatePackageLogger(packageName string) *PackageLogger { - packageContextLogger := bringauto_prerequisites.CreateAndInitialize[PackageLogger]( - logger.timestamp, logger.logDirPath, packageName, +func (logger *GlobalLogger) CreatePackageContextLogger(packageName string, logContext string) *PackageContextLogger { + packageContextLogger := bringauto_prerequisites.CreateAndInitialize[PackageContextLogger]( + logger.timestamp, logger.logDirPath, packageName, logContext, ) return packageContextLogger } diff --git a/modules/bringauto_log/PackageContextLogger.go b/modules/bringauto_log/PackageContextLogger.go index 9392fa7..4f4a410 100644 --- a/modules/bringauto_log/PackageContextLogger.go +++ b/modules/bringauto_log/PackageContextLogger.go @@ -19,6 +19,7 @@ type PackageContextLogger struct { type packageContextLoggerInitArgs struct { Timestamp time.Time LogDirPath string + PackageName string LogContext string } @@ -35,7 +36,7 @@ func (logger *PackageContextLogger) FillDynamic(args *bringauto_prerequisites.Ar var argsStruct packageContextLoggerInitArgs bringauto_prerequisites.GetArgs(args, &argsStruct) logger.timestamp = argsStruct.Timestamp - logger.logDirPath = argsStruct.LogDirPath + logger.logDirPath = argsStruct.LogDirPath + "/" + argsStruct.PackageName logger.logFileName = argsStruct.LogContext + ".txt" } return nil diff --git a/modules/bringauto_log/PackageLogger.go b/modules/bringauto_log/PackageLogger.go deleted file mode 100644 index d1c6dc3..0000000 --- a/modules/bringauto_log/PackageLogger.go +++ /dev/null @@ -1,45 +0,0 @@ -package bringauto_log - -import ( - "bringauto/modules/bringauto_prerequisites" - "os" - "time" -) - -type PackageLogger struct { - Logger -} - -type packageLoggerInitArgs struct { - Timestamp time.Time - LogDirPath string - PackageName string -} - -func (logger *PackageLogger) FillDefault(*bringauto_prerequisites.Args) error { - logger.slogger = getDefaultLogger(os.Stdout) - logger.timestamp = time.Time{} - logger.logDirPath = "" - return nil -} - -func (logger *PackageLogger) FillDynamic(args *bringauto_prerequisites.Args) error { - if !bringauto_prerequisites.IsEmpty(args) { - var argsStruct packageLoggerInitArgs - bringauto_prerequisites.GetArgs(args, &argsStruct) - logger.timestamp = argsStruct.Timestamp - logger.logDirPath = argsStruct.LogDirPath + "/" + argsStruct.PackageName - } - return nil -} - -func (logger *PackageLogger) CheckPrerequisites(*bringauto_prerequisites.Args) error { - return nil -} - -func (logger *PackageLogger) CreatePackageContextLogger(logContext string) *PackageContextLogger { - packageContextLogger := bringauto_prerequisites.CreateAndInitialize[PackageContextLogger]( - logger.timestamp, logger.logDirPath, logContext, - ) - return packageContextLogger -} diff --git a/modules/bringauto_ssh/SFTP.go b/modules/bringauto_ssh/SFTP.go index e5af97c..1166e1a 100644 --- a/modules/bringauto_ssh/SFTP.go +++ b/modules/bringauto_ssh/SFTP.go @@ -1,7 +1,6 @@ package bringauto_ssh import ( - "bringauto/modules/bringauto_log" "bufio" "fmt" "github.com/pkg/sftp" @@ -23,7 +22,7 @@ type SFTP struct { // Empty, existing local directory where the RemoteDir will be copy EmptyLocalDir string SSHCredentials *SSHCredentials - PackageLogger *bringauto_log.PackageLogger + LogWriter io.Writer } // DownloadDirectory @@ -38,20 +37,13 @@ func (sftpd *SFTP) DownloadDirectory() error { SourceDir: "/INSTALL", } - tarLogger := sftpd.PackageLogger.CreatePackageContextLogger(bringauto_log.TarContext) - file, err := tarLogger.GetFile() - - if err != nil { - return err - } - shellEvaluator := ShellEvaluator{ Commands: tar.ConstructCMDLine(), - StdOut: file, + StdOut: sftpd.LogWriter, } err = shellEvaluator.RunOverSSH(*sftpd.SSHCredentials) - file.Close() + if err != nil { return fmt.Errorf("cannot archive /INSTALL dir in docker container - %s", err) } From ee35f78ffbf7e3ce2d8ab556b86796e6d24a0361 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 29 Jul 2024 14:17:31 +0200 Subject: [PATCH 10/91] Fix logging of different versions of same package --- modules/bringauto_build/Build.go | 4 ++-- modules/bringauto_package/Package.go | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/modules/bringauto_build/Build.go b/modules/bringauto_build/Build.go index 6931110..b2e70a0 100644 --- a/modules/bringauto_build/Build.go +++ b/modules/bringauto_build/Build.go @@ -110,7 +110,7 @@ func (build *Build) RunBuild() error { } logger := bringauto_log.GetLogger() - packBuildChainLogger := logger.CreatePackageContextLogger(build.Package.Name, bringauto_log.BuildChainContext) + packBuildChainLogger := logger.CreatePackageContextLogger(build.Package.GetShortPackageName(), bringauto_log.BuildChainContext) file, err := packBuildChainLogger.GetFile() if err != nil { @@ -196,7 +196,7 @@ func (build *Build) downloadInstalledFiles() error { } } - packTarLogger := bringauto_log.GetLogger().CreatePackageContextLogger(build.Package.Name, bringauto_log.TarContext) + packTarLogger := bringauto_log.GetLogger().CreatePackageContextLogger(build.Package.GetShortPackageName(), bringauto_log.TarContext) logFile, err := packTarLogger.GetFile() if err != nil { diff --git a/modules/bringauto_package/Package.go b/modules/bringauto_package/Package.go index ce5bdaa..8e83c18 100644 --- a/modules/bringauto_package/Package.go +++ b/modules/bringauto_package/Package.go @@ -113,6 +113,21 @@ func (packg *Package) CreatePackageName() string { return strings.Join(packageName, "") } +// Returns short package name without versions +func (packg *Package) GetShortPackageName() string { + var packageName []string + + packageName = append(packageName, packg.Name) + if packg.IsDebug { + packageName = append(packageName, "d") + } + if packg.IsDevLib { + packageName = append(packageName, "-dev") + } + + return strings.Join(packageName, "") +} + func createZIPArchive(sourceDir string, archivePath string) error { var files []string var err error From 1a6c17f5ea3f6b81cbc21f6686280f055e91d04b Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 29 Jul 2024 14:46:22 +0200 Subject: [PATCH 11/91] Replace deprecated code --- bap-builder/ContextManager.go | 9 ++++----- modules/bringauto_package/Package.go | 7 +++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/bap-builder/ContextManager.go b/bap-builder/ContextManager.go index 90108a4..b5a05aa 100644 --- a/bap-builder/ContextManager.go +++ b/bap-builder/ContextManager.go @@ -3,7 +3,6 @@ package main import ( "fmt" "io/fs" - "io/ioutil" "os" "path" "path/filepath" @@ -173,17 +172,17 @@ func getAllFilesInSubdirByRegexp(rootDir string, reg *regexp.Regexp) (map[string // get all files from given rootDir which matches given regexp func getAllFilesInDirByRegexp(rootDir string, reg *regexp.Regexp) ([]string, error) { var acceptedFileList []string - fileList, err := ioutil.ReadDir(rootDir) + dirEntryList, err := os.ReadDir(rootDir) if err != nil { return []string{}, fmt.Errorf("cannot list dir %s", rootDir) } - for _, packagesFileInfos := range fileList { - packageNameOk := reg.MatchString(packagesFileInfos.Name()) + for _, dirEntry := range dirEntryList { + packageNameOk := reg.MatchString(dirEntry.Name()) if !packageNameOk { continue } - acceptedFileList = append(acceptedFileList, path.Join(rootDir, packagesFileInfos.Name())) + acceptedFileList = append(acceptedFileList, path.Join(rootDir, dirEntry.Name())) } return acceptedFileList, nil } diff --git a/modules/bringauto_package/Package.go b/modules/bringauto_package/Package.go index 8e83c18..5d8f62e 100644 --- a/modules/bringauto_package/Package.go +++ b/modules/bringauto_package/Package.go @@ -5,7 +5,6 @@ import ( "compress/flate" "fmt" "github.com/mholt/archiver/v3" - "io/ioutil" "os" "path" "regexp" @@ -132,9 +131,9 @@ func createZIPArchive(sourceDir string, archivePath string) error { var files []string var err error - fileInfoList, err := ioutil.ReadDir(sourceDir) - for _, fileInfo := range fileInfoList { - files = append(files, path.Join(sourceDir, fileInfo.Name())) + dirEntryList, err := os.ReadDir(sourceDir) + for _, dirEntry := range dirEntryList { + files = append(files, path.Join(sourceDir, dirEntry.Name())) } zipArchive := archiver.Zip{ From 196abffbdcd9c1bd091b874d5d4eae37a8ecd845 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 12 Aug 2024 10:23:22 +0200 Subject: [PATCH 12/91] Simplify method for creating package name --- bap-builder/PackageMode.go | 4 ++-- modules/bringauto_build/Build.go | 4 ++-- modules/bringauto_package/Package.go | 16 +++++++++------- modules/bringauto_repository/GitLFSRepository.go | 4 ++-- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index 2a2bab8..63244b4 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -167,7 +167,7 @@ func buildAllPackages(cmdLine *BuildPackageCmdLineArgs, contextPath string) erro continue } count++ - logger.Info("Build %s", buildConfigs[0].Package.CreatePackageName()) + logger.Info("Build %s", buildConfigs[0].Package.CreatePackageName(false)) err = buildAndCopyPackage(cmdLine, &buildConfigs) if err != nil { logger.Fatal("cannot build package '%s' - %s", config.Package.Name, err) @@ -204,7 +204,7 @@ func buildSinglePackage(cmdLine *BuildPackageCmdLineArgs, contextPath string) er } buildConfigs := config.GetBuildStructure(*cmdLine.DockerImageName) - logger.Info("Build %s", buildConfigs[0].Package.CreatePackageName()) + logger.Info("Build %s", buildConfigs[0].Package.CreatePackageName(false)) err = buildAndCopyPackage(cmdLine, &buildConfigs) if err != nil { logger.Error("cannot build package '%s' - %s\n", packageName, err) diff --git a/modules/bringauto_build/Build.go b/modules/bringauto_build/Build.go index b2e70a0..bfffc88 100644 --- a/modules/bringauto_build/Build.go +++ b/modules/bringauto_build/Build.go @@ -110,7 +110,7 @@ func (build *Build) RunBuild() error { } logger := bringauto_log.GetLogger() - packBuildChainLogger := logger.CreatePackageContextLogger(build.Package.GetShortPackageName(), bringauto_log.BuildChainContext) + packBuildChainLogger := logger.CreatePackageContextLogger(build.Package.CreatePackageName(true), bringauto_log.BuildChainContext) file, err := packBuildChainLogger.GetFile() if err != nil { @@ -196,7 +196,7 @@ func (build *Build) downloadInstalledFiles() error { } } - packTarLogger := bringauto_log.GetLogger().CreatePackageContextLogger(build.Package.GetShortPackageName(), bringauto_log.TarContext) + packTarLogger := bringauto_log.GetLogger().CreatePackageContextLogger(build.Package.CreatePackageName(true), bringauto_log.TarContext) logFile, err := packTarLogger.GetFile() if err != nil { diff --git a/modules/bringauto_package/Package.go b/modules/bringauto_package/Package.go index 5d8f62e..0ee6c46 100644 --- a/modules/bringauto_package/Package.go +++ b/modules/bringauto_package/Package.go @@ -80,7 +80,7 @@ func (packg *Package) CreatePackage(sourceDir string, outputDir string) error { return err } - packageName := packg.CreatePackageName() + ".zip" + packageName := packg.CreatePackageName(false) + ".zip" err = createZIPArchive(sourceDir, outputDir+"/"+packageName) if err != nil { @@ -91,9 +91,9 @@ func (packg *Package) CreatePackage(sourceDir string, outputDir string) error { } // CreatePackageName -// construct only a package name. +// Construct only a package name. If short is true, returns shortened package name string. // Function returns nonempty string. -func (packg *Package) CreatePackageName() string { +func (packg *Package) CreatePackageName(short bool) string { var packageName []string if packg.IsLibrary { packageName = append([]string{"lib"}, packageName...) @@ -105,10 +105,12 @@ func (packg *Package) CreatePackageName() string { if packg.IsDevLib { packageName = append(packageName, "-dev") } - packageName = append(packageName, "_") - packageName = append(packageName, packg.VersionTag) - packageName = append(packageName, "_") - packageName = append(packageName, packg.PlatformString.Serialize()) + if !short { + packageName = append(packageName, "_") + packageName = append(packageName, packg.VersionTag) + packageName = append(packageName, "_") + packageName = append(packageName, packg.PlatformString.Serialize()) + } return strings.Join(packageName, "") } diff --git a/modules/bringauto_repository/GitLFSRepository.go b/modules/bringauto_repository/GitLFSRepository.go index 19d788b..b3ec7c2 100644 --- a/modules/bringauto_repository/GitLFSRepository.go +++ b/modules/bringauto_repository/GitLFSRepository.go @@ -100,7 +100,7 @@ func (lfs *GitLFSRepository) gitIsStatusEmpty() bool { } func (lfs *GitLFSRepository) gitAddAll(pack *bringauto_package.Package) error { - packageName := pack.CreatePackageName() + packageName := pack.CreatePackageName(false) var ok, _ = lfs.prepareAndRun([]string{ "add", ".", @@ -113,7 +113,7 @@ func (lfs *GitLFSRepository) gitAddAll(pack *bringauto_package.Package) error { } func (lfs *GitLFSRepository) gitCommit(pack *bringauto_package.Package) error { - packageName := pack.CreatePackageName() + packageName := pack.CreatePackageName(false) var ok, _ = lfs.prepareAndRun([]string{ "commit", "-m", From 33fadf80afe9fc550ef9597bb0bdee3c1398b6ca Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 12 Aug 2024 10:27:19 +0200 Subject: [PATCH 13/91] Add check for error --- modules/bringauto_package/Package.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/bringauto_package/Package.go b/modules/bringauto_package/Package.go index 0ee6c46..54c5304 100644 --- a/modules/bringauto_package/Package.go +++ b/modules/bringauto_package/Package.go @@ -134,6 +134,10 @@ func createZIPArchive(sourceDir string, archivePath string) error { var err error dirEntryList, err := os.ReadDir(sourceDir) + if err != nil { + return fmt.Errorf("cannot read %s directory: %s", sourceDir, err) + } + for _, dirEntry := range dirEntryList { files = append(files, path.Join(sourceDir, dirEntry.Name())) } From 36b9c0a4199c567c6cbb5de8f01da30351c10e24 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 12 Aug 2024 10:54:31 +0200 Subject: [PATCH 14/91] Add check for available port --- modules/bringauto_build/Build.go | 5 ++++ modules/bringauto_docker/DockerTools.go | 30 +++++++++++++++++++++ modules/bringauto_package/PlatformString.go | 5 ++++ 3 files changed, 40 insertions(+) create mode 100644 modules/bringauto_docker/DockerTools.go diff --git a/modules/bringauto_build/Build.go b/modules/bringauto_build/Build.go index bfffc88..4160491 100644 --- a/modules/bringauto_build/Build.go +++ b/modules/bringauto_build/Build.go @@ -125,6 +125,11 @@ func (build *Build) RunBuild() error { StdOut: file, } + availablePort := bringauto_docker.IsDefaultPortAvailable() + if !availablePort { + return fmt.Errorf("default port not available") + } + err = bringauto_prerequisites.Initialize(build.Docker) if err != nil { return err diff --git a/modules/bringauto_docker/DockerTools.go b/modules/bringauto_docker/DockerTools.go new file mode 100644 index 0000000..11b4b0e --- /dev/null +++ b/modules/bringauto_docker/DockerTools.go @@ -0,0 +1,30 @@ +package bringauto_docker + +import ( + "bringauto/modules/bringauto_process" + "bytes" +) + +func IsDefaultPortAvailable() bool { + var outBuff, errBuff bytes.Buffer + + process := bringauto_process.Process{ + CommandAbsolutePath: DockerExecutablePathConst, + Args: bringauto_process.ProcessArgs{ + ExtraArgs: &[]string{ + "container", + "ls", + "--filter", + "publish=1122", + "--format", + "{{.ID}}{{.Ports}}", + }, + }, + StdOut: &outBuff, + StdErr: &errBuff, + } + + process.Run() + + return outBuff.Len() == 0 +} \ No newline at end of file diff --git a/modules/bringauto_package/PlatformString.go b/modules/bringauto_package/PlatformString.go index 91a32ed..fc24347 100644 --- a/modules/bringauto_package/PlatformString.go +++ b/modules/bringauto_package/PlatformString.go @@ -118,6 +118,11 @@ func (pstr *PlatformString) determinePlatformString(credentials bringauto_ssh.SS panic(fmt.Errorf("cannot determine PlatformString for explicit mode")) } + availablePort := bringauto_docker.IsDefaultPortAvailable() + if !availablePort { + return fmt.Errorf("default port not available") + } + dockerRun := (*bringauto_docker.DockerRun)(docker) err := dockerRun.Run() if err != nil { From 20b096ac071f4d6a55abc53707b868a375fdf0a0 Mon Sep 17 00:00:00 2001 From: Ondrej Vala <93932164+Mayt34@users.noreply.github.com> Date: Mon, 12 Aug 2024 13:34:03 +0200 Subject: [PATCH 15/91] Unify JSON format across all packages --- example/package/cpprestsdk/cpprestsdk_debug.json | 7 +++++-- example/package/cpprestsdk/cpprestsdk_release.json | 7 +++++-- .../fleet_http_client_debug.json | 13 ++++++++++--- .../fleet_http_client_release.json | 13 ++++++++++--- .../fleet_protocol_cpp_debug.json | 2 +- .../fleet_protocol_cpp_release.json | 2 +- .../fleet_protocol_debug.json | 2 +- .../fleet_protocol_release.json | 2 +- example/package/protobuf/protobuf_debug_v3.17.json | 3 +-- .../package/protobuf/protobuf_release_v3.17.json | 3 +-- 10 files changed, 36 insertions(+), 18 deletions(-) diff --git a/example/package/cpprestsdk/cpprestsdk_debug.json b/example/package/cpprestsdk/cpprestsdk_debug.json index 9af053b..d1d6ab8 100644 --- a/example/package/cpprestsdk/cpprestsdk_debug.json +++ b/example/package/cpprestsdk/cpprestsdk_debug.json @@ -1,6 +1,9 @@ { - "Env": { }, - "DependsOn": [ "zlib", "boost" ], + "Env": {}, + "DependsOn": [ + "zlib", + "boost" + ], "Git": { "URI": "https://github.com/bringauto/cpprestsdk.git", "Revision": "v2.10.20" diff --git a/example/package/cpprestsdk/cpprestsdk_release.json b/example/package/cpprestsdk/cpprestsdk_release.json index c43e8e3..06801e1 100644 --- a/example/package/cpprestsdk/cpprestsdk_release.json +++ b/example/package/cpprestsdk/cpprestsdk_release.json @@ -1,6 +1,9 @@ { - "Env": { }, - "DependsOn": [ "zlib", "boost" ], + "Env": {}, + "DependsOn": [ + "zlib", + "boost" + ], "Git": { "URI": "https://github.com/bringauto/cpprestsdk.git", "Revision": "v2.10.20" diff --git a/example/package/fleet-http-client-shared/fleet_http_client_debug.json b/example/package/fleet-http-client-shared/fleet_http_client_debug.json index 6fb851f..6cf44eb 100644 --- a/example/package/fleet-http-client-shared/fleet_http_client_debug.json +++ b/example/package/fleet-http-client-shared/fleet_http_client_debug.json @@ -1,6 +1,13 @@ { "Env": {}, - "DependsOn": [ "zlib", "ba-logger", "boost", "fleet-protocol-interface", "fleet-protocol-cxx-helpers-static", "cpprestsdk" ], + "DependsOn": [ + "zlib", + "ba-logger", + "boost", + "fleet-protocol-interface", + "fleet-protocol-cxx-helpers-static", + "cpprestsdk" + ], "Git": { "URI": "https://github.com/bringauto/fleet-protocol-http-client-cxx.git", "Revision": "v1.3.0" @@ -12,8 +19,8 @@ "BRINGAUTO_INSTALL": "ON", "BRINGAUTO_PACKAGE": "ON", "BRINGAUTO_SYSTEM_DEP": "ON", - "BRINGAUTO_SAMPLES" : "OFF", - "BRINGAUTO_TESTS" : "OFF" + "BRINGAUTO_SAMPLES": "OFF", + "BRINGAUTO_TESTS": "OFF" } } }, diff --git a/example/package/fleet-http-client-shared/fleet_http_client_release.json b/example/package/fleet-http-client-shared/fleet_http_client_release.json index bd0424c..e95e34e 100644 --- a/example/package/fleet-http-client-shared/fleet_http_client_release.json +++ b/example/package/fleet-http-client-shared/fleet_http_client_release.json @@ -1,6 +1,13 @@ { "Env": {}, - "DependsOn": [ "zlib", "ba-logger", "boost", "fleet-protocol-interface", "fleet-protocol-cxx-helpers-static", "cpprestsdk" ], + "DependsOn": [ + "zlib", + "ba-logger", + "boost", + "fleet-protocol-interface", + "fleet-protocol-cxx-helpers-static", + "cpprestsdk" + ], "Git": { "URI": "https://github.com/bringauto/fleet-protocol-http-client-cxx.git", "Revision": "v1.3.0" @@ -12,8 +19,8 @@ "BRINGAUTO_INSTALL": "ON", "BRINGAUTO_PACKAGE": "ON", "BRINGAUTO_SYSTEM_DEP": "ON", - "BRINGAUTO_SAMPLES" : "OFF", - "BRINGAUTO_TESTS" : "OFF" + "BRINGAUTO_SAMPLES": "OFF", + "BRINGAUTO_TESTS": "OFF" } } }, diff --git a/example/package/fleet-protocol-cpp/fleet_protocol_cpp_debug.json b/example/package/fleet-protocol-cpp/fleet_protocol_cpp_debug.json index d5779a5..00490a5 100644 --- a/example/package/fleet-protocol-cpp/fleet_protocol_cpp_debug.json +++ b/example/package/fleet-protocol-cpp/fleet_protocol_cpp_debug.json @@ -14,7 +14,7 @@ "BRINGAUTO_INSTALL": "ON", "BRINGAUTO_PACKAGE": "ON", "BRINGAUTO_SYSTEM_DEP": "ON", - "BRINGAUTO_TESTS" : "OFF" + "BRINGAUTO_TESTS": "OFF" } } }, diff --git a/example/package/fleet-protocol-cpp/fleet_protocol_cpp_release.json b/example/package/fleet-protocol-cpp/fleet_protocol_cpp_release.json index 1c63330..410c05c 100644 --- a/example/package/fleet-protocol-cpp/fleet_protocol_cpp_release.json +++ b/example/package/fleet-protocol-cpp/fleet_protocol_cpp_release.json @@ -14,7 +14,7 @@ "BRINGAUTO_INSTALL": "ON", "BRINGAUTO_PACKAGE": "ON", "BRINGAUTO_SYSTEM_DEP": "ON", - "BRINGAUTO_TESTS" : "OFF" + "BRINGAUTO_TESTS": "OFF" } } }, diff --git a/example/package/fleet-protocol-interface/fleet_protocol_debug.json b/example/package/fleet-protocol-interface/fleet_protocol_debug.json index 3c1abeb..4221ff5 100644 --- a/example/package/fleet-protocol-interface/fleet_protocol_debug.json +++ b/example/package/fleet-protocol-interface/fleet_protocol_debug.json @@ -12,7 +12,7 @@ "BRINGAUTO_INSTALL": "ON", "BRINGAUTO_PACKAGE": "ON", "BRINGAUTO_SYSTEM_DEP": "ON", - "BRINGAUTO_SAMPLES" : "OFF" + "BRINGAUTO_SAMPLES": "OFF" } } }, diff --git a/example/package/fleet-protocol-interface/fleet_protocol_release.json b/example/package/fleet-protocol-interface/fleet_protocol_release.json index 4e8a060..589a314 100644 --- a/example/package/fleet-protocol-interface/fleet_protocol_release.json +++ b/example/package/fleet-protocol-interface/fleet_protocol_release.json @@ -12,7 +12,7 @@ "BRINGAUTO_INSTALL": "ON", "BRINGAUTO_PACKAGE": "ON", "BRINGAUTO_SYSTEM_DEP": "ON", - "BRINGAUTO_SAMPLES" : "OFF" + "BRINGAUTO_SAMPLES": "OFF" } } }, diff --git a/example/package/protobuf/protobuf_debug_v3.17.json b/example/package/protobuf/protobuf_debug_v3.17.json index 321efac..8e92bc3 100644 --- a/example/package/protobuf/protobuf_debug_v3.17.json +++ b/example/package/protobuf/protobuf_debug_v3.17.json @@ -25,7 +25,6 @@ "IsDebug": true }, "DockerMatrix": { - "ImageNames": [ - ] + "ImageNames": [] } } diff --git a/example/package/protobuf/protobuf_release_v3.17.json b/example/package/protobuf/protobuf_release_v3.17.json index 6539930..3594c97 100644 --- a/example/package/protobuf/protobuf_release_v3.17.json +++ b/example/package/protobuf/protobuf_release_v3.17.json @@ -25,7 +25,6 @@ "IsDebug": false }, "DockerMatrix": { - "ImageNames": [ - ] + "ImageNames": [] } } From ee5cd7a32df98142e7b56c921c90710418c22de1 Mon Sep 17 00:00:00 2001 From: Ondrej Vala <93932164+Mayt34@users.noreply.github.com> Date: Mon, 12 Aug 2024 14:02:04 +0200 Subject: [PATCH 16/91] Update READMEs --- README.md | 123 ++++++++++++++++++++++++---------------------- example/README.md | 31 +++++++++++- 2 files changed, 94 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 8110043..edc4bd1 100644 --- a/README.md +++ b/README.md @@ -1,35 +1,71 @@ - # BAP - BringAuto Packager -Build and track C/C++ project dependencies for apps for TODO any Linux distro! +Build and track C/C++ project dependencies for apps on any Linux distro! -BAM - simple way how to build and maintain our dependencies with almost zero learning curve and out-of-the -box integration into our workflows. +BAM provides a simple way to build and maintain dependencies with almost zero learning curve and out-of-the-box integration into your workflows. -## Usage +## Requirements -bap-builder build and stores all dependencies in the git repository -(LFS enabled recommended) +- Docker >= 20.10 (installed according to the official Docker documentation) +- git >= 2.25 -1. create git repository (optionally with LFS) +Standalone binaries are built for Linux kernel >= 5.10.0-amd64. -``` -mkdir lfsrepo && cd lfsrepo -git init -cd ../ -``` +## Build -2. Build all docker images needed for build +### Requirements for Build -``` -bap-builder build-image --context ./example --name debian11 -``` +- Go >= 1.22 -3. Build all packages for given distro +### Build from Source +Clone the repository and, in the repository root directory, run: + +```bash +go get bringauto/bap-builder +cd bap-builder +go build bringauto/bap-builder ``` -bap-builder build-package --context ./example --image-name debian11 --output-dir ./lfsrepo --all -``` + +## Build Standalone Binaries + +To build a complete release package, use the `build.sh` script. + +Additional requirements for `build.sh`: + +- zip +- uname +- sed + +## Usage + +The `bap-builder` builds and stores all dependencies in the git repository (LFS enabled is recommended). + +1. Create a git repository (optionally with LFS): + + ```bash + mkdir lfsrepo && cd lfsrepo + git init + cd ../ + ``` + +2. Build all Docker images needed for the build: + + ```bash + bap-builder build-image --context ./example --name debian11 + ``` + +3. Build all packages for the given distro: + + ```bash + bap-builder build-package --context ./example --image-name debian11 --output-dir ./lfsrepo --all + ``` + +*Note: If you do not have `bap-builder` in your system path, you need to use `./bap-builder/bap-builder` instead of `bap-builder`.* + +## Example + +In the `example` directory, there is a simple example of how to track dependencies for the `example` project. ## Motivation @@ -38,7 +74,7 @@ is consistent and work well. That's really hard to do because every disto has a different version of the same library. If you want to release you app to a new Linux distro you need to test every dependency, solve bugs, maintain your app in time... -There is no simple way how to track, maintain and pack consistent dependency set for C/C++ project without worrying +There is no simple way how to track, maintain and pack consistent dependency set for C/C++ project without worrying about different version of libraries, tools and the whole environment. The needed requirement is @@ -54,6 +90,7 @@ If you want to use one of existing solutions (Connan, Nix, ...) to develop relia In other ways the public repository of the system is not authoritative you cannot rely on the package pushed to the public repository.\ If you develop a reliable application you need to have authoritative source of the dependencies + - **You need to create your own instance of the system (or pay to someone to host instance for you)** - **You need to create and push your own packages to obtain authoritative source of dependencies.** @@ -70,48 +107,18 @@ BAM solve all these problems! It uses technologies and workflows that are known You can easily build and track dependencies for your project, download then and use them. -## Requirements - -- Docker >= 20.10 (installed by official Docker documentation) -- git >= 2.25 - -Standalone binaries are built for Linux kernel >= 5.10.0-amd64 - -## Build - -The project requires go >= 1.22. - -``` -go get bringauto/bap-builder -cd bap-builder -go build bringauto/bap-builder -``` - -## Build standalone binaries - -There is a script `build.sh` by that we can build a complete release package. - -Additional requirements for `build.sh`: - -- zip -- uname -- sed - ## FAQ -### Q: I have got a wierd error +### Q: I am encountering a weird error -Many errors are caused by problem with SSH connection to the Docker container -or impossibility to start Docker container itself. +Many errors are caused by issues with the SSH connection to the Docker container or the inability to start the Docker container itself. -In this case +In this case: -- check if there are no running docker container that -are instance of one of the Docker Images in the context directory -- check if there are no other container that uses port `1122` +- Check if there are no running Docker containers that are instances of one of the Docker images in the context directory. +- Check if there are no other containers using port `1122`. ## TODO -- SFTP copy and package creation are too slow (~20 minutes for Boost) -- Refactor `error` handling (use `Log` library, ...) and error messages - +- Improve SFTP copy and package creation speed (~20 minutes for Boost). +- Refactor error handling (use a logging library, etc.) and error messages. diff --git a/example/README.md b/example/README.md index 8a3da7f..b8e2084 100644 --- a/example/README.md +++ b/example/README.md @@ -1,4 +1,31 @@ +# BringAuto Package Example Context Directory -# BringAuto Package example context directory +This example context directory is used for testing and guidance. -Example context directory used for testing and guidance. \ No newline at end of file +## Helper Scripts + +- **`add_docker_to_matrix.sh`** - Adds a new Docker image to all package JSON files in the context directory. In the example, it adds the `ubuntu2310` Docker image to all package JSON files. + +- **`change_docker_name.sh`** - Changes the Docker image name in all package JSON files in the context directory. In the example, it changes `fleet-os` to `fleet-os-2`. + +## Package JSON Structure + +- **`Env`**: Environment variables used for the project. + +- **`DependsOn`**: List of external dependencies required for this project. + +- **`Git`**: Details about the Git repository for fetching the project source code. + - **`URI`**: The URL of the Git repository. + - **`Revision`**: The specific version tag or commit identifier. + +- **`Build`**: Contains build configuration settings using CMake, including various CMake options. + +- **`Package`**: Defines metadata for the project package. + - **`Name`**: The name of the package. + - **`VersionTag`**: The version tag for the package. + - **`PlatformString`**: Specifies the platform compatibility mode. + - **`IsLibrary`**: Indicates if the project is a library. + - **`IsDevLib`**: Specifies if the library is a development library. + - **`IsDebug`**: Indicates if the build is intended for debugging. + +- **`DockerMatrix`**: Lists Docker images for different platforms used for building or testing the project. From 4900bfca60c038e7a86fb424b6356232e8913ff5 Mon Sep 17 00:00:00 2001 From: Ondrej Vala <93932164+Mayt34@users.noreply.github.com> Date: Tue, 13 Aug 2024 15:05:26 +0200 Subject: [PATCH 17/91] Update doc --- doc/BuildProcess.md | 12 +++++------- doc/CMakeProjectRequirements.md | 2 +- doc/ContextDirectoryStructure.md | 26 ++++++++++++-------------- doc/DockerContainerRequiremetns.md | 2 +- doc/PackageDependencies.md | 2 +- doc/PackageJSONStructure.md | 22 +++++++++++----------- doc/PlatformString.md | 2 +- doc/README.md | 8 ++------ example/README.md | 22 ---------------------- 9 files changed, 34 insertions(+), 64 deletions(-) diff --git a/doc/BuildProcess.md b/doc/BuildProcess.md index 62c7578..17a86e6 100644 --- a/doc/BuildProcess.md +++ b/doc/BuildProcess.md @@ -5,7 +5,7 @@ Each package has a Config file in the JSON format. (Config file details in [Cont ## Build All packages -**Config phase** +### Config phase for all packages - the default Package config structure is created (with defaults filled in, look for `Config` in the config module). @@ -13,7 +13,7 @@ Each package has a Config file in the JSON format. (Config file details in [Cont - the default Package config is merged with the confile read from file. The config file data has a precedence over Default one. -**Build phase** +### Build phase for all packages - the package configs are stored in linear list - package config are topological sorted by `Dependencies`, @@ -30,15 +30,13 @@ packages are not mixed together. ## Build single package -**Config phase** +### Config phase for single package Same as for All build except that only configs for given `package_group` are read and managed. -**Build phase** +### Build phase for single package Same as for All build except that the dependencies are ignored. (so If you have not initialized `install_sysroot` then the package build fail) - - -[Context Directory Structure]: ./ContextDirectoryStructure.md \ No newline at end of file +[Context Directory Structure]: ./ContextDirectoryStructure.md diff --git a/doc/CMakeProjectRequirements.md b/doc/CMakeProjectRequirements.md index 55a8bd5..ff7c04c 100644 --- a/doc/CMakeProjectRequirements.md +++ b/doc/CMakeProjectRequirements.md @@ -7,4 +7,4 @@ Requirements - Project must be able to be installed by GNU Make - `make install` - Project must NOT override `CMAKE_INSTALL_PREFIX` CMake variable - it's used for the project installation to - a given directory and package creation. If you override it the build fail! \ No newline at end of file + a given directory and package creation. If you override it the build fail! diff --git a/doc/ContextDirectoryStructure.md b/doc/ContextDirectoryStructure.md index 40fbfb1..4d30374 100644 --- a/doc/ContextDirectoryStructure.md +++ b/doc/ContextDirectoryStructure.md @@ -6,21 +6,20 @@ configurations needed for BAM to work. In the Content Directory the definitions of packages, Docker images, ... are stored. -``` +``` plaintext / - docker/ - / - Dockerfile - ... - package/ - / - .json - .json - ... - ... + docker/ + / + Dockerfile + ... + package/ + / + .json + .json + ... + ... ``` - ## Docker Name The image name is recognized by a name of a directory in the `docker/` directory. @@ -41,5 +40,4 @@ Each package config mas have '.json' extension. The Package JSON format is described by [PackageJSONStructure] - -[PackageJSONStructure]: ./PackageJSONStructure.md \ No newline at end of file +[PackageJSONStructure]: ./PackageJSONStructure.md diff --git a/doc/DockerContainerRequiremetns.md b/doc/DockerContainerRequiremetns.md index 659744f..6bbe443 100644 --- a/doc/DockerContainerRequiremetns.md +++ b/doc/DockerContainerRequiremetns.md @@ -21,7 +21,7 @@ Each image that we can use for build our dependencies `lsb_release` and `uname` are used to construct platform string. -`lsb_release` must support +`lsb_release` must support - `-s` - short print that is easily parsable by machine - `-r` - release version (for Debian 11 it prints "11" if used with `-s` switch) diff --git a/doc/PackageDependencies.md b/doc/PackageDependencies.md index a4159e3..4de90a8 100644 --- a/doc/PackageDependencies.md +++ b/doc/PackageDependencies.md @@ -10,4 +10,4 @@ Basically, we can divide Package dependencies into two sets The "Dependencies" in our context mean "Local dependencies". -System dependencies are part of the Docker image and must be present on the host system. \ No newline at end of file +System dependencies are part of the Docker image and must be present on the host system. diff --git a/doc/PackageJSONStructure.md b/doc/PackageJSONStructure.md index 0f991a2..8880e7d 100644 --- a/doc/PackageJSONStructure.md +++ b/doc/PackageJSONStructure.md @@ -1,6 +1,6 @@ -### package_config +# package_config -``` +``` json { "Env": { "ENV_A": "Value A", @@ -15,7 +15,7 @@ }, "Build": { "CMake": { - "CMakeListDir": "", + "CMakeListDir": , "Defines": { // CMake variables passed to CMake -D switch "CMAKE_BUILD_TYPE": "Debug", "MY_NICE_VAR": "VarValue" @@ -38,25 +38,25 @@ } ``` -### uri +## uri valid Git URi that can be used by `git clone` command -### GitRevision_HashTagBranch +## GitRevision_HashTagBranch Valid git Hash, Tag or branch -### path_to_cmake_list_dir +## path_to_cmake_list_dir Directory where the CMakeLists.txt is located. Default value is "./" Path is relative against project git root. -### version_tag +## version_tag `version_tag` represents a version in normalized form. -``` +``` plaintext version_tag = 'v'x'.'y'.'z where x, y, z are from { 0, 1, 2, ... } ``` @@ -67,11 +67,11 @@ Examples: - v0.0.5 - v5.98.0 -### platform_string_mode +## platform_string_mode Platform String mode determine how the PlatformString is constructed -``` +``` plaintext platform_string_mode = "auto" | "any_machine" | "explicit" ``` @@ -83,7 +83,7 @@ as a machine part of the PlatformString "any". Debian 11 (x86_64) generated exam **explicit** - user must fill all three parts manually: -``` +``` json ... "PlatformString": { "Mode": "Explicit", diff --git a/doc/PlatformString.md b/doc/PlatformString.md index c9be58a..34a1d3f 100644 --- a/doc/PlatformString.md +++ b/doc/PlatformString.md @@ -3,4 +3,4 @@ Platform string is a string in form `machine`-`distro` -TODO: detailed description \ No newline at end of file +TODO: detailed description diff --git a/doc/README.md b/doc/README.md index d8de653..a937a4f 100644 --- a/doc/README.md +++ b/doc/README.md @@ -9,7 +9,6 @@ BringAuto Packager uses docker as a build environment and Git LFS as a storage b Both of these technologies (Git, Docker) are well known and do not need any special training for programmers whose want to use them. - ## Basics Package is a set of "data" mainly represented by files stored on the computer disk. @@ -59,8 +58,7 @@ Each package name consist from three parts `base_package_name` should contain only [a-zA-Z0-9-] chars. - -**Library package name creation** +### Library package name creation - `prefix` = "lib" - `debug_suffix` = "d" if the package is built as "Debug" @@ -68,7 +66,7 @@ Each package name consist from three parts - `library_type` = "-dev" if the package is development package (contains headers, ...) - `library_type` = "" if the package is a package contains only runtime lib -**Executable package name creation** +### Executable package name creation - `prefix` = "" - `debug_suffix` = "d" if the package is built as "Debug" @@ -85,8 +83,6 @@ Each package name consist from three parts - [Build Process] - [Package Dependencies] - - [Context Directory Structure]: ./ContextDirectoryStructure.md [Package JSON Structure]: ./PackageJSONStructure.md [Docker Container Requirements]: ./DockerContainerRequiremetns.md diff --git a/example/README.md b/example/README.md index b8e2084..bb2a9b8 100644 --- a/example/README.md +++ b/example/README.md @@ -7,25 +7,3 @@ This example context directory is used for testing and guidance. - **`add_docker_to_matrix.sh`** - Adds a new Docker image to all package JSON files in the context directory. In the example, it adds the `ubuntu2310` Docker image to all package JSON files. - **`change_docker_name.sh`** - Changes the Docker image name in all package JSON files in the context directory. In the example, it changes `fleet-os` to `fleet-os-2`. - -## Package JSON Structure - -- **`Env`**: Environment variables used for the project. - -- **`DependsOn`**: List of external dependencies required for this project. - -- **`Git`**: Details about the Git repository for fetching the project source code. - - **`URI`**: The URL of the Git repository. - - **`Revision`**: The specific version tag or commit identifier. - -- **`Build`**: Contains build configuration settings using CMake, including various CMake options. - -- **`Package`**: Defines metadata for the project package. - - **`Name`**: The name of the package. - - **`VersionTag`**: The version tag for the package. - - **`PlatformString`**: Specifies the platform compatibility mode. - - **`IsLibrary`**: Indicates if the project is a library. - - **`IsDevLib`**: Specifies if the library is a development library. - - **`IsDebug`**: Indicates if the build is intended for debugging. - -- **`DockerMatrix`**: Lists Docker images for different platforms used for building or testing the project. From be060d239c0cb27360d244a7fc18e059f6d4d08c Mon Sep 17 00:00:00 2001 From: Mikulas Date: Wed, 14 Aug 2024 09:58:12 +0200 Subject: [PATCH 18/91] Add check for overwriting sysroot files And also adds warning for nonempty sysroot directory. --- bap-builder/PackageMode.go | 32 ++++++++++++++++++++++++++++ modules/bringauto_sysroot/Sysroot.go | 27 +++++++++++++++++++++-- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index 63244b4..3a541af 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -10,6 +10,8 @@ import ( "bringauto/modules/bringauto_sysroot" "fmt" "strconv" + "os" + "io" ) type ( @@ -119,9 +121,39 @@ func (list *buildDepList) sortDependencies(rootName string, dependsMap *map[stri return &sorted } +// isDirEmpty +// Returns true if specified dir do not exists or exists but is empty, otherwise returns false +func isDirEmpty(dirPath string) (bool, error) { + f, err := os.Open(dirPath) + if err != nil { // The directory do not exists + return true, nil + } + defer f.Close() + + _, err = f.Readdirnames(1) + + if err == io.EOF { // The directory exists, but is empty + return true, nil + } + return false, err +} + +func checkSysrootDirectory() { + isDirEmpty, err := isDirEmpty(bringauto_sysroot.SysrootDirectoryName) + if !isDirEmpty { + logger := bringauto_log.GetLogger() + if err != nil { + logger.Warn("Cannot read in sysroot directory: %s", err) + } else { + logger.Warn("Sysroot directory is not empty - the package build may fail") + } + } +} + // BuildPackage // process Package mode of the program func BuildPackage(cmdLine *BuildPackageCmdLineArgs, contextPath string) error { + checkSysrootDirectory() buildAll := cmdLine.All if *buildAll { return buildAllPackages(cmdLine, contextPath) diff --git a/modules/bringauto_sysroot/Sysroot.go b/modules/bringauto_sysroot/Sysroot.go index 8e791c1..9938deb 100644 --- a/modules/bringauto_sysroot/Sysroot.go +++ b/modules/bringauto_sysroot/Sysroot.go @@ -6,11 +6,13 @@ import ( "fmt" "github.com/otiai10/copy" "os" + "io/fs" "path/filepath" + "strings" ) const ( - sysrootDirectoryName = "install_sysroot" + SysrootDirectoryName = "install_sysroot" ) // Sysroot represents a standard Linux sysroot with all needed libraries installed. @@ -39,6 +41,9 @@ func (sysroot *Sysroot) CheckPrerequisites(args *bringauto_prerequisites.Args) e // CopyToSysroot copy source to a sysroot func (sysroot *Sysroot) CopyToSysroot(source string) error { + if sysroot.anyFileAlreadyExistsInSysroot(source) { + return fmt.Errorf("trying to ovewrite files in sysroot - sysroot consistency interrupted") + } var err error copyOptions := copy.Options{ OnSymlink: onSymlink, @@ -52,6 +57,24 @@ func (sysroot *Sysroot) CopyToSysroot(source string) error { return nil } +func (sysroot *Sysroot) anyFileAlreadyExistsInSysroot(source string) bool { + sysrootPath := sysroot.GetSysrootPath() + + err := filepath.WalkDir(source, func(path string, d fs.DirEntry, err error) error { + if !d.IsDir() { + filePath := strings.TrimPrefix(path, source) + _, err := os.Stat(sysrootPath + filePath) + if err == nil { + return fmt.Errorf("file already exists in sysroot") + } + } + + return nil + }) + + return err != nil +} + // GetSysrootPath returns absolute path ot the sysroot func (sysroot *Sysroot) GetSysrootPath() string { workingDir, err := os.Getwd() @@ -65,7 +88,7 @@ func (sysroot *Sysroot) GetSysrootPath() string { sysrootDirName += "_debug" } - sysrootDir := filepath.Join(workingDir, sysrootDirectoryName, sysrootDirName) + sysrootDir := filepath.Join(workingDir, SysrootDirectoryName, sysrootDirName) return sysrootDir } From 18a97c50c6ca3a04b6732d85d709b37759ad8b02 Mon Sep 17 00:00:00 2001 From: Ondrej Vala <93932164+Mayt34@users.noreply.github.com> Date: Wed, 14 Aug 2024 10:52:38 +0200 Subject: [PATCH 19/91] Update Package JSON README --- README.md | 4 ++ doc/DockerContainerRequiremetns.md | 2 +- doc/PackageJSONStructure.md | 84 ++++++++++++++---------------- doc/ReliablePackageSource.md | 2 +- 4 files changed, 44 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index edc4bd1..aaed58c 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,10 @@ The `bap-builder` builds and stores all dependencies in the git repository (LFS *Note: If you do not have `bap-builder` in your system path, you need to use `./bap-builder/bap-builder` instead of `bap-builder`.* +## Documentation + +The detailed documentation is available in the `doc` directory. + ## Example In the `example` directory, there is a simple example of how to track dependencies for the `example` project. diff --git a/doc/DockerContainerRequiremetns.md b/doc/DockerContainerRequiremetns.md index 6bbe443..a2c0861 100644 --- a/doc/DockerContainerRequiremetns.md +++ b/doc/DockerContainerRequiremetns.md @@ -1,5 +1,5 @@ -# Docker container requirements +# Docker Container Requirements Each image that we can use for build our dependencies diff --git a/doc/PackageJSONStructure.md b/doc/PackageJSONStructure.md index 8880e7d..fcf05e2 100644 --- a/doc/PackageJSONStructure.md +++ b/doc/PackageJSONStructure.md @@ -1,63 +1,55 @@ -# package_config +# Package JSON Structure + +This document provides an example of the package configuration file. Note that this example does not follow the JSON format strictly because it contains comments and the values are just sample values. For a better understanding of the JSON format, check the `example/package` directory in this repository. + +Not all fields are required, and some fields have default values. ``` json { - "Env": { + "Env": { // Environment variables used for the project "ENV_A": "Value A", - "ENV_B": "Value B", + "ENV_B": "Value B" }, - "DependsOn": [ - + "DependsOn": [ // List of external dependencies required for this project + "protobuf", + "fleet-protocol-interface", + "zlib" ], - "Git": { - "URI": , - "Revision": + "Git": { // Details about the Git repository for fetching the project source code + "URI": "https://github.com/bringauto/example-repo.git", // Valid Git URI that can be used with the "git clone" command + "Revision": "v1.2.0" // Valid git hash, tag, or branch }, - "Build": { + "Build": { // Build configuration settings using CMake, including various CMake options "CMake": { - "CMakeListDir": , - "Defines": { // CMake variables passed to CMake -D switch + "CMakeListDir": "/cmake", // Directory where the CMakeLists.txt is located. Default value is "./", path is relative to the module's Git root + "Defines": { // CMake variables passed with the CMake -D switch "CMAKE_BUILD_TYPE": "Debug", "MY_NICE_VAR": "VarValue" } } }, - "Package": { - "Name": "PackageName", // package name from which to construct the package archive name - "VersionTag": , + "Package": { // Metadata for the project package + "Name": "PackageName", // Package name used to construct the package archive name + "VersionTag": "v5.98.0", // Detailed in the VersionTag section "PlatformString": { - "Mode": + "Mode": "auto", // Detailed in the Platform_String_Mode section }, - "IsLibrary": true, // if true add 'lib' prefix to the package name - "IsDevLib": true, // if true add '-dev' suffix to package name - "IsDebug": true // if true add 'd' to the package name (but before -dev prefix) + "IsLibrary": true, // If true, adds 'lib' prefix to the package name + "IsDevLib": true, // If true, adds '-dev' suffix to the package name + "IsDebug": true // If true, adds 'd' to the package name (but before the -dev suffix) }, - "DockerMatrix": { // from which Docker images names from docker/ repository this package wil be built + "DockerMatrix": { // Specifies the Docker images from the "docker/" directory used to build this package "ImageNames": [ "ubuntu1804", "ubuntu2004", "debian11" ] } } ``` -## uri - -valid Git URi that can be used by `git clone` command - -## GitRevision_HashTagBranch - -Valid git Hash, Tag or branch - -## path_to_cmake_list_dir +## Version_Tag -Directory where the CMakeLists.txt is located. Default value is "./" - -Path is relative against project git root. - -## version_tag - -`version_tag` represents a version in normalized form. +`VersionTag` represents a version in normalized form. ``` plaintext -version_tag = 'v'x'.'y'.'z +VersionTag = 'v'x'.'y'.'z where x, y, z are from { 0, 1, 2, ... } ``` @@ -67,31 +59,31 @@ Examples: - v0.0.5 - v5.98.0 -## platform_string_mode +## Platform_String_Mode -Platform String mode determine how the PlatformString is constructed +Platform String mode determines how the PlatformString is constructed. ``` plaintext -platform_string_mode = "auto" | "any_machine" | "explicit" +Mode = "auto" | "any_machine" | "explicit" ``` -**auto** - construct PlatformString for a target machine by an automatic way. -Debian 11 (x86_64) generated example: "x86-64-debian-11" +**auto** - Constructs the PlatformString for a target machine automatically. +Example for Debian 11 (x86_64): "x86-64-debian-11" -**any_machine** - construct platform string that is specific only for Distro name and distro release. -as a machine part of the PlatformString "any". Debian 11 (x86_64) generated example: "any-debian-11" +**any_machine** - Constructs the platform string specific only to the Distro name and distro release, with the machine part of the PlatformString set to "any". +Example for Debian 11 (x86_64): "any-debian-11" -**explicit** - user must fill all three parts manually: +**explicit** - User must fill in all three parts manually: ``` json ... "PlatformString": { - "Mode": "Explicit", + "Mode": "explicit", "String": { "DistroName": "Debian", "DistroRelease": "11", "Machine": "x86-64" } - }, + } ... ``` diff --git a/doc/ReliablePackageSource.md b/doc/ReliablePackageSource.md index d24b502..9e90fde 100644 --- a/doc/ReliablePackageSource.md +++ b/doc/ReliablePackageSource.md @@ -1,5 +1,5 @@ -# Reliable Package Source rules +# Reliable Package Source Rules ## Problem From 989c900f4eb56c3ddb52bccfaf7d750bef550a8f Mon Sep 17 00:00:00 2001 From: Mikulas Date: Wed, 14 Aug 2024 11:27:34 +0200 Subject: [PATCH 20/91] Rework of package name getters --- bap-builder/PackageMode.go | 4 +-- modules/bringauto_build/Build.go | 4 +-- modules/bringauto_package/Package.go | 34 +++++++------------ .../bringauto_repository/GitLFSRepository.go | 4 +-- modules/bringauto_repository/Repository.go | 2 +- 5 files changed, 19 insertions(+), 29 deletions(-) diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index 3a541af..a063208 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -199,7 +199,7 @@ func buildAllPackages(cmdLine *BuildPackageCmdLineArgs, contextPath string) erro continue } count++ - logger.Info("Build %s", buildConfigs[0].Package.CreatePackageName(false)) + logger.Info("Build %s", buildConfigs[0].Package.GetFullPackageName()) err = buildAndCopyPackage(cmdLine, &buildConfigs) if err != nil { logger.Fatal("cannot build package '%s' - %s", config.Package.Name, err) @@ -236,7 +236,7 @@ func buildSinglePackage(cmdLine *BuildPackageCmdLineArgs, contextPath string) er } buildConfigs := config.GetBuildStructure(*cmdLine.DockerImageName) - logger.Info("Build %s", buildConfigs[0].Package.CreatePackageName(false)) + logger.Info("Build %s", buildConfigs[0].Package.GetFullPackageName()) err = buildAndCopyPackage(cmdLine, &buildConfigs) if err != nil { logger.Error("cannot build package '%s' - %s\n", packageName, err) diff --git a/modules/bringauto_build/Build.go b/modules/bringauto_build/Build.go index 4160491..fd3b8d8 100644 --- a/modules/bringauto_build/Build.go +++ b/modules/bringauto_build/Build.go @@ -110,7 +110,7 @@ func (build *Build) RunBuild() error { } logger := bringauto_log.GetLogger() - packBuildChainLogger := logger.CreatePackageContextLogger(build.Package.CreatePackageName(true), bringauto_log.BuildChainContext) + packBuildChainLogger := logger.CreatePackageContextLogger(build.Package.GetShortPackageName(), bringauto_log.BuildChainContext) file, err := packBuildChainLogger.GetFile() if err != nil { @@ -201,7 +201,7 @@ func (build *Build) downloadInstalledFiles() error { } } - packTarLogger := bringauto_log.GetLogger().CreatePackageContextLogger(build.Package.CreatePackageName(true), bringauto_log.TarContext) + packTarLogger := bringauto_log.GetLogger().CreatePackageContextLogger(build.Package.GetShortPackageName(), bringauto_log.TarContext) logFile, err := packTarLogger.GetFile() if err != nil { diff --git a/modules/bringauto_package/Package.go b/modules/bringauto_package/Package.go index 54c5304..6ecc50a 100644 --- a/modules/bringauto_package/Package.go +++ b/modules/bringauto_package/Package.go @@ -80,7 +80,7 @@ func (packg *Package) CreatePackage(sourceDir string, outputDir string) error { return err } - packageName := packg.CreatePackageName(false) + ".zip" + packageName := packg.GetFullPackageName() + ".zip" err = createZIPArchive(sourceDir, outputDir+"/"+packageName) if err != nil { @@ -90,10 +90,9 @@ func (packg *Package) CreatePackage(sourceDir string, outputDir string) error { return nil } -// CreatePackageName -// Construct only a package name. If short is true, returns shortened package name string. -// Function returns nonempty string. -func (packg *Package) CreatePackageName(short bool) string { +// GetShortPackageName +// Returns short package name without version and platform string. +func (packg *Package) GetShortPackageName() string { var packageName []string if packg.IsLibrary { packageName = append([]string{"lib"}, packageName...) @@ -105,27 +104,18 @@ func (packg *Package) CreatePackageName(short bool) string { if packg.IsDevLib { packageName = append(packageName, "-dev") } - if !short { - packageName = append(packageName, "_") - packageName = append(packageName, packg.VersionTag) - packageName = append(packageName, "_") - packageName = append(packageName, packg.PlatformString.Serialize()) - } return strings.Join(packageName, "") } -// Returns short package name without versions -func (packg *Package) GetShortPackageName() string { +// GetFullPackageName +// Returns full package name with version and platform string. +func (packg *Package) GetFullPackageName() string { var packageName []string - - packageName = append(packageName, packg.Name) - if packg.IsDebug { - packageName = append(packageName, "d") - } - if packg.IsDevLib { - packageName = append(packageName, "-dev") - } - + packageName = append(packageName, packg.GetShortPackageName()) + packageName = append(packageName, "_") + packageName = append(packageName, packg.VersionTag) + packageName = append(packageName, "_") + packageName = append(packageName, packg.PlatformString.Serialize()) return strings.Join(packageName, "") } diff --git a/modules/bringauto_repository/GitLFSRepository.go b/modules/bringauto_repository/GitLFSRepository.go index b3ec7c2..782c836 100644 --- a/modules/bringauto_repository/GitLFSRepository.go +++ b/modules/bringauto_repository/GitLFSRepository.go @@ -100,7 +100,7 @@ func (lfs *GitLFSRepository) gitIsStatusEmpty() bool { } func (lfs *GitLFSRepository) gitAddAll(pack *bringauto_package.Package) error { - packageName := pack.CreatePackageName(false) + packageName := pack.GetFullPackageName() var ok, _ = lfs.prepareAndRun([]string{ "add", ".", @@ -113,7 +113,7 @@ func (lfs *GitLFSRepository) gitAddAll(pack *bringauto_package.Package) error { } func (lfs *GitLFSRepository) gitCommit(pack *bringauto_package.Package) error { - packageName := pack.CreatePackageName(false) + packageName := pack.GetFullPackageName() var ok, _ = lfs.prepareAndRun([]string{ "commit", "-m", diff --git a/modules/bringauto_repository/Repository.go b/modules/bringauto_repository/Repository.go index 2229431..29334f9 100644 --- a/modules/bringauto_repository/Repository.go +++ b/modules/bringauto_repository/Repository.go @@ -8,6 +8,6 @@ type RepositoryInterface interface { // CopyToRepository copy package files represented by sourceDir to the repository. // Each repository has a different semantics for managing structure of th repository. // - // Repository must not change the package name represented by pack.CreatePackageName() + // Repository must not change the package name represented by pack.GetFullPackageName() CopyToRepository(pack bringauto_package.Package, sourceDir string) error } From 6a43a27a87400f2f9d5e67b95224a7ea52f8b44d Mon Sep 17 00:00:00 2001 From: Mikulas Date: Wed, 14 Aug 2024 11:49:48 +0200 Subject: [PATCH 21/91] Rework of empty sysroot directory check --- bap-builder/PackageMode.go | 36 +++-------------------- modules/bringauto_sysroot/Sysroot.go | 4 +-- modules/bringauto_sysroot/SysrootTools.go | 27 +++++++++++++++++ 3 files changed, 33 insertions(+), 34 deletions(-) create mode 100644 modules/bringauto_sysroot/SysrootTools.go diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index a063208..e007bb1 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -10,8 +10,6 @@ import ( "bringauto/modules/bringauto_sysroot" "fmt" "strconv" - "os" - "io" ) type ( @@ -121,39 +119,13 @@ func (list *buildDepList) sortDependencies(rootName string, dependsMap *map[stri return &sorted } -// isDirEmpty -// Returns true if specified dir do not exists or exists but is empty, otherwise returns false -func isDirEmpty(dirPath string) (bool, error) { - f, err := os.Open(dirPath) - if err != nil { // The directory do not exists - return true, nil - } - defer f.Close() - - _, err = f.Readdirnames(1) - - if err == io.EOF { // The directory exists, but is empty - return true, nil - } - return false, err -} - -func checkSysrootDirectory() { - isDirEmpty, err := isDirEmpty(bringauto_sysroot.SysrootDirectoryName) - if !isDirEmpty { - logger := bringauto_log.GetLogger() - if err != nil { - logger.Warn("Cannot read in sysroot directory: %s", err) - } else { - logger.Warn("Sysroot directory is not empty - the package build may fail") - } - } -} - // BuildPackage // process Package mode of the program func BuildPackage(cmdLine *BuildPackageCmdLineArgs, contextPath string) error { - checkSysrootDirectory() + if !bringauto_sysroot.IsSysrootDirectoryEmpty() { + logger := bringauto_log.GetLogger() + logger.Warn("Sysroot directory is not empty - the package build may fail") + } buildAll := cmdLine.All if *buildAll { return buildAllPackages(cmdLine, contextPath) diff --git a/modules/bringauto_sysroot/Sysroot.go b/modules/bringauto_sysroot/Sysroot.go index 9938deb..8ef3752 100644 --- a/modules/bringauto_sysroot/Sysroot.go +++ b/modules/bringauto_sysroot/Sysroot.go @@ -12,7 +12,7 @@ import ( ) const ( - SysrootDirectoryName = "install_sysroot" + sysrootDirectoryName = "install_sysroot" ) // Sysroot represents a standard Linux sysroot with all needed libraries installed. @@ -88,7 +88,7 @@ func (sysroot *Sysroot) GetSysrootPath() string { sysrootDirName += "_debug" } - sysrootDir := filepath.Join(workingDir, SysrootDirectoryName, sysrootDirName) + sysrootDir := filepath.Join(workingDir, sysrootDirectoryName, sysrootDirName) return sysrootDir } diff --git a/modules/bringauto_sysroot/SysrootTools.go b/modules/bringauto_sysroot/SysrootTools.go new file mode 100644 index 0000000..a9aa267 --- /dev/null +++ b/modules/bringauto_sysroot/SysrootTools.go @@ -0,0 +1,27 @@ +package bringauto_sysroot + +import ( + "bringauto/modules/bringauto_log" + "os" + "io" +) + +// IsSysrootDirectoryEmpty +// Returns true if specified dir do not exists or exists but is empty, otherwise returns false +func IsSysrootDirectoryEmpty() bool { + f, err := os.Open(sysrootDirectoryName) + if err != nil { // The directory do not exists + return true + } + defer f.Close() + + _, err = f.Readdirnames(1) + + if err == io.EOF { // The directory exists, but is empty + return true + } else if err != nil { + bringauto_log.GetLogger().Warn("Cannot read in sysroot directory: %s", err) + } + + return false +} From 491e5cdfe68346525edb9c93e9293eceb39e50de Mon Sep 17 00:00:00 2001 From: Mikulas Date: Thu, 15 Aug 2024 14:11:37 +0200 Subject: [PATCH 22/91] Add --build-deps flag, refactor, doc update When building single package the new flag can be used to build all dependencies. The PackageMode was refactored and new feature mentioned in doc. --- bap-builder/CmdArgs.go | 9 ++++ bap-builder/PackageMode.go | 99 +++++++++++++++++++++++++++++--------- doc/BuildProcess.md | 6 ++- 3 files changed, 90 insertions(+), 24 deletions(-) diff --git a/bap-builder/CmdArgs.go b/bap-builder/CmdArgs.go index 71c5224..072beb2 100644 --- a/bap-builder/CmdArgs.go +++ b/bap-builder/CmdArgs.go @@ -27,6 +27,8 @@ type BuildPackageCmdLineArgs struct { All *bool // Name of the package to build (name of the directory in packages/ dir) Name *string + // Build all dependencies of package when building single package + BuildDeps *bool // DockerImageName is a name of docker image to which packages will be build. // If empty all docker images from DockerMatrix in config file are used for a given package. // If not empty, only packages which contains DockerImageName in DockerMatrix will be built. @@ -87,6 +89,13 @@ func (cmd *CmdLineArgs) InitFlags() { Help: "Name of the package to build", }, ) + cmd.BuildPackageArgs.BuildDeps = cmd.buildPackageParser.Flag("", "build-deps", + &argparse.Options{ + Required: false, + Default: false, + Help: "Build all dependencies of package when building single package", + }, + ) cmd.BuildPackageArgs.OutputDir = cmd.buildPackageParser.String("", "output-dir", &argparse.Options{ Required: true, diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index e007bb1..ea4ec60 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -140,27 +140,17 @@ func buildAllPackages(cmdLine *BuildPackageCmdLineArgs, contextPath string) erro contextManager := ContextManager{ ContextPath: contextPath, } - packagesDefs, err := contextManager.GetAllPackagesJsonDefPaths() + packageJsonPathMap, err := contextManager.GetAllPackagesJsonDefPaths() if err != nil { return err } - defsList := make(map[string][]*bringauto_config.Config) - for _, packageJsonDef := range packagesDefs { - for _, defdef := range packageJsonDef { - var config bringauto_config.Config - err = config.LoadJSONConfig(defdef) - packageName := config.Package.Name - _, found := defsList[packageName] - if !found { - defsList[packageName] = []*bringauto_config.Config{} - } - defsList[packageName] = append(defsList[packageName], &config) - } - + defsMap := make(ConfigMapType) + for _, packageJsonPathList := range packageJsonPathMap { + addConfigsToDefsMap(&defsMap, packageJsonPathList) } depsList := buildDepList{} - configList := depsList.TopologicalSort(defsList) + configList := depsList.TopologicalSort(defsMap) logger := bringauto_log.GetLogger() @@ -192,21 +182,36 @@ func buildSinglePackage(cmdLine *BuildPackageCmdLineArgs, contextPath string) er ContextPath: contextPath, } packageName := *cmdLine.Name - packageJsonDefsList, err := contextManager.GetPackageJsonDefPaths(packageName) + packageJsonPathList, err := contextManager.GetPackageJsonDefPaths(packageName) if err != nil { return err } logger := bringauto_log.GetLogger() - for _, packageJsonDef := range packageJsonDefsList { - var config bringauto_config.Config - err = config.LoadJSONConfig(packageJsonDef) - if err != nil { - logger.Warn("package '%s' JSON config def problem - %s\n", packageName, err) - continue + var configList []*bringauto_config.Config + + if *cmdLine.BuildDeps { + for _, packageJsonPath := range packageJsonPathList { + packageJsonPathList = append(packageJsonPathList, getAllDepsJsonPaths(packageJsonPath, contextManager)...) } + defsMap := make(ConfigMapType) + addConfigsToDefsMap(&defsMap, packageJsonPathList) + depList := buildDepList{} + configList = depList.TopologicalSort(defsMap) + } else { + for _, packageJsonPath := range packageJsonPathList { + var config bringauto_config.Config + err = config.LoadJSONConfig(packageJsonPath) + if err != nil { + logger.Warn("Couldn't load JSON config from %s path - %s", packageJsonPath, err) + continue + } + configList = append(configList, &config) + } + } + for _, config := range configList { buildConfigs := config.GetBuildStructure(*cmdLine.DockerImageName) logger.Info("Build %s", buildConfigs[0].Package.GetFullPackageName()) err = buildAndCopyPackage(cmdLine, &buildConfigs) @@ -218,6 +223,56 @@ func buildSinglePackage(cmdLine *BuildPackageCmdLineArgs, contextPath string) er return nil } +func addConfigsToDefsMap(defsMap *ConfigMapType, packageJsonPathList []string) { + for _, packageJsonPath := range packageJsonPathList { + var config bringauto_config.Config + err := config.LoadJSONConfig(packageJsonPath) + if err != nil { + logger := bringauto_log.GetLogger() + logger.Error("Couldn't load JSON config from %s path - %s", packageJsonPath, err) + continue + } + packageName := config.Package.Name + _, found := (*defsMap)[packageName] + if !found { + (*defsMap)[packageName] = []*bringauto_config.Config{} + } + (*defsMap)[packageName] = append((*defsMap)[packageName], &config) + } +} + +func getAllDepsJsonPaths(packageJsonPath string, contextManager ContextManager) []string { + var jsonPathListWithDeps []string + var config bringauto_config.Config + logger := bringauto_log.GetLogger() + err := config.LoadJSONConfig(packageJsonPath) + if err != nil { + logger.Warn("Couldn't load JSON config from %s path - %s", packageJsonPath, err) + return jsonPathListWithDeps + } + for _, packageDep := range config.DependsOn { + packageDepsJsonPaths, err := contextManager.GetPackageJsonDefPaths(packageDep) + if err != nil { + logger.Warn("Couldn't get Json Path of %s package", packageDep) + continue + } + var depConfig bringauto_config.Config + for _, packageDepJsonPath := range packageDepsJsonPaths { + err := depConfig.LoadJSONConfig(packageDepJsonPath) + if err != nil { + logger.Warn("Couldn't load JSON config from %s path - %s", packageJsonPath, err) + continue + } + if depConfig.Package.IsDebug == config.Package.IsDebug { + jsonPathListWithDeps = append(jsonPathListWithDeps, packageDepJsonPath) + jsonPathListWithDeps = append(jsonPathListWithDeps, getAllDepsJsonPaths(packageDepJsonPath, contextManager)...) + } + } + } + + return jsonPathListWithDeps +} + func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_build.Build) error { if *cmdLine.OutputDirMode != OutputDirModeGitLFS { return fmt.Errorf("invalid OutputDirmode. Only GitLFS is supported") diff --git a/doc/BuildProcess.md b/doc/BuildProcess.md index 17a86e6..bbd14b4 100644 --- a/doc/BuildProcess.md +++ b/doc/BuildProcess.md @@ -36,7 +36,9 @@ Same as for All build except that only configs for given `package_group` are rea ### Build phase for single package -Same as for All build except that the dependencies are ignored. (so If you have not initialized `install_sysroot` -then the package build fail) +Same as for All build except that the dependencies are ignored. (so if you have not initialized +`install_sysroot` then the package build may fail) + +If you want to build package with all dependencies, you can add `--build-deps` flag to script call. [Context Directory Structure]: ./ContextDirectoryStructure.md From f4c154061495201e2938bffd1f0457c866b8210c Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 19 Aug 2024 10:10:40 +0200 Subject: [PATCH 23/91] Move getting package with deps paths to ContextManager --- bap-builder/ContextManager.go | 51 +++++++++++++++++++++++++++++++++++ bap-builder/PackageMode.go | 47 ++++++-------------------------- 2 files changed, 59 insertions(+), 39 deletions(-) diff --git a/bap-builder/ContextManager.go b/bap-builder/ContextManager.go index b5a05aa..cfa0eea 100644 --- a/bap-builder/ContextManager.go +++ b/bap-builder/ContextManager.go @@ -1,6 +1,8 @@ package main import ( + "bringauto/modules/bringauto_config" + "bringauto/modules/bringauto_log" "fmt" "io/fs" "os" @@ -87,6 +89,55 @@ func (context *ContextManager) GetPackageJsonDefPaths(packageName string) ([]str return packageDefs, nil } +func (context *ContextManager) getAllDepsJsonPaths(packageJsonPath string) []string { + var config bringauto_config.Config + logger := bringauto_log.GetLogger() + err := config.LoadJSONConfig(packageJsonPath) + if err != nil { + logger.Warn("Couldn't load JSON config from %s path - %s", packageJsonPath, err) + return []string{} + } + var jsonPathListWithDeps []string + for _, packageDep := range config.DependsOn { + packageDepsJsonPaths, err := context.GetPackageJsonDefPaths(packageDep) + if err != nil { + logger.Warn("Couldn't get Json Path of %s package", packageDep) + continue + } + var depConfig bringauto_config.Config + for _, packageDepJsonPath := range packageDepsJsonPaths { + err := depConfig.LoadJSONConfig(packageDepJsonPath) + if err != nil { + logger.Warn("Couldn't load JSON config from %s path - %s", packageJsonPath, err) + continue + } + if depConfig.Package.IsDebug == config.Package.IsDebug { + jsonPathListWithDeps = append(jsonPathListWithDeps, packageDepJsonPath) + jsonPathListWithDeps = append(jsonPathListWithDeps, context.getAllDepsJsonPaths(packageDepJsonPath)...) + } + } + } + + return jsonPathListWithDeps +} + +// GetPackageWithDepsJsonDefPaths +// returns all json definitions for given package and all its dependencies json definitions recursively +func (context *ContextManager) GetPackageWithDepsJsonDefPaths(packageName string) ([]string, error) { + packageDefs, err := context.GetPackageJsonDefPaths(packageName) + if err != nil { + return []string{}, fmt.Errorf("cannot get config paths for package '%s'", packageName) + } + var packageDeps []string + for _, packageDef := range packageDefs { + packageDeps = append(packageDeps, context.getAllDepsJsonPaths(packageDef)...) + } + + packageDefs = append(packageDefs, packageDeps...) + + return packageDefs, nil +} + // GetImageDockerfilePath // returns Dockerfile path for the given Image locate in the given context func (context *ContextManager) GetImageDockerfilePath(imageName string) (string, error) { diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index ea4ec60..c558a4a 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -182,24 +182,25 @@ func buildSinglePackage(cmdLine *BuildPackageCmdLineArgs, contextPath string) er ContextPath: contextPath, } packageName := *cmdLine.Name - packageJsonPathList, err := contextManager.GetPackageJsonDefPaths(packageName) - if err != nil { - return err - } - + var err error logger := bringauto_log.GetLogger() var configList []*bringauto_config.Config if *cmdLine.BuildDeps { - for _, packageJsonPath := range packageJsonPathList { - packageJsonPathList = append(packageJsonPathList, getAllDepsJsonPaths(packageJsonPath, contextManager)...) + packageJsonPathList, err := contextManager.GetPackageWithDepsJsonDefPaths(packageName) + if err != nil { + return err } defsMap := make(ConfigMapType) addConfigsToDefsMap(&defsMap, packageJsonPathList) depList := buildDepList{} configList = depList.TopologicalSort(defsMap) } else { + packageJsonPathList, err := contextManager.GetPackageJsonDefPaths(packageName) + if err != nil { + return err + } for _, packageJsonPath := range packageJsonPathList { var config bringauto_config.Config err = config.LoadJSONConfig(packageJsonPath) @@ -241,38 +242,6 @@ func addConfigsToDefsMap(defsMap *ConfigMapType, packageJsonPathList []string) { } } -func getAllDepsJsonPaths(packageJsonPath string, contextManager ContextManager) []string { - var jsonPathListWithDeps []string - var config bringauto_config.Config - logger := bringauto_log.GetLogger() - err := config.LoadJSONConfig(packageJsonPath) - if err != nil { - logger.Warn("Couldn't load JSON config from %s path - %s", packageJsonPath, err) - return jsonPathListWithDeps - } - for _, packageDep := range config.DependsOn { - packageDepsJsonPaths, err := contextManager.GetPackageJsonDefPaths(packageDep) - if err != nil { - logger.Warn("Couldn't get Json Path of %s package", packageDep) - continue - } - var depConfig bringauto_config.Config - for _, packageDepJsonPath := range packageDepsJsonPaths { - err := depConfig.LoadJSONConfig(packageDepJsonPath) - if err != nil { - logger.Warn("Couldn't load JSON config from %s path - %s", packageJsonPath, err) - continue - } - if depConfig.Package.IsDebug == config.Package.IsDebug { - jsonPathListWithDeps = append(jsonPathListWithDeps, packageDepJsonPath) - jsonPathListWithDeps = append(jsonPathListWithDeps, getAllDepsJsonPaths(packageDepJsonPath, contextManager)...) - } - } - } - - return jsonPathListWithDeps -} - func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_build.Build) error { if *cmdLine.OutputDirMode != OutputDirModeGitLFS { return fmt.Errorf("invalid OutputDirmode. Only GitLFS is supported") From ea0c6903888f019bff334d25852a9e989f31ba02 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 19 Aug 2024 14:02:59 +0200 Subject: [PATCH 24/91] Add file logging to build-image command, refactor --- bap-builder/DockerMode.go | 14 ++++++------ modules/bringauto_build/Build.go | 4 ++-- modules/bringauto_docker/DockerBuild.go | 22 ++++++++++--------- ...ckageContextLogger.go => ContextLogger.go} | 22 ++++++++++--------- modules/bringauto_log/GlobalLogger.go | 6 ++--- 5 files changed, 36 insertions(+), 32 deletions(-) rename modules/bringauto_log/{PackageContextLogger.go => ContextLogger.go} (60%) diff --git a/bap-builder/DockerMode.go b/bap-builder/DockerMode.go index d79b6d4..5179a3c 100644 --- a/bap-builder/DockerMode.go +++ b/bap-builder/DockerMode.go @@ -43,9 +43,9 @@ func buildAllDockerImages(contextPath string) error { DockerfileDir: dockerfileDir, Tag: imageName, } - buildOk := dockerBuild.Build() - if buildOk != nil { - return fmt.Errorf("build failed for '%s'", imageName) + err = dockerBuild.Build() + if err != nil { + return fmt.Errorf("Build failed for %s image", imageName) } } return nil @@ -54,11 +54,11 @@ func buildAllDockerImages(contextPath string) error { // buildSingleDockerImage // builds a single docker image specified by a name. // -func buildSingleDockerImage(contextPath string, name string) error { +func buildSingleDockerImage(contextPath string, imageName string) error { contextManager := ContextManager{ ContextPath: contextPath, } - dockerfilePath, err := contextManager.GetImageDockerfilePath(name) + dockerfilePath, err := contextManager.GetImageDockerfilePath(imageName) if err != nil { return err } @@ -66,11 +66,11 @@ func buildSingleDockerImage(contextPath string, name string) error { dockerfileDir := path.Dir(dockerfilePath) dockerBuild := bringauto_docker.DockerBuild{ DockerfileDir: dockerfileDir, - Tag: name, + Tag: imageName, } buildOk := dockerBuild.Build() if buildOk != nil { - return fmt.Errorf("cannot build Docker image - %s", name) + return fmt.Errorf("Build failed for %s image", imageName) } return nil } diff --git a/modules/bringauto_build/Build.go b/modules/bringauto_build/Build.go index fd3b8d8..234eefa 100644 --- a/modules/bringauto_build/Build.go +++ b/modules/bringauto_build/Build.go @@ -110,7 +110,7 @@ func (build *Build) RunBuild() error { } logger := bringauto_log.GetLogger() - packBuildChainLogger := logger.CreatePackageContextLogger(build.Package.GetShortPackageName(), bringauto_log.BuildChainContext) + packBuildChainLogger := logger.CreateContextLogger(build.Docker.ImageName, build.Package.GetShortPackageName(), bringauto_log.BuildChainContext) file, err := packBuildChainLogger.GetFile() if err != nil { @@ -201,7 +201,7 @@ func (build *Build) downloadInstalledFiles() error { } } - packTarLogger := bringauto_log.GetLogger().CreatePackageContextLogger(build.Package.GetShortPackageName(), bringauto_log.TarContext) + packTarLogger := bringauto_log.GetLogger().CreateContextLogger(build.Docker.ImageName, build.Package.GetShortPackageName(), bringauto_log.TarContext) logFile, err := packTarLogger.GetFile() if err != nil { diff --git a/modules/bringauto_docker/DockerBuild.go b/modules/bringauto_docker/DockerBuild.go index e812ec1..4be49ff 100644 --- a/modules/bringauto_docker/DockerBuild.go +++ b/modules/bringauto_docker/DockerBuild.go @@ -2,7 +2,6 @@ package bringauto_docker import ( "bringauto/modules/bringauto_log" - "bytes" "fmt" "os/exec" ) @@ -24,30 +23,33 @@ func (dockerBuild *DockerBuild) Build() error { logger := bringauto_log.GetLogger() logger.Info("Build Docker Image: %s", dockerBuild.Tag) - var ok, _, err = dockerBuild.prepareAndRun(prepareBuildArgs) + var ok = dockerBuild.prepareAndRun(prepareBuildArgs) if !ok { - return fmt.Errorf("DockerBuild build error - %s", err) + return fmt.Errorf("DockerBuild build error") } return nil } -func (dockerBuild *DockerBuild) prepareAndRun(f func(build *DockerBuild) []string) (bool, *bytes.Buffer, *bytes.Buffer) { +func (dockerBuild *DockerBuild) prepareAndRun(f func(build *DockerBuild) []string) bool { + logger := bringauto_log.GetLogger() + contextLogger := logger.CreateContextLogger(dockerBuild.Tag, "", bringauto_log.ImageBuildContext) + file, _ := contextLogger.GetFile() + var cmd exec.Cmd - var outBuffer, errBuffer bytes.Buffer cmdArgs := f(dockerBuild) cmdArgs = append([]string{DockerExecutablePathConst}, cmdArgs...) cmd.Args = cmdArgs cmd.Path = DockerExecutablePathConst - cmd.Stderr = &errBuffer - cmd.Stdout = &outBuffer + cmd.Stderr = file + cmd.Stdout = file err := cmd.Run() if err != nil { - return false, &outBuffer, &errBuffer + return false } if cmd.ProcessState.ExitCode() > 0 { - return false, &outBuffer, &errBuffer + return false } - return true, &outBuffer, &errBuffer + return true } func prepareBuildArgs(dockerBuild *DockerBuild) []string { diff --git a/modules/bringauto_log/PackageContextLogger.go b/modules/bringauto_log/ContextLogger.go similarity index 60% rename from modules/bringauto_log/PackageContextLogger.go rename to modules/bringauto_log/ContextLogger.go index 4f4a410..94345ca 100644 --- a/modules/bringauto_log/PackageContextLogger.go +++ b/modules/bringauto_log/ContextLogger.go @@ -9,21 +9,23 @@ import ( const ( BuildChainContext = "build_chain" TarContext = "tar" + ImageBuildContext = "image_build" ) -type PackageContextLogger struct { +type ContextLogger struct { Logger logFileName string } -type packageContextLoggerInitArgs struct { +type contextLoggerInitArgs struct { Timestamp time.Time LogDirPath string - PackageName string + ImageName string + PackageName string // If is empty, then it is considered as non-package log LogContext string } -func (logger *PackageContextLogger) FillDefault(*bringauto_prerequisites.Args) error { +func (logger *ContextLogger) FillDefault(*bringauto_prerequisites.Args) error { logger.slogger = getDefaultLogger(os.Stdout) logger.timestamp = time.Time{} logger.logDirPath = "" @@ -31,18 +33,18 @@ func (logger *PackageContextLogger) FillDefault(*bringauto_prerequisites.Args) e return nil } -func (logger *PackageContextLogger) FillDynamic(args *bringauto_prerequisites.Args) error { +func (logger *ContextLogger) FillDynamic(args *bringauto_prerequisites.Args) error { if !bringauto_prerequisites.IsEmpty(args) { - var argsStruct packageContextLoggerInitArgs + var argsStruct contextLoggerInitArgs bringauto_prerequisites.GetArgs(args, &argsStruct) logger.timestamp = argsStruct.Timestamp - logger.logDirPath = argsStruct.LogDirPath + "/" + argsStruct.PackageName + logger.logDirPath = argsStruct.LogDirPath + "/" + argsStruct.ImageName + "/" + argsStruct.PackageName logger.logFileName = argsStruct.LogContext + ".txt" } return nil } -func (logger *PackageContextLogger) initLogDir() error { +func (logger *ContextLogger) initLogDir() error { _, err := os.Stat(logger.logDirPath) if os.IsNotExist(err) { err = os.MkdirAll(logger.logDirPath, 0700) @@ -54,13 +56,13 @@ func (logger *PackageContextLogger) initLogDir() error { return nil } -func (logger *PackageContextLogger) CheckPrerequisites(*bringauto_prerequisites.Args) error { +func (logger *ContextLogger) CheckPrerequisites(*bringauto_prerequisites.Args) error { if logger.logDirPath != "" { logger.initLogDir() } return nil } -func (logger *PackageContextLogger) GetFile() (*os.File, error) { +func (logger *ContextLogger) GetFile() (*os.File, error) { return os.OpenFile(logger.logDirPath + "/" + logger.logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) } diff --git a/modules/bringauto_log/GlobalLogger.go b/modules/bringauto_log/GlobalLogger.go index f4a064f..8b31c29 100644 --- a/modules/bringauto_log/GlobalLogger.go +++ b/modules/bringauto_log/GlobalLogger.go @@ -52,9 +52,9 @@ func (logger *GlobalLogger) CheckPrerequisites(*bringauto_prerequisites.Args) er return nil } -func (logger *GlobalLogger) CreatePackageContextLogger(packageName string, logContext string) *PackageContextLogger { - packageContextLogger := bringauto_prerequisites.CreateAndInitialize[PackageContextLogger]( - logger.timestamp, logger.logDirPath, packageName, logContext, +func (logger *GlobalLogger) CreateContextLogger(imageName string, packageName string, logContext string) *ContextLogger { + packageContextLogger := bringauto_prerequisites.CreateAndInitialize[ContextLogger]( + logger.timestamp, logger.logDirPath, imageName, packageName, logContext, ) return packageContextLogger } From abcbe53e7dd86e9e9ba4f1f663e08a173ddd8238 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 19 Aug 2024 14:51:46 +0200 Subject: [PATCH 25/91] Add method doc --- modules/bringauto_config/Config.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/bringauto_config/Config.go b/modules/bringauto_config/Config.go index 9976b6b..39ab6d4 100644 --- a/modules/bringauto_config/Config.go +++ b/modules/bringauto_config/Config.go @@ -81,6 +81,8 @@ func (config *Config) SaveToJSONConfig(configPath string) error { return nil } +// Returns array of builds structs for specific image name. The returned array will contain max one build. +// It is an array for simple handling of result using for loop. func (config *Config) GetBuildStructure(imageName string) []bringauto_build.Build { var buildConfigs []bringauto_build.Build for _, value := range config.DockerMatrix.ImageNames { From 12c2485252cbc0d8a388320cec69ce57527fd122 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 19 Aug 2024 14:53:59 +0200 Subject: [PATCH 26/91] Add state logs and log for succesfull builds --- bap-builder/DockerMode.go | 2 ++ bap-builder/PackageMode.go | 7 ++++++- modules/bringauto_build/Build.go | 2 ++ modules/bringauto_docker/DockerBuild.go | 2 -- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/bap-builder/DockerMode.go b/bap-builder/DockerMode.go index 5179a3c..7266e7e 100644 --- a/bap-builder/DockerMode.go +++ b/bap-builder/DockerMode.go @@ -43,10 +43,12 @@ func buildAllDockerImages(contextPath string) error { DockerfileDir: dockerfileDir, Tag: imageName, } + logger.Info("Build Docker Image: %s", imageName) err = dockerBuild.Build() if err != nil { return fmt.Errorf("Build failed for %s image", imageName) } + logger.Info("Build OK") } return nil } diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index c558a4a..1197929 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -214,7 +214,6 @@ func buildSinglePackage(cmdLine *BuildPackageCmdLineArgs, contextPath string) er for _, config := range configList { buildConfigs := config.GetBuildStructure(*cmdLine.DockerImageName) - logger.Info("Build %s", buildConfigs[0].Package.GetFullPackageName()) err = buildAndCopyPackage(cmdLine, &buildConfigs) if err != nil { logger.Error("cannot build package '%s' - %s\n", packageName, err) @@ -256,6 +255,7 @@ func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_bu if err != nil { return err } + logger := bringauto_log.GetLogger() for _, buildConfig := range *build { platformString, err := determinePlatformString(&buildConfig) @@ -263,6 +263,8 @@ func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_bu return err } + logger.Info("Build %s", buildConfig.Package.GetFullPackageName()) + sysroot := bringauto_sysroot.Sysroot{ IsDebug: buildConfig.Package.IsDebug, PlatformString: platformString, @@ -275,11 +277,13 @@ func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_bu return err } + logger.Info("Copy to Git repository") err = repo.CopyToRepository(*buildConfig.Package, buildConfig.GetLocalInstallDirPath()) if err != nil { return err } + logger.Info("Copy to local sysroot directory") err = sysroot.CopyToSysroot(buildConfig.GetLocalInstallDirPath()) if err != nil { return err @@ -289,6 +293,7 @@ func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_bu if err != nil { return err } + logger.Info("Build OK") } return nil } diff --git a/modules/bringauto_build/Build.go b/modules/bringauto_build/Build.go index 234eefa..19da686 100644 --- a/modules/bringauto_build/Build.go +++ b/modules/bringauto_build/Build.go @@ -159,6 +159,8 @@ func (build *Build) RunBuild() error { return err } + logger.Info("Copying install files from container to local directory") + err = build.downloadInstalledFiles() return err } diff --git a/modules/bringauto_docker/DockerBuild.go b/modules/bringauto_docker/DockerBuild.go index 4be49ff..b61c1c3 100644 --- a/modules/bringauto_docker/DockerBuild.go +++ b/modules/bringauto_docker/DockerBuild.go @@ -20,8 +20,6 @@ func (dockerBuild *DockerBuild) Build() error { if dockerBuild.DockerfileDir == "" { return fmt.Errorf("DockerBuild - DockerfileDir is empty") } - logger := bringauto_log.GetLogger() - logger.Info("Build Docker Image: %s", dockerBuild.Tag) var ok = dockerBuild.prepareAndRun(prepareBuildArgs) if !ok { From c70970037eb7525fc9ccbe1359756dccbc71a08a Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 19 Aug 2024 15:03:39 +0200 Subject: [PATCH 27/91] Add indent logging --- bap-builder/DockerMode.go | 2 +- bap-builder/PackageMode.go | 8 +++++--- modules/bringauto_build/Build.go | 2 +- modules/bringauto_log/Logger.go | 12 ++++++++++++ 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/bap-builder/DockerMode.go b/bap-builder/DockerMode.go index 7266e7e..a27c700 100644 --- a/bap-builder/DockerMode.go +++ b/bap-builder/DockerMode.go @@ -48,7 +48,7 @@ func buildAllDockerImages(contextPath string) error { if err != nil { return fmt.Errorf("Build failed for %s image", imageName) } - logger.Info("Build OK") + logger.InfoIndent("Build OK") } return nil } diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index 1197929..eb336f5 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -272,18 +272,20 @@ func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_bu err = bringauto_prerequisites.Initialize(&sysroot) buildConfig.SetSysroot(&sysroot) + + logger.InfoIndent("Run build inside container") err = buildConfig.RunBuild() if err != nil { return err } - logger.Info("Copy to Git repository") + logger.InfoIndent("Copy to Git repository") err = repo.CopyToRepository(*buildConfig.Package, buildConfig.GetLocalInstallDirPath()) if err != nil { return err } - logger.Info("Copy to local sysroot directory") + logger.InfoIndent("Copy to local sysroot directory") err = sysroot.CopyToSysroot(buildConfig.GetLocalInstallDirPath()) if err != nil { return err @@ -293,7 +295,7 @@ func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_bu if err != nil { return err } - logger.Info("Build OK") + logger.InfoIndent("Build OK") } return nil } diff --git a/modules/bringauto_build/Build.go b/modules/bringauto_build/Build.go index 19da686..cfb0b27 100644 --- a/modules/bringauto_build/Build.go +++ b/modules/bringauto_build/Build.go @@ -159,7 +159,7 @@ func (build *Build) RunBuild() error { return err } - logger.Info("Copying install files from container to local directory") + logger.InfoIndent("Copying install files from container to local directory") err = build.downloadInstalledFiles() return err diff --git a/modules/bringauto_log/Logger.go b/modules/bringauto_log/Logger.go index 7bdf91e..2b2967a 100644 --- a/modules/bringauto_log/Logger.go +++ b/modules/bringauto_log/Logger.go @@ -8,6 +8,10 @@ import ( "fmt" ) +const ( + indent = " " +) + type Logger struct { slogger *slog.Logger timestamp time.Time @@ -26,6 +30,14 @@ func (logger *Logger) Info(msg string, args ...any) { } } +func (logger *Logger) InfoIndent(msg string, args ...any) { + if len(args) == 0 { + logger.slogger.Info(indent + msg) + } else { + logger.slogger.Info(indent + fmt.Sprintf(msg, args)) + } +} + func (logger *Logger) Warn(msg string, args ...any) { if len(args) == 0 { logger.slogger.Warn(msg) From e19ed27e188ccf1ee2cfae01960e0b61409613d1 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Wed, 21 Aug 2024 11:26:30 +0200 Subject: [PATCH 28/91] Remove CopyRecursive - not used --- modules/bringauto_ssh/SFTP.go | 87 ++++------------------------------- 1 file changed, 10 insertions(+), 77 deletions(-) diff --git a/modules/bringauto_ssh/SFTP.go b/modules/bringauto_ssh/SFTP.go index 1166e1a..dcc3820 100644 --- a/modules/bringauto_ssh/SFTP.go +++ b/modules/bringauto_ssh/SFTP.go @@ -3,16 +3,16 @@ package bringauto_ssh import ( "bufio" "fmt" - "github.com/pkg/sftp" - "github.com/mholt/archiver/v3" "io" "os" - "path" "regexp" + + "github.com/mholt/archiver/v3" + "github.com/pkg/sftp" ) const ( - archiveName string = "install_arch.tar" + archiveName string = "install_arch.tar" archiveNameSep string = string(os.PathSeparator) + archiveName ) @@ -22,7 +22,7 @@ type SFTP struct { // Empty, existing local directory where the RemoteDir will be copy EmptyLocalDir string SSHCredentials *SSHCredentials - LogWriter io.Writer + LogWriter io.Writer } // DownloadDirectory @@ -34,7 +34,7 @@ func (sftpd *SFTP) DownloadDirectory() error { tar := Tar{ ArchiveName: archiveName, - SourceDir: "/INSTALL", + SourceDir: "/INSTALL", } shellEvaluator := ShellEvaluator{ @@ -77,16 +77,16 @@ func (sftpd *SFTP) DownloadDirectory() error { localArchivePath := sftpd.EmptyLocalDir + archiveNameSep - err = sftpd.copyFile(sftpClient, sftpd.RemoteDir + archiveNameSep, localArchivePath) + err = sftpd.copyFile(sftpClient, sftpd.RemoteDir+archiveNameSep, localArchivePath) if err != nil { return fmt.Errorf("cannot copy recursive %s", err) } tarArchive := archiver.Tar{ - OverwriteExisting: false, - MkdirAll: false, + OverwriteExisting: false, + MkdirAll: false, ImplicitTopLevelFolder: false, - ContinueOnError: true, + ContinueOnError: true, } err = tarArchive.Unarchive(localArchivePath, sftpd.EmptyLocalDir) @@ -128,73 +128,6 @@ func (sftpd *SFTP) copyFile(sftpClient *sftp.Client, remoteFile string, localDir return nil } -// deprecated, reason: very slow with big trees, new approach: create tar and and copy single archive with copyFile() -func (sftpd *SFTP) copyRecursive(sftpClient *sftp.Client, remoteDir string, localDir string) error { - var err error - _, err = sftpClient.Lstat(sftpd.RemoteDir) - if os.IsNotExist(err) { - return fmt.Errorf("requested remote file %s does not exist", sftpd.RemoteDir) - } - normalizedRemoteDir, _ := normalizePath(remoteDir) - normalizedLocalDir, _ := normalizePath(localDir) - - allDone := make(chan bool, 2000) - fileCount := 0 - - walk := sftpClient.Walk(normalizedRemoteDir) - for walk.Step() { - if walk.Err() != nil { - continue - } - remotePath, _ := normalizePath(walk.Path()) - if normalizedRemoteDir == remotePath { - continue - } - relativeRemotePath := remotePath[len(normalizedRemoteDir):] - absoluteLocalPath := path.Join(normalizedLocalDir, relativeRemotePath) - remotePathStat, err := sftpClient.Lstat(remotePath) - if err != nil { - return fmt.Errorf("cannot get Lstat if remote %s", normalizedRemoteDir) - } - - if remotePathStat.IsDir() { - err = os.MkdirAll(absoluteLocalPath, remotePathStat.Mode().Perm()) - if err != nil { - return fmt.Errorf("cannot create local directory - %s", err) - } - err = sftpd.copyRecursive(sftpClient, remotePath, absoluteLocalPath) - if err != nil { - return fmt.Errorf("sftp copy - %s", err) - } - continue - } - - sourceFile, err := sftpClient.Open(remotePath) - if err != nil { - return fmt.Errorf("cannot open file for read - %s,%s", remotePath, err) - } - destFile, err := os.OpenFile(absoluteLocalPath, os.O_RDWR|os.O_CREATE, remotePathStat.Mode().Perm()) - if err != nil { - return err - } - - fileCount += 1 - go func() { - defer func() { allDone <- true }() - - copyIOFile(sourceFile, destFile) - }() - - } - //Problem: this system copy files directory by directory in linear manner - // just stupid wait mechanism - for i := 0; i < fileCount; i++ { - <-allDone - } - - return nil -} - func copyIOFile(sourceFile *sftp.File, destFile *os.File) { sourceFileBuff := bufio.NewReaderSize(sourceFile, 1024*1024) destFileBuff := bufio.NewWriterSize(destFile, 1027*1024) From da63695c244c30ae22651df28d9e5cf74021f894 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Wed, 21 Aug 2024 11:43:41 +0200 Subject: [PATCH 29/91] Change directory name of logs --- modules/bringauto_log/GlobalLogger.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/bringauto_log/GlobalLogger.go b/modules/bringauto_log/GlobalLogger.go index 8b31c29..e6ca5df 100644 --- a/modules/bringauto_log/GlobalLogger.go +++ b/modules/bringauto_log/GlobalLogger.go @@ -4,7 +4,6 @@ import ( "bringauto/modules/bringauto_prerequisites" "os" "time" - "strconv" ) var globalLoggerSingleton *GlobalLogger @@ -22,7 +21,7 @@ type GlobalLogger struct { } type globalLoggerInitArgs struct { - Timestamp time.Time + Timestamp time.Time LogDirPath string } @@ -44,7 +43,7 @@ func (logger *GlobalLogger) FillDynamic(args *bringauto_prerequisites.Args) erro } func (logger *GlobalLogger) getTimestampString() string { - return strconv.FormatInt(logger.timestamp.Unix(), 10) + return logger.timestamp.Format("2006-01-02_15:04:05") } func (logger *GlobalLogger) CheckPrerequisites(*bringauto_prerequisites.Args) error { From 452e943f6c2cd3b6e043a825805527d65bdbfd28 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Wed, 21 Aug 2024 11:47:56 +0200 Subject: [PATCH 30/91] Remove unused timestamp from ContextLogger --- modules/bringauto_log/ContextLogger.go | 6 +----- modules/bringauto_log/GlobalLogger.go | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/modules/bringauto_log/ContextLogger.go b/modules/bringauto_log/ContextLogger.go index 94345ca..318ab97 100644 --- a/modules/bringauto_log/ContextLogger.go +++ b/modules/bringauto_log/ContextLogger.go @@ -3,7 +3,6 @@ package bringauto_log import ( "bringauto/modules/bringauto_prerequisites" "os" - "time" ) const ( @@ -18,7 +17,6 @@ type ContextLogger struct { } type contextLoggerInitArgs struct { - Timestamp time.Time LogDirPath string ImageName string PackageName string // If is empty, then it is considered as non-package log @@ -27,7 +25,6 @@ type contextLoggerInitArgs struct { func (logger *ContextLogger) FillDefault(*bringauto_prerequisites.Args) error { logger.slogger = getDefaultLogger(os.Stdout) - logger.timestamp = time.Time{} logger.logDirPath = "" logger.logFileName = "" return nil @@ -37,8 +34,7 @@ func (logger *ContextLogger) FillDynamic(args *bringauto_prerequisites.Args) err if !bringauto_prerequisites.IsEmpty(args) { var argsStruct contextLoggerInitArgs bringauto_prerequisites.GetArgs(args, &argsStruct) - logger.timestamp = argsStruct.Timestamp - logger.logDirPath = argsStruct.LogDirPath + "/" + argsStruct.ImageName + "/" + argsStruct.PackageName + logger.logDirPath = argsStruct.LogDirPath + "/" + argsStruct.ImageName + "/" + argsStruct.PackageName logger.logFileName = argsStruct.LogContext + ".txt" } return nil diff --git a/modules/bringauto_log/GlobalLogger.go b/modules/bringauto_log/GlobalLogger.go index e6ca5df..c0e5388 100644 --- a/modules/bringauto_log/GlobalLogger.go +++ b/modules/bringauto_log/GlobalLogger.go @@ -53,7 +53,7 @@ func (logger *GlobalLogger) CheckPrerequisites(*bringauto_prerequisites.Args) er func (logger *GlobalLogger) CreateContextLogger(imageName string, packageName string, logContext string) *ContextLogger { packageContextLogger := bringauto_prerequisites.CreateAndInitialize[ContextLogger]( - logger.timestamp, logger.logDirPath, imageName, packageName, logContext, + logger.logDirPath, imageName, packageName, logContext, ) return packageContextLogger } From e71675d1d8b5ecb1d8442e0734e2247874df40c4 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Wed, 21 Aug 2024 14:04:49 +0200 Subject: [PATCH 31/91] Check only for platform specific sysroot dir --- bap-builder/PackageMode.go | 7 +++--- modules/bringauto_log/Logger.go | 8 +++++++ modules/bringauto_sysroot/Sysroot.go | 22 ++++++++++++++++++ modules/bringauto_sysroot/SysrootTools.go | 27 ----------------------- 4 files changed, 33 insertions(+), 31 deletions(-) delete mode 100644 modules/bringauto_sysroot/SysrootTools.go diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index eb336f5..e47bff9 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -122,10 +122,6 @@ func (list *buildDepList) sortDependencies(rootName string, dependsMap *map[stri // BuildPackage // process Package mode of the program func BuildPackage(cmdLine *BuildPackageCmdLineArgs, contextPath string) error { - if !bringauto_sysroot.IsSysrootDirectoryEmpty() { - logger := bringauto_log.GetLogger() - logger.Warn("Sysroot directory is not empty - the package build may fail") - } buildAll := cmdLine.All if *buildAll { return buildAllPackages(cmdLine, contextPath) @@ -270,6 +266,9 @@ func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_bu PlatformString: platformString, } err = bringauto_prerequisites.Initialize(&sysroot) + if !sysroot.IsSysrootDirectoryEmpty() { + logger.WarnIndent("Sysroot directory is not empty - the package build may fail") + } buildConfig.SetSysroot(&sysroot) diff --git a/modules/bringauto_log/Logger.go b/modules/bringauto_log/Logger.go index 2b2967a..3fe4e0e 100644 --- a/modules/bringauto_log/Logger.go +++ b/modules/bringauto_log/Logger.go @@ -46,6 +46,14 @@ func (logger *Logger) Warn(msg string, args ...any) { } } +func (logger *Logger) WarnIndent(msg string, args ...any) { + if len(args) == 0 { + logger.slogger.Warn(indent + msg) + } else { + logger.slogger.Warn(indent + fmt.Sprintf(msg, args)) + } +} + func (logger *Logger) Error(msg string, args ...any) { if len(args) == 0 { logger.slogger.Error(msg) diff --git a/modules/bringauto_sysroot/Sysroot.go b/modules/bringauto_sysroot/Sysroot.go index 8ef3752..648a13c 100644 --- a/modules/bringauto_sysroot/Sysroot.go +++ b/modules/bringauto_sysroot/Sysroot.go @@ -1,11 +1,13 @@ package bringauto_sysroot import ( + "bringauto/modules/bringauto_log" "bringauto/modules/bringauto_package" "bringauto/modules/bringauto_prerequisites" "fmt" "github.com/otiai10/copy" "os" + "io" "io/fs" "path/filepath" "strings" @@ -105,6 +107,26 @@ func (sysroot *Sysroot) CreateSysrootDir() { } } +// IsSysrootDirectoryEmpty +// Returns true if specified dir do not exists or exists but is empty, otherwise returns false +func (sysroot *Sysroot) IsSysrootDirectoryEmpty() bool { + f, err := os.Open(sysroot.GetSysrootPath()) + if err != nil { // The directory do not exists + return true + } + defer f.Close() + + _, err = f.Readdirnames(1) + + if err == io.EOF { // The directory exists, but is empty + return true + } else if err != nil { + bringauto_log.GetLogger().Warn("Cannot read in sysroot directory: %s", err) + } + + return false +} + func onSymlink(src string) copy.SymlinkAction { return copy.Shallow } diff --git a/modules/bringauto_sysroot/SysrootTools.go b/modules/bringauto_sysroot/SysrootTools.go deleted file mode 100644 index a9aa267..0000000 --- a/modules/bringauto_sysroot/SysrootTools.go +++ /dev/null @@ -1,27 +0,0 @@ -package bringauto_sysroot - -import ( - "bringauto/modules/bringauto_log" - "os" - "io" -) - -// IsSysrootDirectoryEmpty -// Returns true if specified dir do not exists or exists but is empty, otherwise returns false -func IsSysrootDirectoryEmpty() bool { - f, err := os.Open(sysrootDirectoryName) - if err != nil { // The directory do not exists - return true - } - defer f.Close() - - _, err = f.Readdirnames(1) - - if err == io.EOF { // The directory exists, but is empty - return true - } else if err != nil { - bringauto_log.GetLogger().Warn("Cannot read in sysroot directory: %s", err) - } - - return false -} From f8e057e7fda8dd6caa3a5ae5f3b0902cf191ca78 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Wed, 21 Aug 2024 14:36:27 +0200 Subject: [PATCH 32/91] Script exits after failed build when building Single package now --- bap-builder/PackageMode.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index e47bff9..d38cb2e 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -212,8 +212,7 @@ func buildSinglePackage(cmdLine *BuildPackageCmdLineArgs, contextPath string) er buildConfigs := config.GetBuildStructure(*cmdLine.DockerImageName) err = buildAndCopyPackage(cmdLine, &buildConfigs) if err != nil { - logger.Error("cannot build package '%s' - %s\n", packageName, err) - continue + logger.Fatal("cannot build package '%s' - %s", packageName, err) } } return nil From 965140f4ae4f07bee917691959cbddaa4ad3737e Mon Sep 17 00:00:00 2001 From: Mikulas Date: Wed, 21 Aug 2024 15:11:38 +0200 Subject: [PATCH 33/91] Better error handling --- bap-builder/ContextManager.go | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/bap-builder/ContextManager.go b/bap-builder/ContextManager.go index cfa0eea..f607d20 100644 --- a/bap-builder/ContextManager.go +++ b/bap-builder/ContextManager.go @@ -2,7 +2,6 @@ package main import ( "bringauto/modules/bringauto_config" - "bringauto/modules/bringauto_log" "fmt" "io/fs" "os" @@ -89,36 +88,36 @@ func (context *ContextManager) GetPackageJsonDefPaths(packageName string) ([]str return packageDefs, nil } -func (context *ContextManager) getAllDepsJsonPaths(packageJsonPath string) []string { +func (context *ContextManager) getAllDepsJsonPaths(packageJsonPath string) ([]string, error) { var config bringauto_config.Config - logger := bringauto_log.GetLogger() err := config.LoadJSONConfig(packageJsonPath) if err != nil { - logger.Warn("Couldn't load JSON config from %s path - %s", packageJsonPath, err) - return []string{} + return []string{}, fmt.Errorf("couldn't load JSON config from %s path - %s", packageJsonPath, err) } var jsonPathListWithDeps []string for _, packageDep := range config.DependsOn { packageDepsJsonPaths, err := context.GetPackageJsonDefPaths(packageDep) if err != nil { - logger.Warn("Couldn't get Json Path of %s package", packageDep) - continue + return []string{}, fmt.Errorf("couldn't get Json Path of %s package", packageDep) } var depConfig bringauto_config.Config for _, packageDepJsonPath := range packageDepsJsonPaths { err := depConfig.LoadJSONConfig(packageDepJsonPath) if err != nil { - logger.Warn("Couldn't load JSON config from %s path - %s", packageJsonPath, err) - continue + return []string{}, fmt.Errorf("couldn't load JSON config from %s path - %s", packageJsonPath, err) } if depConfig.Package.IsDebug == config.Package.IsDebug { jsonPathListWithDeps = append(jsonPathListWithDeps, packageDepJsonPath) - jsonPathListWithDeps = append(jsonPathListWithDeps, context.getAllDepsJsonPaths(packageDepJsonPath)...) + jsonPathListWithDepsTmp, err := context.getAllDepsJsonPaths(packageDepJsonPath) + if err != nil { + return []string{}, err + } + jsonPathListWithDeps = append(jsonPathListWithDeps, jsonPathListWithDepsTmp...) } } } - return jsonPathListWithDeps + return jsonPathListWithDeps, nil } // GetPackageWithDepsJsonDefPaths @@ -126,11 +125,15 @@ func (context *ContextManager) getAllDepsJsonPaths(packageJsonPath string) []str func (context *ContextManager) GetPackageWithDepsJsonDefPaths(packageName string) ([]string, error) { packageDefs, err := context.GetPackageJsonDefPaths(packageName) if err != nil { - return []string{}, fmt.Errorf("cannot get config paths for package '%s'", packageName) + return []string{}, fmt.Errorf("cannot get config paths for package '%s' - %s", packageName, err) } var packageDeps []string for _, packageDef := range packageDefs { - packageDeps = append(packageDeps, context.getAllDepsJsonPaths(packageDef)...) + packageDepsTmp, err := context.getAllDepsJsonPaths(packageDef) + if err != nil { + return []string{}, err + } + packageDeps = append(packageDeps, packageDepsTmp...) } packageDefs = append(packageDefs, packageDeps...) From ef89ca5bf01e8d397e2882479f8a995d7da0c602 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Wed, 11 Sep 2024 10:59:03 +0300 Subject: [PATCH 34/91] Add sigint handling --- bap-builder/PackageMode.go | 5 ++ bap-builder/main.go | 3 + modules/bringauto_build/Build.go | 35 +++++++---- modules/bringauto_process/SignalHandler.go | 70 ++++++++++++++++++++++ 4 files changed, 100 insertions(+), 13 deletions(-) create mode 100644 modules/bringauto_process/SignalHandler.go diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index d38cb2e..c71717e 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -8,6 +8,7 @@ import ( "bringauto/modules/bringauto_prerequisites" "bringauto/modules/bringauto_repository" "bringauto/modules/bringauto_sysroot" + "bringauto/modules/bringauto_process" "fmt" "strconv" ) @@ -278,6 +279,10 @@ func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_bu } logger.InfoIndent("Copy to Git repository") + + removeHandler := bringauto_process.AddHandler(buildConfig.CleanUp) + defer removeHandler() + err = repo.CopyToRepository(*buildConfig.Package, buildConfig.GetLocalInstallDirPath()) if err != nil { return err diff --git a/bap-builder/main.go b/bap-builder/main.go index a8627e0..8dba29c 100644 --- a/bap-builder/main.go +++ b/bap-builder/main.go @@ -3,8 +3,10 @@ package main import ( "bringauto/modules/bringauto_log" "bringauto/modules/bringauto_prerequisites" + "bringauto/modules/bringauto_process" "os" "time" + "syscall" ) func main() { @@ -17,6 +19,7 @@ func main() { if err != nil { return } + bringauto_process.RegisterSignal(syscall.SIGINT) if args.BuildImage { err = BuildDockerImage(&args.BuildImagesArgs, *args.Context) diff --git a/modules/bringauto_build/Build.go b/modules/bringauto_build/Build.go index cfb0b27..e9d7459 100644 --- a/modules/bringauto_build/Build.go +++ b/modules/bringauto_build/Build.go @@ -8,6 +8,7 @@ import ( "bringauto/modules/bringauto_prerequisites" "bringauto/modules/bringauto_ssh" "bringauto/modules/bringauto_sysroot" + "bringauto/modules/bringauto_process" "fmt" "os" "path/filepath" @@ -136,23 +137,14 @@ func (build *Build) RunBuild() error { } dockerRun := (*bringauto_docker.DockerRun)(build.Docker) + removeHandler := bringauto_process.AddHandler(build.stopAndRemoveContainer) + defer removeHandler() + err = dockerRun.Run() if err != nil { return err } - defer func() { - dockerStop := (*bringauto_docker.DockerStop)(build.Docker) - dockerRm := (*bringauto_docker.DockerRm)(build.Docker) - var err error - err = dockerStop.Stop() - if err != nil { - logger.Error("cannot stop container: %s\n", err) - } - err = dockerRm.RemoveContainer() - if err != nil { - logger.Error("cannot remove container: %s\n", err) - } - }() + defer build.stopAndRemoveContainer() err = shellEvaluator.RunOverSSH(*build.SSHCredentials) if err != nil { @@ -179,6 +171,23 @@ func (build *Build) GetLocalInstallDirPath() string { return copyBaseDir } +func (build *Build) stopAndRemoveContainer() error { + var err error + + dockerStop := (*bringauto_docker.DockerStop)(build.Docker) + dockerRm := (*bringauto_docker.DockerRm)(build.Docker) + logger := bringauto_log.GetLogger() + err = dockerStop.Stop() + if err != nil { + logger.Error("Can't stop container - %s", err) + } + err = dockerRm.RemoveContainer() + if err != nil { + logger.Error("Can't remove container - %s", err) + } + return nil +} + func (build *Build) CleanUp() error { var err error copyDir := build.GetLocalInstallDirPath() diff --git a/modules/bringauto_process/SignalHandler.go b/modules/bringauto_process/SignalHandler.go new file mode 100644 index 0000000..75b5650 --- /dev/null +++ b/modules/bringauto_process/SignalHandler.go @@ -0,0 +1,70 @@ +// Functions for global signal handling +// +// The Register() must be called to start handling signal by this module. Then AddHandler() can be +// called to add handlers to execute when the registered signal is received. The method returns +// function which should be deferred after AddHandler() call by caller. This returned function +// basically removes handler from execution after signal is received. When registered signal is +// received, all added handlers (and not removed yet) will be executed in reverse order and then +// the program exits with status code 1. +// +// Note: Do not use with concurrent programming. Can behave unexpectedly! + +package bringauto_process + +import ( + "bringauto/modules/bringauto_log" + "fmt" + "os" + "sync" + "os/signal" +) + + +var lock sync.Mutex +var handlers []func() error + +// Registers handling of specified signals to bringauto_process package +func RegisterSignal(sig ...os.Signal) { + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, sig...) + go func() { + _ = <-sigs + lock.Lock() + defer lock.Unlock() + logger := bringauto_log.GetLogger() + logger.Info("SIGINT received - %d handlers to execute", len(handlers)) + executeAllHandlers() + os.Exit(1) + }() +} + +// Adds handler for execution after signal is received by bringauto_process package. Returns handler +// remover which should be deferred by caller. +func AddHandler(handler func() error) func() { + lock.Lock() + defer lock.Unlock() + handlers = append(handlers, handler) + return func() { + lock.Lock() + defer lock.Unlock() + removeLastHandler() + } +} + +func removeLastHandler() error { + bringauto_log.GetLogger().Info("Removing last handler, new size: %d", len(handlers) - 1) + if len(handlers) == 0 { + return fmt.Errorf("no handler to remove") + } + handlers = handlers[:len(handlers) - 1] + return nil +} + +func executeAllHandlers() { + for i := len(handlers)-1; i >= 0; i-- { + err := handlers[i]() + if err != nil { + bringauto_log.GetLogger().Error("Handler returned error - %s", err) + } + } +} From 2c924f3ee793f4cdae466786a9f9aff25edfd253 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Tue, 17 Sep 2024 12:04:13 +0300 Subject: [PATCH 35/91] Handler remover function called without defer --- bap-builder/PackageMode.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index c71717e..7f22662 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -281,7 +281,6 @@ func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_bu logger.InfoIndent("Copy to Git repository") removeHandler := bringauto_process.AddHandler(buildConfig.CleanUp) - defer removeHandler() err = repo.CopyToRepository(*buildConfig.Package, buildConfig.GetLocalInstallDirPath()) if err != nil { @@ -295,6 +294,7 @@ func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_bu } err = buildConfig.CleanUp() + removeHandler() if err != nil { return err } From c4d54cefae9f6381843810c95304788c10b6f90a Mon Sep 17 00:00:00 2001 From: Mikulas Date: Tue, 17 Sep 2024 12:14:34 +0300 Subject: [PATCH 36/91] Unify log strings --- bap-builder/PackageMode.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index 7f22662..5de154e 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -278,7 +278,7 @@ func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_bu return err } - logger.InfoIndent("Copy to Git repository") + logger.InfoIndent("Copying to Git repository") removeHandler := bringauto_process.AddHandler(buildConfig.CleanUp) @@ -287,7 +287,7 @@ func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_bu return err } - logger.InfoIndent("Copy to local sysroot directory") + logger.InfoIndent("Copying to local sysroot directory") err = sysroot.CopyToSysroot(buildConfig.GetLocalInstallDirPath()) if err != nil { return err From f1574aba4e85136cb1c55b1a224e916e134faf8f Mon Sep 17 00:00:00 2001 From: Mikulas Date: Tue, 17 Sep 2024 12:27:01 +0300 Subject: [PATCH 37/91] Remove not needed logs --- modules/bringauto_process/SignalHandler.go | 1 - modules/bringauto_ssh/SSHSession.go | 1 - 2 files changed, 2 deletions(-) diff --git a/modules/bringauto_process/SignalHandler.go b/modules/bringauto_process/SignalHandler.go index 75b5650..4354fda 100644 --- a/modules/bringauto_process/SignalHandler.go +++ b/modules/bringauto_process/SignalHandler.go @@ -52,7 +52,6 @@ func AddHandler(handler func() error) func() { } func removeLastHandler() error { - bringauto_log.GetLogger().Info("Removing last handler, new size: %d", len(handlers) - 1) if len(handlers) == 0 { return fmt.Errorf("no handler to remove") } diff --git a/modules/bringauto_ssh/SSHSession.go b/modules/bringauto_ssh/SSHSession.go index 1b0ce76..0da422f 100644 --- a/modules/bringauto_ssh/SSHSession.go +++ b/modules/bringauto_ssh/SSHSession.go @@ -70,7 +70,6 @@ func (session *SSHSession) LoginMultipleAttempts(credentials SSHCredentials) err } else { numberOfAttempts += 1 time.Sleep(waitingInSecondsBeforeRetryConst * time.Second) - bringauto_log.GetLogger().Info("Cannot login over ssh. Again...") } } return nil From df083656b66e9ba34cd2d1871d9b81b08828c6a4 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Tue, 17 Sep 2024 13:01:49 +0300 Subject: [PATCH 38/91] Remove unused import --- modules/bringauto_ssh/SSHSession.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/bringauto_ssh/SSHSession.go b/modules/bringauto_ssh/SSHSession.go index 0da422f..e203405 100644 --- a/modules/bringauto_ssh/SSHSession.go +++ b/modules/bringauto_ssh/SSHSession.go @@ -1,7 +1,6 @@ package bringauto_ssh import ( - "bringauto/modules/bringauto_log" "bringauto/modules/bringauto_prerequisites" "fmt" "golang.org/x/crypto/ssh" From 53e0221b77ed003cac29701d1b1c672d350f06c1 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Tue, 17 Sep 2024 13:03:11 +0300 Subject: [PATCH 39/91] Add error state for build-deps and all flags used simultaneously --- bap-builder/CmdArgs.go | 3 +++ bap-builder/main.go | 1 + 2 files changed, 4 insertions(+) diff --git a/bap-builder/CmdArgs.go b/bap-builder/CmdArgs.go index 072beb2..fddcfc3 100644 --- a/bap-builder/CmdArgs.go +++ b/bap-builder/CmdArgs.go @@ -143,6 +143,9 @@ func (cmd *CmdLineArgs) ParseArgs(args []string) error { cmd.BuildImage = cmd.buildImageParser.Happened() cmd.BuildPackage = cmd.buildPackageParser.Happened() + if *cmd.BuildPackageArgs.All && *cmd.BuildPackageArgs.BuildDeps { + return fmt.Errorf("all and build-deps flags at the same time") + } return nil } diff --git a/bap-builder/main.go b/bap-builder/main.go index 8dba29c..4b064dc 100644 --- a/bap-builder/main.go +++ b/bap-builder/main.go @@ -17,6 +17,7 @@ func main() { args.InitFlags() err = args.ParseArgs(os.Args) if err != nil { + logger.Error("Can't parse cmd line arguments - %s", err) return } bringauto_process.RegisterSignal(syscall.SIGINT) From 8258e25a9fc1a2ecdecb3dd9eea9c131f79b8724 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Tue, 17 Sep 2024 13:04:19 +0300 Subject: [PATCH 40/91] Fix logging more errors and redundant brackets --- modules/bringauto_log/Logger.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/bringauto_log/Logger.go b/modules/bringauto_log/Logger.go index 3fe4e0e..bf0a653 100644 --- a/modules/bringauto_log/Logger.go +++ b/modules/bringauto_log/Logger.go @@ -26,7 +26,7 @@ func (logger *Logger) Info(msg string, args ...any) { if len(args) == 0 { logger.slogger.Info(msg) } else { - logger.slogger.Info(fmt.Sprintf(msg, args)) + logger.slogger.Info(fmt.Sprintf(msg, args...)) } } @@ -34,7 +34,7 @@ func (logger *Logger) InfoIndent(msg string, args ...any) { if len(args) == 0 { logger.slogger.Info(indent + msg) } else { - logger.slogger.Info(indent + fmt.Sprintf(msg, args)) + logger.slogger.Info(indent + fmt.Sprintf(msg, args...)) } } @@ -42,7 +42,7 @@ func (logger *Logger) Warn(msg string, args ...any) { if len(args) == 0 { logger.slogger.Warn(msg) } else { - logger.slogger.Warn(fmt.Sprintf(msg, args)) + logger.slogger.Warn(fmt.Sprintf(msg, args...)) } } @@ -50,7 +50,7 @@ func (logger *Logger) WarnIndent(msg string, args ...any) { if len(args) == 0 { logger.slogger.Warn(indent + msg) } else { - logger.slogger.Warn(indent + fmt.Sprintf(msg, args)) + logger.slogger.Warn(indent + fmt.Sprintf(msg, args...)) } } @@ -58,7 +58,7 @@ func (logger *Logger) Error(msg string, args ...any) { if len(args) == 0 { logger.slogger.Error(msg) } else { - logger.slogger.Error(fmt.Sprintf(msg, args)) + logger.slogger.Error(fmt.Sprintf(msg, args...)) } } @@ -66,7 +66,7 @@ func (logger *Logger) Fatal(msg string, args ...any) { if len(args) == 0 { logger.slogger.Error(msg) } else { - logger.slogger.Error(fmt.Sprintf(msg, args)) + logger.slogger.Error(fmt.Sprintf(msg, args...)) } os.Exit(1) } From e03af073a4713ddcc7530680b5d0a4ae43c5ef99 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Tue, 17 Sep 2024 13:27:10 +0300 Subject: [PATCH 41/91] Remove redundant build log with --all --- bap-builder/PackageMode.go | 1 - 1 file changed, 1 deletion(-) diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index 5de154e..4d1ee86 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -158,7 +158,6 @@ func buildAllPackages(cmdLine *BuildPackageCmdLineArgs, contextPath string) erro continue } count++ - logger.Info("Build %s", buildConfigs[0].Package.GetFullPackageName()) err = buildAndCopyPackage(cmdLine, &buildConfigs) if err != nil { logger.Fatal("cannot build package '%s' - %s", config.Package.Name, err) From f2ad0db434cdc7ac85cd552a999d83f4c119d458 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Wed, 18 Sep 2024 11:21:44 +0300 Subject: [PATCH 42/91] Add logs when building single package --- bap-builder/DockerMode.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bap-builder/DockerMode.go b/bap-builder/DockerMode.go index a27c700..65cc00f 100644 --- a/bap-builder/DockerMode.go +++ b/bap-builder/DockerMode.go @@ -64,15 +64,17 @@ func buildSingleDockerImage(contextPath string, imageName string) error { if err != nil { return err } - + logger := bringauto_log.GetLogger() dockerfileDir := path.Dir(dockerfilePath) dockerBuild := bringauto_docker.DockerBuild{ DockerfileDir: dockerfileDir, Tag: imageName, } + logger.Info("Build Docker Image: %s", imageName) buildOk := dockerBuild.Build() if buildOk != nil { return fmt.Errorf("Build failed for %s image", imageName) } + logger.InfoIndent("Build OK") return nil } From 77a025d3c57c66dd424b9ad34c88a1dfc6edcb38 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Wed, 18 Sep 2024 12:07:04 +0300 Subject: [PATCH 43/91] Add ErrorIndent function --- modules/bringauto_log/Logger.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/bringauto_log/Logger.go b/modules/bringauto_log/Logger.go index bf0a653..ed5450c 100644 --- a/modules/bringauto_log/Logger.go +++ b/modules/bringauto_log/Logger.go @@ -62,6 +62,14 @@ func (logger *Logger) Error(msg string, args ...any) { } } +func (logger *Logger) ErrorIndent(msg string, args ...any) { + if len(args) == 0 { + logger.slogger.Error(indent + msg) + } else { + logger.slogger.Error(indent + fmt.Sprintf(msg, args...)) + } +} + func (logger *Logger) Fatal(msg string, args ...any) { if len(args) == 0 { logger.slogger.Error(msg) From 621ee776a46dbcdc8a5fddbcd032a59c1549f0a3 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Wed, 18 Sep 2024 12:07:38 +0300 Subject: [PATCH 44/91] DockerMode refactor --- bap-builder/DockerMode.go | 56 +++++++++++++++------------------------ 1 file changed, 22 insertions(+), 34 deletions(-) diff --git a/bap-builder/DockerMode.go b/bap-builder/DockerMode.go index 65cc00f..6a5055a 100644 --- a/bap-builder/DockerMode.go +++ b/bap-builder/DockerMode.go @@ -3,7 +3,6 @@ package main import ( "bringauto/modules/bringauto_log" "bringauto/modules/bringauto_docker" - "fmt" "path" ) @@ -11,59 +10,47 @@ import ( // process Docker mode of cmd line // func BuildDockerImage(cmdLine *BuildImageCmdLineArgs, contextPath string) error { + contextManager := ContextManager{ + ContextPath: contextPath, + } buildAll := cmdLine.All if *buildAll { - return buildAllDockerImages(contextPath) + return buildAllDockerImages(contextManager) } - return buildSingleDockerImage(contextPath, *cmdLine.Name) + + dockerfilePath, err := contextManager.GetImageDockerfilePath(*cmdLine.Name) + if err != nil { + return err + } + buildSingleDockerImage(*cmdLine.Name, dockerfilePath) + return nil } // buildAllDockerImages // builds all docker images in the given contextPath. // It returns nil if everything is ok, or not nil in case of error // -func buildAllDockerImages(contextPath string) error { - contextManager := ContextManager{ - ContextPath: contextPath, - } - dockerfileList, err := contextManager.GetAllImagesDockerfilePaths() +func buildAllDockerImages(contextManager ContextManager) error { + dockerfilePathList, err := contextManager.GetAllImagesDockerfilePaths() if err != nil { return err } logger := bringauto_log.GetLogger() - - for imageName, dockerfilePathList := range dockerfileList { - if len(dockerfilePathList) != 1 { + for imageName, dockerfilePath := range dockerfilePathList { + if len(dockerfilePath) != 1 { logger.Warn("Bug: multiple Dockerfile present for same image name %s", imageName) continue } - dockerfileDir := path.Dir(dockerfilePathList[0]) - dockerBuild := bringauto_docker.DockerBuild{ - DockerfileDir: dockerfileDir, - Tag: imageName, - } - logger.Info("Build Docker Image: %s", imageName) - err = dockerBuild.Build() - if err != nil { - return fmt.Errorf("Build failed for %s image", imageName) - } - logger.InfoIndent("Build OK") + buildSingleDockerImage(imageName, dockerfilePath[0]) } return nil } // buildSingleDockerImage -// builds a single docker image specified by a name. +// builds a single docker image specified by an image name and a path to Dockerfile. // -func buildSingleDockerImage(contextPath string, imageName string) error { - contextManager := ContextManager{ - ContextPath: contextPath, - } - dockerfilePath, err := contextManager.GetImageDockerfilePath(imageName) - if err != nil { - return err - } +func buildSingleDockerImage(imageName string, dockerfilePath string) error { logger := bringauto_log.GetLogger() dockerfileDir := path.Dir(dockerfilePath) dockerBuild := bringauto_docker.DockerBuild{ @@ -71,9 +58,10 @@ func buildSingleDockerImage(contextPath string, imageName string) error { Tag: imageName, } logger.Info("Build Docker Image: %s", imageName) - buildOk := dockerBuild.Build() - if buildOk != nil { - return fmt.Errorf("Build failed for %s image", imageName) + err := dockerBuild.Build() + if err != nil { + logger.ErrorIndent("Can't build image - %s", err) + return err } logger.InfoIndent("Build OK") return nil From 70dae6ffccb91cbb164550a86b7dc4217cf8969d Mon Sep 17 00:00:00 2001 From: Mikulas Date: Wed, 18 Sep 2024 12:19:33 +0300 Subject: [PATCH 45/91] Add comment about absent SIGINT handler --- bap-builder/DockerMode.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bap-builder/DockerMode.go b/bap-builder/DockerMode.go index 6a5055a..53488a6 100644 --- a/bap-builder/DockerMode.go +++ b/bap-builder/DockerMode.go @@ -58,6 +58,9 @@ func buildSingleDockerImage(imageName string, dockerfilePath string) error { Tag: imageName, } logger.Info("Build Docker Image: %s", imageName) + + // Building image does not require any handler when SIGINT is received. 'docker build' creates + // image after all steps from Dockerfile are successfully executed. err := dockerBuild.Build() if err != nil { logger.ErrorIndent("Can't build image - %s", err) From ac7106cb494b95c2c6db11518120da756b7b4934 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 23 Sep 2024 14:48:43 +0300 Subject: [PATCH 46/91] Doc update in ContextManager --- bap-builder/ContextManager.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bap-builder/ContextManager.go b/bap-builder/ContextManager.go index f607d20..5140c81 100644 --- a/bap-builder/ContextManager.go +++ b/bap-builder/ContextManager.go @@ -88,6 +88,8 @@ func (context *ContextManager) GetPackageJsonDefPaths(packageName string) ([]str return packageDefs, nil } +// getAllDepsJsonPaths +// returns all json defintions paths recursively for given package specified by its json definition path func (context *ContextManager) getAllDepsJsonPaths(packageJsonPath string) ([]string, error) { var config bringauto_config.Config err := config.LoadJSONConfig(packageJsonPath) @@ -121,7 +123,7 @@ func (context *ContextManager) getAllDepsJsonPaths(packageJsonPath string) ([]st } // GetPackageWithDepsJsonDefPaths -// returns all json definitions for given package and all its dependencies json definitions recursively +// returns all json definitions paths for given package and all its dependencies json definitions paths recursively func (context *ContextManager) GetPackageWithDepsJsonDefPaths(packageName string) ([]string, error) { packageDefs, err := context.GetPackageJsonDefPaths(packageName) if err != nil { From 58c8cb27c5b7ebdae601146578840bd28db2d8d5 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 23 Sep 2024 14:49:00 +0300 Subject: [PATCH 47/91] Doc update in PackageMode --- bap-builder/PackageMode.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index 4d1ee86..f0423d1 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -131,8 +131,8 @@ func BuildPackage(cmdLine *BuildPackageCmdLineArgs, contextPath string) error { } // buildAllPackages -// builds all docker images in the given contextPath. -// It returns nil if everything is ok, or not nil in case of error +// Builds all packages specified in contextPath. Also takes care of building all deps for all +// packages in correct order. It returns nil if everything is ok, or not nil in case of error. func buildAllPackages(cmdLine *BuildPackageCmdLineArgs, contextPath string) error { contextManager := ContextManager{ ContextPath: contextPath, @@ -171,8 +171,8 @@ func buildAllPackages(cmdLine *BuildPackageCmdLineArgs, contextPath string) erro } // buildSinglePackage -// build single package specified by a name -// It returns nil if everything is ok, or not nil in case of error +// Builds single package specified by name in cmdLine. Also takes care of building all deps for +// given package in correct order. It returns nil if everything is ok, or not nil in case of error. func buildSinglePackage(cmdLine *BuildPackageCmdLineArgs, contextPath string) error { contextManager := ContextManager{ ContextPath: contextPath, @@ -218,6 +218,8 @@ func buildSinglePackage(cmdLine *BuildPackageCmdLineArgs, contextPath string) er return nil } +// addConfigsToDefsMap +// Adds all configs in packageJsonPathList to defsMap. func addConfigsToDefsMap(defsMap *ConfigMapType, packageJsonPathList []string) { for _, packageJsonPath := range packageJsonPathList { var config bringauto_config.Config @@ -236,6 +238,8 @@ func addConfigsToDefsMap(defsMap *ConfigMapType, packageJsonPathList []string) { } } +// buildAndCopyPackage +// Builds single package, takes care of every step of build for single package. func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_build.Build) error { if *cmdLine.OutputDirMode != OutputDirModeGitLFS { return fmt.Errorf("invalid OutputDirmode. Only GitLFS is supported") From 44196885e23bc1fc3ec10d4bff74762bea4d02a3 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 23 Sep 2024 14:52:14 +0300 Subject: [PATCH 48/91] Add doc to bringauto_log module --- modules/bringauto_log/ContextLogger.go | 15 +++++++++- modules/bringauto_log/GlobalLogger.go | 11 ++++++++ modules/bringauto_log/Handler.go | 16 +++++++++++ modules/bringauto_log/Logger.go | 39 ++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) diff --git a/modules/bringauto_log/ContextLogger.go b/modules/bringauto_log/ContextLogger.go index 318ab97..88e83d1 100644 --- a/modules/bringauto_log/ContextLogger.go +++ b/modules/bringauto_log/ContextLogger.go @@ -11,15 +11,23 @@ const ( ImageBuildContext = "image_build" ) +// ContextLogger +// Is used for getting writable file for context logs of a package. type ContextLogger struct { Logger + // logFileName Whole log file name with context and extrnsion logFileName string } type contextLoggerInitArgs struct { + // LogDirPath Directory path, where logs will be save. LogDirPath string + // ImageName Image name to use as part of path to log file. ImageName string - PackageName string // If is empty, then it is considered as non-package log + // PackageName Package name to use as part of path to log file. If is empty, then it is considered + // as non-package log. + PackageName string + // LogContext Context of a log. Is used as log file name. LogContext string } @@ -40,6 +48,8 @@ func (logger *ContextLogger) FillDynamic(args *bringauto_prerequisites.Args) err return nil } +// initLogDir +// Creates directory for logs if it does not exists already. func (logger *ContextLogger) initLogDir() error { _, err := os.Stat(logger.logDirPath) if os.IsNotExist(err) { @@ -59,6 +69,9 @@ func (logger *ContextLogger) CheckPrerequisites(*bringauto_prerequisites.Args) e return nil } +// GetFile +// Returns writable file for writing logs for specified context of a package. The file must be +// closed by the caller. func (logger *ContextLogger) GetFile() (*os.File, error) { return os.OpenFile(logger.logDirPath + "/" + logger.logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) } diff --git a/modules/bringauto_log/GlobalLogger.go b/modules/bringauto_log/GlobalLogger.go index c0e5388..de9383b 100644 --- a/modules/bringauto_log/GlobalLogger.go +++ b/modules/bringauto_log/GlobalLogger.go @@ -6,8 +6,11 @@ import ( "time" ) +// globalLoggerSingleton Singleton module global variable for GlobalLogger. var globalLoggerSingleton *GlobalLogger +// GetLogger +// Returns GlobalLogger singleton to use for logging. func GetLogger() *GlobalLogger { if globalLoggerSingleton == nil { globalLoggerSingleton = bringauto_prerequisites.CreateAndInitialize[GlobalLogger]() @@ -16,12 +19,16 @@ func GetLogger() *GlobalLogger { return globalLoggerSingleton } +// GlobalLogger +// Struct used for logging on program level. type GlobalLogger struct { Logger } type globalLoggerInitArgs struct { + // Timestamp Current timestamp used for creating ContextLoggers. Timestamp time.Time + // LogDirPath Directory path, where created ContextLoggers will save logs. LogDirPath string } @@ -42,6 +49,8 @@ func (logger *GlobalLogger) FillDynamic(args *bringauto_prerequisites.Args) erro return nil } +// getTimestampString +// Return timestamp formatted string for use in path. func (logger *GlobalLogger) getTimestampString() string { return logger.timestamp.Format("2006-01-02_15:04:05") } @@ -51,6 +60,8 @@ func (logger *GlobalLogger) CheckPrerequisites(*bringauto_prerequisites.Args) er return nil } +// CreateContextLogger +// Creates ContextLogger for specified imageName, packageName and logContext. func (logger *GlobalLogger) CreateContextLogger(imageName string, packageName string, logContext string) *ContextLogger { packageContextLogger := bringauto_prerequisites.CreateAndInitialize[ContextLogger]( logger.logDirPath, imageName, packageName, logContext, diff --git a/modules/bringauto_log/Handler.go b/modules/bringauto_log/Handler.go index 92ca661..38da8b0 100644 --- a/modules/bringauto_log/Handler.go +++ b/modules/bringauto_log/Handler.go @@ -8,6 +8,7 @@ import ( "strconv" ) +// Color codes constants const ( red = 31 blue = 34 @@ -15,20 +16,29 @@ const ( white = 97 ) +// Handler +// Struct for setting style of logging specified by slog module. type Handler struct { + // writer Writer to use for logs writer io.Writer } +// NewHandler +// Returns new Handler struct with writer. func NewHandler(writer io.Writer) *Handler { handler := &Handler{writer: writer} return handler } +// Enabled +// Mandatory function as specified by slog module. func (handler *Handler) Enabled(_ context.Context, _ slog.Level) bool { return true } +// colorizeLevel +// Returns colorized level string. func colorizeLevel(level slog.Level) string { var colorCode int switch level.String() { @@ -43,6 +53,8 @@ func colorizeLevel(level slog.Level) string { return fmt.Sprintf("\033[%sm%s\033[0m", strconv.Itoa(colorCode), level.String()) } +// Handle +// Mandatory function which sets style as specified by slog module. func (handler *Handler) Handle(_ context.Context, r slog.Record) error { buf := make([]byte, 0, 1024) formated := r.Time.Format("2006-01-02 15:04:05") @@ -58,10 +70,14 @@ func (handler *Handler) Handle(_ context.Context, r slog.Record) error { return err } +// WithGroup +// Mandatory function as specified by slog module. Does not adjust logging. func (handler *Handler) WithGroup(name string) slog.Handler { return handler } +// WithAttrs +// Mandatory function as specified by slog module. Does not adjust logging. func (handler *Handler) WithAttrs(attrs []slog.Attr) slog.Handler { return handler } diff --git a/modules/bringauto_log/Logger.go b/modules/bringauto_log/Logger.go index ed5450c..cc42501 100644 --- a/modules/bringauto_log/Logger.go +++ b/modules/bringauto_log/Logger.go @@ -1,3 +1,16 @@ +// Bringauto package for consistent logging in packager. +// +// The Logger struct is a base for other loggers. The GlobalLogger is used for logging to console +// and for creating ContextLoggers. ContextLogger is used for logging output of tools or programs +// to log files. +// +// The Logger struct shouldn't be used directly. The GlobalLogger should be created and initialized +// with bringauto_prerequisites.CreateAndInitialize at the beginning of the program. Then anywhere +// in the codebase the bringauto_log.GetLogger() can be called to get created GlobalLogger +// singleton. Thanks to singleton design pattern, the GlobalLogger doesn't have to be forwarded +// throughout the codebase. GlobalLogger can create ContextLoggers, which can return writable log +// file with GetFile() method. These log file can be used for e.g. build output from docker +// container. package bringauto_log import ( @@ -12,16 +25,24 @@ const ( indent = " " ) +// Struct which is used as a base for GlobalLogger and ContextLogger. Contains methods for global +// logging. type Logger struct { + // slogger slog.Logger struct. slogger *slog.Logger timestamp time.Time logDirPath string } +// getDefaultLogger +// Returns default logger with style defined by Handler struct. func getDefaultLogger(writer io.Writer) *slog.Logger { return slog.New(NewHandler(writer)) } +// Info +// Global logging function with Info level. Formatted string with args can be added similarly as +// with fmt.printf function. func (logger *Logger) Info(msg string, args ...any) { if len(args) == 0 { logger.slogger.Info(msg) @@ -30,6 +51,9 @@ func (logger *Logger) Info(msg string, args ...any) { } } +// InfoIndent +// Global logging function with Info level with added pre-indent. Formatted string with args can be +// added similarly as with fmt.printf function. func (logger *Logger) InfoIndent(msg string, args ...any) { if len(args) == 0 { logger.slogger.Info(indent + msg) @@ -38,6 +62,9 @@ func (logger *Logger) InfoIndent(msg string, args ...any) { } } +// Warn +// Global logging function with Warning level. Formatted string with args can be added similarly as +// with fmt.printf function. func (logger *Logger) Warn(msg string, args ...any) { if len(args) == 0 { logger.slogger.Warn(msg) @@ -46,6 +73,9 @@ func (logger *Logger) Warn(msg string, args ...any) { } } +// WarnIndent +// Global logging function with Warning level with added pre-indent. Formatted string with args can +// be added similarly as with fmt.printf function. func (logger *Logger) WarnIndent(msg string, args ...any) { if len(args) == 0 { logger.slogger.Warn(indent + msg) @@ -54,6 +84,9 @@ func (logger *Logger) WarnIndent(msg string, args ...any) { } } +// Error +// Global logging function with Error level. Formatted string with args can be added similarly as +// with fmt.printf function. func (logger *Logger) Error(msg string, args ...any) { if len(args) == 0 { logger.slogger.Error(msg) @@ -62,6 +95,9 @@ func (logger *Logger) Error(msg string, args ...any) { } } +// ErrorIndent +// Global logging function with Error level with added pre-indent. Formatted string with args can +// be added similarly as with fmt.printf function. func (logger *Logger) ErrorIndent(msg string, args ...any) { if len(args) == 0 { logger.slogger.Error(indent + msg) @@ -70,6 +106,9 @@ func (logger *Logger) ErrorIndent(msg string, args ...any) { } } +// Fatal +// Global logging function with Error level. Formatted string with args can be added similarly as +// with fmt.printf function. After writing a log, the whole program exits with code 1. func (logger *Logger) Fatal(msg string, args ...any) { if len(args) == 0 { logger.slogger.Error(msg) From fe056d63dc5849a10d7cc15580fe783d67c91782 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 23 Sep 2024 15:32:51 +0300 Subject: [PATCH 49/91] bringauto_log module refactor Removed not used functionality for global logging in ContextLogger. GlobalLogger functionality moved to Logger to simplify bringauto_log structure and removed GlobalLogger. --- bap-builder/main.go | 2 +- modules/bringauto_log/ContextLogger.go | 5 +- modules/bringauto_log/GlobalLogger.go | 70 ----------------------- modules/bringauto_log/Logger.go | 79 ++++++++++++++++++++++---- 4 files changed, 70 insertions(+), 86 deletions(-) delete mode 100644 modules/bringauto_log/GlobalLogger.go diff --git a/bap-builder/main.go b/bap-builder/main.go index 4b064dc..5bf900f 100644 --- a/bap-builder/main.go +++ b/bap-builder/main.go @@ -12,7 +12,7 @@ import ( func main() { var err error var args CmdLineArgs - logger := bringauto_prerequisites.CreateAndInitialize[bringauto_log.GlobalLogger](time.Now(), "./log") + logger := bringauto_prerequisites.CreateAndInitialize[bringauto_log.Logger](time.Now(), "./log") args.InitFlags() err = args.ParseArgs(os.Args) diff --git a/modules/bringauto_log/ContextLogger.go b/modules/bringauto_log/ContextLogger.go index 88e83d1..355b09f 100644 --- a/modules/bringauto_log/ContextLogger.go +++ b/modules/bringauto_log/ContextLogger.go @@ -14,7 +14,7 @@ const ( // ContextLogger // Is used for getting writable file for context logs of a package. type ContextLogger struct { - Logger + logDirPath string // logFileName Whole log file name with context and extrnsion logFileName string } @@ -32,7 +32,6 @@ type contextLoggerInitArgs struct { } func (logger *ContextLogger) FillDefault(*bringauto_prerequisites.Args) error { - logger.slogger = getDefaultLogger(os.Stdout) logger.logDirPath = "" logger.logFileName = "" return nil @@ -55,7 +54,7 @@ func (logger *ContextLogger) initLogDir() error { if os.IsNotExist(err) { err = os.MkdirAll(logger.logDirPath, 0700) if err != nil { - logger.Error("Failed to create log directory - %s", err) + GetLogger().Error("Failed to create log directory - %s", err) return err } } diff --git a/modules/bringauto_log/GlobalLogger.go b/modules/bringauto_log/GlobalLogger.go deleted file mode 100644 index de9383b..0000000 --- a/modules/bringauto_log/GlobalLogger.go +++ /dev/null @@ -1,70 +0,0 @@ -package bringauto_log - -import ( - "bringauto/modules/bringauto_prerequisites" - "os" - "time" -) - -// globalLoggerSingleton Singleton module global variable for GlobalLogger. -var globalLoggerSingleton *GlobalLogger - -// GetLogger -// Returns GlobalLogger singleton to use for logging. -func GetLogger() *GlobalLogger { - if globalLoggerSingleton == nil { - globalLoggerSingleton = bringauto_prerequisites.CreateAndInitialize[GlobalLogger]() - globalLoggerSingleton.Warn("Global logger was not initialized. Printing to console.") - } - return globalLoggerSingleton -} - -// GlobalLogger -// Struct used for logging on program level. -type GlobalLogger struct { - Logger -} - -type globalLoggerInitArgs struct { - // Timestamp Current timestamp used for creating ContextLoggers. - Timestamp time.Time - // LogDirPath Directory path, where created ContextLoggers will save logs. - LogDirPath string -} - -func (logger *GlobalLogger) FillDefault(*bringauto_prerequisites.Args) error { - logger.slogger = getDefaultLogger(os.Stdout) - logger.timestamp = time.Time{} - logger.logDirPath = "" - return nil -} - -func (logger *GlobalLogger) FillDynamic(args *bringauto_prerequisites.Args) error { - if !bringauto_prerequisites.IsEmpty(args) { - var argsStruct globalLoggerInitArgs - bringauto_prerequisites.GetArgs(args, &argsStruct) - logger.timestamp = argsStruct.Timestamp - logger.logDirPath = argsStruct.LogDirPath + "/" + logger.getTimestampString() - } - return nil -} - -// getTimestampString -// Return timestamp formatted string for use in path. -func (logger *GlobalLogger) getTimestampString() string { - return logger.timestamp.Format("2006-01-02_15:04:05") -} - -func (logger *GlobalLogger) CheckPrerequisites(*bringauto_prerequisites.Args) error { - globalLoggerSingleton = logger - return nil -} - -// CreateContextLogger -// Creates ContextLogger for specified imageName, packageName and logContext. -func (logger *GlobalLogger) CreateContextLogger(imageName string, packageName string, logContext string) *ContextLogger { - packageContextLogger := bringauto_prerequisites.CreateAndInitialize[ContextLogger]( - logger.logDirPath, imageName, packageName, logContext, - ) - return packageContextLogger -} diff --git a/modules/bringauto_log/Logger.go b/modules/bringauto_log/Logger.go index cc42501..bdaddaf 100644 --- a/modules/bringauto_log/Logger.go +++ b/modules/bringauto_log/Logger.go @@ -1,19 +1,18 @@ // Bringauto package for consistent logging in packager. // -// The Logger struct is a base for other loggers. The GlobalLogger is used for logging to console -// and for creating ContextLoggers. ContextLogger is used for logging output of tools or programs -// to log files. +// The Logger is used for logging to console and for creating ContextLoggers. ContextLogger is used +// for logging output of tools or programs to log files. // -// The Logger struct shouldn't be used directly. The GlobalLogger should be created and initialized -// with bringauto_prerequisites.CreateAndInitialize at the beginning of the program. Then anywhere -// in the codebase the bringauto_log.GetLogger() can be called to get created GlobalLogger -// singleton. Thanks to singleton design pattern, the GlobalLogger doesn't have to be forwarded -// throughout the codebase. GlobalLogger can create ContextLoggers, which can return writable log -// file with GetFile() method. These log file can be used for e.g. build output from docker -// container. +// The Logger should be created and initialized with bringauto_prerequisites.CreateAndInitialize +// at the beginning of the program. Then anywhere in the codebase the bringauto_log.GetLogger() +// can be called to get created Logger singleton. Thanks to singleton design pattern, the Logger +// doesn't have to be forwarded throughout the codebase. Logger can create ContextLoggers, which +// can return writable log file with GetFile() method. These log file can be used for e.g. build +// output from docker container. package bringauto_log import ( + "bringauto/modules/bringauto_prerequisites" "io" "log/slog" "os" @@ -25,8 +24,20 @@ const ( indent = " " ) -// Struct which is used as a base for GlobalLogger and ContextLogger. Contains methods for global -// logging. +// loggerSingleton Singleton module global variable for Logger. +var loggerSingleton *Logger + +// GetLogger +// Returns Logger singleton to use for logging. +func GetLogger() *Logger { + if loggerSingleton == nil { + loggerSingleton = bringauto_prerequisites.CreateAndInitialize[Logger]() + loggerSingleton.Warn("Logger was not initialized. Printing to console.") + } + return loggerSingleton +} + +// Struct which is used for logging on program level and for creating ContextLoggers. type Logger struct { // slogger slog.Logger struct. slogger *slog.Logger @@ -34,12 +45,47 @@ type Logger struct { logDirPath string } +type loggerInitArgs struct { + // Timestamp Current timestamp used for creating ContextLoggers. + Timestamp time.Time + // LogDirPath Directory path, where created ContextLoggers will save logs. + LogDirPath string +} + +func (logger *Logger) FillDefault(*bringauto_prerequisites.Args) error { + logger.slogger = getDefaultLogger(os.Stdout) + logger.timestamp = time.Time{} + logger.logDirPath = "" + return nil +} + +func (logger *Logger) FillDynamic(args *bringauto_prerequisites.Args) error { + if !bringauto_prerequisites.IsEmpty(args) { + var argsStruct loggerInitArgs + bringauto_prerequisites.GetArgs(args, &argsStruct) + logger.timestamp = argsStruct.Timestamp + logger.logDirPath = argsStruct.LogDirPath + "/" + logger.getTimestampString() + } + return nil +} + +func (logger *Logger) CheckPrerequisites(*bringauto_prerequisites.Args) error { + loggerSingleton = logger + return nil +} + // getDefaultLogger // Returns default logger with style defined by Handler struct. func getDefaultLogger(writer io.Writer) *slog.Logger { return slog.New(NewHandler(writer)) } +// getTimestampString +// Return timestamp formatted string for use in path. +func (logger *Logger) getTimestampString() string { + return logger.timestamp.Format("2006-01-02_15:04:05") +} + // Info // Global logging function with Info level. Formatted string with args can be added similarly as // with fmt.printf function. @@ -117,3 +163,12 @@ func (logger *Logger) Fatal(msg string, args ...any) { } os.Exit(1) } + +// CreateContextLogger +// Creates ContextLogger for specified imageName, packageName and logContext. +func (logger *Logger) CreateContextLogger(imageName string, packageName string, logContext string) *ContextLogger { + packageContextLogger := bringauto_prerequisites.CreateAndInitialize[ContextLogger]( + logger.logDirPath, imageName, packageName, logContext, + ) + return packageContextLogger +} From 86d8fc534665867a7221ecd09fceb5083dc937d7 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 23 Sep 2024 15:54:44 +0300 Subject: [PATCH 50/91] Change indent const to byte encoding --- modules/bringauto_log/Logger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/bringauto_log/Logger.go b/modules/bringauto_log/Logger.go index bdaddaf..cba4670 100644 --- a/modules/bringauto_log/Logger.go +++ b/modules/bringauto_log/Logger.go @@ -21,7 +21,7 @@ import ( ) const ( - indent = " " + indent = "\x20\x20\x20\x20" // four spaces ) // loggerSingleton Singleton module global variable for Logger. From 1ef47dcb8c9fad23ba58113fe16eb45f19e17ece Mon Sep 17 00:00:00 2001 From: Mikulas Date: Tue, 24 Sep 2024 08:41:23 +0300 Subject: [PATCH 51/91] Add handling error --- modules/bringauto_log/ContextLogger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/bringauto_log/ContextLogger.go b/modules/bringauto_log/ContextLogger.go index 355b09f..f1d93a6 100644 --- a/modules/bringauto_log/ContextLogger.go +++ b/modules/bringauto_log/ContextLogger.go @@ -63,7 +63,7 @@ func (logger *ContextLogger) initLogDir() error { func (logger *ContextLogger) CheckPrerequisites(*bringauto_prerequisites.Args) error { if logger.logDirPath != "" { - logger.initLogDir() + return logger.initLogDir() } return nil } From 4cbc3e482304b1540cb6497011f63bdaafd30641 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Tue, 24 Sep 2024 09:08:11 +0300 Subject: [PATCH 52/91] Add doc to Tar.go --- modules/bringauto_ssh/Tar.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/bringauto_ssh/Tar.go b/modules/bringauto_ssh/Tar.go index 0a03ae3..ec8c7e5 100644 --- a/modules/bringauto_ssh/Tar.go +++ b/modules/bringauto_ssh/Tar.go @@ -6,8 +6,12 @@ import ( "os" ) +// Tar +// Struct for creating tar archive using a tar tool type Tar struct { + // ArchiveName name of the archive which will be created ArchiveName string + // SourceDir source directory where are files which will be added to archive (without root folder) SourceDir string } @@ -25,6 +29,8 @@ func (tar *Tar) CheckPrerequisites(*bringauto_prerequisites.Args) error { return nil } +// ConstructCMDLine +// Constructs command for tar tool. func (tar *Tar) ConstructCMDLine() []string { var cmdLine []string cmdLine = append(cmdLine, "tar") @@ -35,4 +41,4 @@ func (tar *Tar) ConstructCMDLine() []string { cmdLine = append(cmdLine, ".") return []string{strings.Join(cmdLine, " ")} -} \ No newline at end of file +} From 44abe303bff1280fa3449a96fa9ca4b4f4ece87c Mon Sep 17 00:00:00 2001 From: Mikulas Date: Tue, 24 Sep 2024 10:25:24 +0300 Subject: [PATCH 53/91] ContextManager refactor --- bap-builder/ContextManager.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/bap-builder/ContextManager.go b/bap-builder/ContextManager.go index 5140c81..6fcfc79 100644 --- a/bap-builder/ContextManager.go +++ b/bap-builder/ContextManager.go @@ -108,14 +108,15 @@ func (context *ContextManager) getAllDepsJsonPaths(packageJsonPath string) ([]st if err != nil { return []string{}, fmt.Errorf("couldn't load JSON config from %s path - %s", packageJsonPath, err) } - if depConfig.Package.IsDebug == config.Package.IsDebug { - jsonPathListWithDeps = append(jsonPathListWithDeps, packageDepJsonPath) - jsonPathListWithDepsTmp, err := context.getAllDepsJsonPaths(packageDepJsonPath) - if err != nil { - return []string{}, err - } - jsonPathListWithDeps = append(jsonPathListWithDeps, jsonPathListWithDepsTmp...) + if depConfig.Package.IsDebug != config.Package.IsDebug { + continue } + jsonPathListWithDeps = append(jsonPathListWithDeps, packageDepJsonPath) + jsonPathListWithDepsTmp, err := context.getAllDepsJsonPaths(packageDepJsonPath) + if err != nil { + return []string{}, err + } + jsonPathListWithDeps = append(jsonPathListWithDeps, jsonPathListWithDepsTmp...) } } From 0cc9ecc699975b2709c4f03501f7a9243500e6b7 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Wed, 25 Sep 2024 11:24:37 +0300 Subject: [PATCH 54/91] Move constants used by multiple modules to new module --- modules/bringauto_build/Build.go | 5 +++-- modules/bringauto_build/Const.go | 2 -- modules/bringauto_const/Const.go | 13 +++++++++++++ modules/bringauto_docker/Docker.go | 3 ++- modules/bringauto_docker/DockerTools.go | 6 ++++-- modules/bringauto_ssh/SFTP.go | 5 +++-- modules/bringauto_ssh/SSHSession.go | 3 ++- 7 files changed, 27 insertions(+), 10 deletions(-) create mode 100644 modules/bringauto_const/Const.go diff --git a/modules/bringauto_build/Build.go b/modules/bringauto_build/Build.go index e9d7459..737208a 100644 --- a/modules/bringauto_build/Build.go +++ b/modules/bringauto_build/Build.go @@ -4,6 +4,7 @@ import ( "bringauto/modules/bringauto_docker" "bringauto/modules/bringauto_git" "bringauto/modules/bringauto_log" + "bringauto/modules/bringauto_const" "bringauto/modules/bringauto_package" "bringauto/modules/bringauto_prerequisites" "bringauto/modules/bringauto_ssh" @@ -84,7 +85,7 @@ func (build *Build) RunBuild() error { if found { return fmt.Errorf("do not specify CMAKE_INSTALL_PREFIX") } - build.CMake.Defines["CMAKE_INSTALL_PREFIX"] = dockerInstallDirConst + build.CMake.Defines["CMAKE_INSTALL_PREFIX"] = bringauto_const.DockerInstallDirConst if build.sysroot != nil { build.sysroot.CreateSysrootDir() @@ -222,7 +223,7 @@ func (build *Build) downloadInstalledFiles() error { defer logFile.Close() sftpClient := bringauto_ssh.SFTP{ - RemoteDir: dockerInstallDirConst, + RemoteDir: bringauto_const.DockerInstallDirConst, EmptyLocalDir: copyDir, SSHCredentials: build.SSHCredentials, LogWriter: logFile, diff --git a/modules/bringauto_build/Const.go b/modules/bringauto_build/Const.go index 3be2f0f..f5e4949 100644 --- a/modules/bringauto_build/Const.go +++ b/modules/bringauto_build/Const.go @@ -5,8 +5,6 @@ import ( ) const ( - // Where to install files on the remote machine - dockerInstallDirConst = string(filepath.Separator) + "INSTALL" // Where to clone a git repository on the remote machine dockerGitCloneDirConst = string(filepath.Separator) + "git" // Where to copy file from remote machine before the package is created diff --git a/modules/bringauto_const/Const.go b/modules/bringauto_const/Const.go new file mode 100644 index 0000000..9969594 --- /dev/null +++ b/modules/bringauto_const/Const.go @@ -0,0 +1,13 @@ +// Package for collecting all constants used by more modules in one place +package bringauto_const + +import ( + "path/filepath" +) + +const ( + // Where to install files on the remote machine + DockerInstallDirConst = string(filepath.Separator) + "INSTALL" + // Default SSH port of docker container + DefaultSSHPort = 1122 +) diff --git a/modules/bringauto_docker/Docker.go b/modules/bringauto_docker/Docker.go index 5a4fd3c..e74a24c 100644 --- a/modules/bringauto_docker/Docker.go +++ b/modules/bringauto_docker/Docker.go @@ -3,6 +3,7 @@ package bringauto_docker import ( "bringauto/modules/bringauto_prerequisites" "bringauto/modules/bringauto_process" + "bringauto/modules/bringauto_const" "fmt" "os" ) @@ -33,7 +34,7 @@ func (docker *Docker) FillDefault(*bringauto_prerequisites.Args) error { RunAsDaemon: true, ImageName: defaultImageNameConst, Ports: map[int]int{ - 1122: 22, + bringauto_const.DefaultSSHPort: 22, }, } return nil diff --git a/modules/bringauto_docker/DockerTools.go b/modules/bringauto_docker/DockerTools.go index 11b4b0e..68417ad 100644 --- a/modules/bringauto_docker/DockerTools.go +++ b/modules/bringauto_docker/DockerTools.go @@ -2,7 +2,9 @@ package bringauto_docker import ( "bringauto/modules/bringauto_process" + "bringauto/modules/bringauto_const" "bytes" + "strconv" ) func IsDefaultPortAvailable() bool { @@ -15,7 +17,7 @@ func IsDefaultPortAvailable() bool { "container", "ls", "--filter", - "publish=1122", + "publish=" + strconv.Itoa(bringauto_const.DefaultSSHPort), "--format", "{{.ID}}{{.Ports}}", }, @@ -27,4 +29,4 @@ func IsDefaultPortAvailable() bool { process.Run() return outBuff.Len() == 0 -} \ No newline at end of file +} diff --git a/modules/bringauto_ssh/SFTP.go b/modules/bringauto_ssh/SFTP.go index dcc3820..241b93a 100644 --- a/modules/bringauto_ssh/SFTP.go +++ b/modules/bringauto_ssh/SFTP.go @@ -1,6 +1,7 @@ package bringauto_ssh import ( + "bringauto/modules/bringauto_const" "bufio" "fmt" "io" @@ -34,7 +35,7 @@ func (sftpd *SFTP) DownloadDirectory() error { tar := Tar{ ArchiveName: archiveName, - SourceDir: "/INSTALL", + SourceDir: bringauto_const.DockerInstallDirConst, } shellEvaluator := ShellEvaluator{ @@ -45,7 +46,7 @@ func (sftpd *SFTP) DownloadDirectory() error { err = shellEvaluator.RunOverSSH(*sftpd.SSHCredentials) if err != nil { - return fmt.Errorf("cannot archive /INSTALL dir in docker container - %s", err) + return fmt.Errorf("cannot archive %s dir in docker container - %s", bringauto_const.DockerInstallDirConst, err) } sshSession := SSHSession{} diff --git a/modules/bringauto_ssh/SSHSession.go b/modules/bringauto_ssh/SSHSession.go index e203405..2401016 100644 --- a/modules/bringauto_ssh/SSHSession.go +++ b/modules/bringauto_ssh/SSHSession.go @@ -2,6 +2,7 @@ package bringauto_ssh import ( "bringauto/modules/bringauto_prerequisites" + "bringauto/modules/bringauto_const" "fmt" "golang.org/x/crypto/ssh" "io" @@ -32,7 +33,7 @@ type SSHSession struct { func (cred *SSHCredentials) FillDefault(*bringauto_prerequisites.Args) error { *cred = SSHCredentials{ IPAddress: "127.0.0.1", - Port: 1122, + Port: bringauto_const.DefaultSSHPort, Username: "root", Password: "1234", } From 7045109e5f8e0b527911bf79e31d3c48f6dff9ff Mon Sep 17 00:00:00 2001 From: Mikulas Date: Thu, 26 Sep 2024 11:00:14 +0300 Subject: [PATCH 55/91] Doc fixed --- bap-builder/CmdArgs.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bap-builder/CmdArgs.go b/bap-builder/CmdArgs.go index fddcfc3..d9bfcb5 100644 --- a/bap-builder/CmdArgs.go +++ b/bap-builder/CmdArgs.go @@ -27,7 +27,7 @@ type BuildPackageCmdLineArgs struct { All *bool // Name of the package to build (name of the directory in packages/ dir) Name *string - // Build all dependencies of package when building single package + // BuildDeps Build all dependencies of package when building single package BuildDeps *bool // DockerImageName is a name of docker image to which packages will be build. // If empty all docker images from DockerMatrix in config file are used for a given package. @@ -36,7 +36,7 @@ type BuildPackageCmdLineArgs struct { DockerImageName *string // OutputDir relative (to program working dir) ot absolute path where the package will be stored OutputDir *string - // Output dir mode + // OutputDirMode Output dir mode OutputDirMode *OutputDirMode } From 051bd8a5f53f80cc9e5b0f6d793c7007898c7a16 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Fri, 27 Sep 2024 10:46:38 +0300 Subject: [PATCH 56/91] Refactor using PlatformString The PlatformString is determined only once - better performance, less docker container runs. --- bap-builder/PackageMode.go | 61 ++++++++++++++---------------- modules/bringauto_config/Config.go | 11 +++--- 2 files changed, 33 insertions(+), 39 deletions(-) diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index f0423d1..3290db2 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -5,6 +5,8 @@ import ( "bringauto/modules/bringauto_build" "bringauto/modules/bringauto_config" "bringauto/modules/bringauto_package" + "bringauto/modules/bringauto_docker" + "bringauto/modules/bringauto_ssh" "bringauto/modules/bringauto_prerequisites" "bringauto/modules/bringauto_repository" "bringauto/modules/bringauto_sysroot" @@ -123,17 +125,21 @@ func (list *buildDepList) sortDependencies(rootName string, dependsMap *map[stri // BuildPackage // process Package mode of the program func BuildPackage(cmdLine *BuildPackageCmdLineArgs, contextPath string) error { + platformString, err := determinePlatformString(*cmdLine.DockerImageName) + if err != nil { + return err + } buildAll := cmdLine.All if *buildAll { - return buildAllPackages(cmdLine, contextPath) + return buildAllPackages(cmdLine, contextPath, platformString) } - return buildSinglePackage(cmdLine, contextPath) + return buildSinglePackage(cmdLine, contextPath, platformString) } // buildAllPackages // Builds all packages specified in contextPath. Also takes care of building all deps for all // packages in correct order. It returns nil if everything is ok, or not nil in case of error. -func buildAllPackages(cmdLine *BuildPackageCmdLineArgs, contextPath string) error { +func buildAllPackages(cmdLine *BuildPackageCmdLineArgs, contextPath string, platformString *bringauto_package.PlatformString) error { contextManager := ContextManager{ ContextPath: contextPath, } @@ -153,12 +159,12 @@ func buildAllPackages(cmdLine *BuildPackageCmdLineArgs, contextPath string) erro count := int32(0) for _, config := range configList { - buildConfigs := config.GetBuildStructure(*cmdLine.DockerImageName) + buildConfigs := config.GetBuildStructure(*cmdLine.DockerImageName, platformString) if len(buildConfigs) == 0 { continue } count++ - err = buildAndCopyPackage(cmdLine, &buildConfigs) + err = buildAndCopyPackage(cmdLine, &buildConfigs, platformString) if err != nil { logger.Fatal("cannot build package '%s' - %s", config.Package.Name, err) } @@ -173,7 +179,7 @@ func buildAllPackages(cmdLine *BuildPackageCmdLineArgs, contextPath string) erro // buildSinglePackage // Builds single package specified by name in cmdLine. Also takes care of building all deps for // given package in correct order. It returns nil if everything is ok, or not nil in case of error. -func buildSinglePackage(cmdLine *BuildPackageCmdLineArgs, contextPath string) error { +func buildSinglePackage(cmdLine *BuildPackageCmdLineArgs, contextPath string, platformString *bringauto_package.PlatformString) error { contextManager := ContextManager{ ContextPath: contextPath, } @@ -209,8 +215,8 @@ func buildSinglePackage(cmdLine *BuildPackageCmdLineArgs, contextPath string) er } for _, config := range configList { - buildConfigs := config.GetBuildStructure(*cmdLine.DockerImageName) - err = buildAndCopyPackage(cmdLine, &buildConfigs) + buildConfigs := config.GetBuildStructure(*cmdLine.DockerImageName, platformString) + err = buildAndCopyPackage(cmdLine, &buildConfigs, platformString) if err != nil { logger.Fatal("cannot build package '%s' - %s", packageName, err) } @@ -240,7 +246,7 @@ func addConfigsToDefsMap(defsMap *ConfigMapType, packageJsonPathList []string) { // buildAndCopyPackage // Builds single package, takes care of every step of build for single package. -func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_build.Build) error { +func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_build.Build, platformString *bringauto_package.PlatformString) error { if *cmdLine.OutputDirMode != OutputDirModeGitLFS { return fmt.Errorf("invalid OutputDirmode. Only GitLFS is supported") } @@ -257,11 +263,6 @@ func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_bu logger := bringauto_log.GetLogger() for _, buildConfig := range *build { - platformString, err := determinePlatformString(&buildConfig) - if err != nil { - return err - } - logger.Info("Build %s", buildConfig.Package.GetFullPackageName()) sysroot := bringauto_sysroot.Sysroot{ @@ -306,24 +307,18 @@ func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_bu return nil } -// determinePlatformString will construct platform string suitable -// for sysroot. -// For example: the any_machine platformString must be copied to all machine-specific sysroot for -// a given image. -func determinePlatformString(build *bringauto_build.Build) (*bringauto_package.PlatformString, error) { - platformStringSpecialized := build.Package.PlatformString - if build.Package.PlatformString.Mode == bringauto_package.ModeAnyMachine { - platformStringStruct := bringauto_package.PlatformString{ - Mode: bringauto_package.ModeAuto, - } - platformStringStruct.Mode = bringauto_package.ModeAuto - err := bringauto_prerequisites.Initialize[bringauto_package.PlatformString](&platformStringStruct, - build.SSHCredentials, build.Docker, - ) - if err != nil { - return nil, err - } - platformStringSpecialized.String.Machine = platformStringStruct.String.Machine +// determinePlatformString +// Will construct platform string suitable for sysroot. +func determinePlatformString(dockerImageName string) (*bringauto_package.PlatformString, error) { + defaultDocker := bringauto_prerequisites.CreateAndInitialize[bringauto_docker.Docker]() + defaultDocker.ImageName = dockerImageName + + sshCreds := bringauto_prerequisites.CreateAndInitialize[bringauto_ssh.SSHCredentials]() + + platformString := bringauto_package.PlatformString{ + Mode: bringauto_package.ModeAuto, } - return &platformStringSpecialized, nil + + err := bringauto_prerequisites.Initialize[bringauto_package.PlatformString](&platformString, sshCreds, defaultDocker) + return &platformString, err } diff --git a/modules/bringauto_config/Config.go b/modules/bringauto_config/Config.go index 39ab6d4..628777e 100644 --- a/modules/bringauto_config/Config.go +++ b/modules/bringauto_config/Config.go @@ -6,7 +6,6 @@ import ( "bringauto/modules/bringauto_git" "bringauto/modules/bringauto_package" "bringauto/modules/bringauto_prerequisites" - "bringauto/modules/bringauto_ssh" "encoding/json" "fmt" "os" @@ -83,13 +82,13 @@ func (config *Config) SaveToJSONConfig(configPath string) error { // Returns array of builds structs for specific image name. The returned array will contain max one build. // It is an array for simple handling of result using for loop. -func (config *Config) GetBuildStructure(imageName string) []bringauto_build.Build { +func (config *Config) GetBuildStructure(imageName string, platformString *bringauto_package.PlatformString) []bringauto_build.Build { var buildConfigs []bringauto_build.Build for _, value := range config.DockerMatrix.ImageNames { if imageName != "" && imageName != value { continue } - build := config.fillBuildStructure(value) + build := config.fillBuildStructure(imageName, platformString) defaultBuild := bringauto_prerequisites.CreateAndInitialize[bringauto_build.Build]() err := copier.CopyWithOption(defaultBuild, build, copier.Option{DeepCopy: true, IgnoreEmpty: true}) if err != nil { @@ -101,7 +100,7 @@ func (config *Config) GetBuildStructure(imageName string) []bringauto_build.Buil return buildConfigs } -func (config *Config) fillBuildStructure(dockerImageName string) bringauto_build.Build { +func (config *Config) fillBuildStructure(dockerImageName string, platformString *bringauto_package.PlatformString) bringauto_build.Build { var err error defaultDocker := bringauto_prerequisites.CreateAndInitialize[bringauto_docker.Docker]() defaultDocker.ImageName = dockerImageName @@ -122,9 +121,9 @@ func (config *Config) fillBuildStructure(dockerImageName string) bringauto_build panic(err) } - sshCreds := bringauto_prerequisites.CreateAndInitialize[bringauto_ssh.SSHCredentials]() tmpPackage := config.Package - err = bringauto_prerequisites.Initialize(&tmpPackage, sshCreds, defaultDocker) + err = bringauto_prerequisites.Initialize(&tmpPackage) + tmpPackage.PlatformString = *platformString if err != nil { panic(err) } From 9b9770d0e88d6f5fc5f227ddcbb4fd2a38dccb9f Mon Sep 17 00:00:00 2001 From: Mikulas Date: Fri, 27 Sep 2024 11:00:30 +0300 Subject: [PATCH 57/91] Change of sysroot check The check if sysroot directory is empty is now done only once at the beginning of the program. --- bap-builder/PackageMode.go | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index 3290db2..b9338d9 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -129,6 +129,7 @@ func BuildPackage(cmdLine *BuildPackageCmdLineArgs, contextPath string) error { if err != nil { return err } + checkSysrootDirs(platformString) buildAll := cmdLine.All if *buildAll { return buildAllPackages(cmdLine, contextPath, platformString) @@ -270,10 +271,6 @@ func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_bu PlatformString: platformString, } err = bringauto_prerequisites.Initialize(&sysroot) - if !sysroot.IsSysrootDirectoryEmpty() { - logger.WarnIndent("Sysroot directory is not empty - the package build may fail") - } - buildConfig.SetSysroot(&sysroot) logger.InfoIndent("Run build inside container") @@ -322,3 +319,26 @@ func determinePlatformString(dockerImageName string) (*bringauto_package.Platfor err := bringauto_prerequisites.Initialize[bringauto_package.PlatformString](&platformString, sshCreds, defaultDocker) return &platformString, err } + +// checkSysrootDirs +// Checks if sysroot release and debug directories are empty. If not, prints a warning. +func checkSysrootDirs(platformString *bringauto_package.PlatformString) (error) { + sysroot := bringauto_sysroot.Sysroot{ + IsDebug: false, + PlatformString: platformString, + } + err := bringauto_prerequisites.Initialize(&sysroot) + if err != nil { + return err + } + + logger := bringauto_log.GetLogger() + if !sysroot.IsSysrootDirectoryEmpty() { + logger.WarnIndent("Sysroot release directory is not empty - the package build may fail") + } + sysroot.IsDebug = true + if !sysroot.IsSysrootDirectoryEmpty() { + logger.WarnIndent("Sysroot debug directory is not empty - the package build may fail") + } + return nil +} From fc03f936766ff4b1e507b659979d26be6ac4953d Mon Sep 17 00:00:00 2001 From: Mikulas Date: Fri, 27 Sep 2024 15:31:03 +0300 Subject: [PATCH 58/91] Rename SignalHandler functions --- bap-builder/PackageMode.go | 2 +- bap-builder/main.go | 2 +- modules/bringauto_build/Build.go | 2 +- modules/bringauto_process/SignalHandler.go | 16 ++++++++-------- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index b9338d9..7a6e0a5 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -281,7 +281,7 @@ func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_bu logger.InfoIndent("Copying to Git repository") - removeHandler := bringauto_process.AddHandler(buildConfig.CleanUp) + removeHandler := bringauto_process.SignalHandlerAddHandler(buildConfig.CleanUp) err = repo.CopyToRepository(*buildConfig.Package, buildConfig.GetLocalInstallDirPath()) if err != nil { diff --git a/bap-builder/main.go b/bap-builder/main.go index 5bf900f..124d715 100644 --- a/bap-builder/main.go +++ b/bap-builder/main.go @@ -20,7 +20,7 @@ func main() { logger.Error("Can't parse cmd line arguments - %s", err) return } - bringauto_process.RegisterSignal(syscall.SIGINT) + bringauto_process.SignalHandlerRegisterSignal(syscall.SIGINT) if args.BuildImage { err = BuildDockerImage(&args.BuildImagesArgs, *args.Context) diff --git a/modules/bringauto_build/Build.go b/modules/bringauto_build/Build.go index 737208a..9f0cb42 100644 --- a/modules/bringauto_build/Build.go +++ b/modules/bringauto_build/Build.go @@ -138,7 +138,7 @@ func (build *Build) RunBuild() error { } dockerRun := (*bringauto_docker.DockerRun)(build.Docker) - removeHandler := bringauto_process.AddHandler(build.stopAndRemoveContainer) + removeHandler := bringauto_process.SignalHandlerAddHandler(build.stopAndRemoveContainer) defer removeHandler() err = dockerRun.Run() diff --git a/modules/bringauto_process/SignalHandler.go b/modules/bringauto_process/SignalHandler.go index 4354fda..119fa64 100644 --- a/modules/bringauto_process/SignalHandler.go +++ b/modules/bringauto_process/SignalHandler.go @@ -1,11 +1,11 @@ // Functions for global signal handling // -// The Register() must be called to start handling signal by this module. Then AddHandler() can be -// called to add handlers to execute when the registered signal is received. The method returns -// function which should be deferred after AddHandler() call by caller. This returned function -// basically removes handler from execution after signal is received. When registered signal is -// received, all added handlers (and not removed yet) will be executed in reverse order and then -// the program exits with status code 1. +// The SignalHandlerRegisterSignal() must be called to start handling signal by this module. Then +// SignalHandlerAddHandler() can be called to add handlers to execute when the registered signal is +// received. The method returns function which should be deferred after SignalHandlerAddHandler() +// call by caller. This returned function basically removes handler from execution after signal is +// received. When registered signal is received, all added handlers (and not removed yet) will be +// executed in reverse order and then the program exits with status code 1. // // Note: Do not use with concurrent programming. Can behave unexpectedly! @@ -24,7 +24,7 @@ var lock sync.Mutex var handlers []func() error // Registers handling of specified signals to bringauto_process package -func RegisterSignal(sig ...os.Signal) { +func SignalHandlerRegisterSignal(sig ...os.Signal) { sigs := make(chan os.Signal, 1) signal.Notify(sigs, sig...) go func() { @@ -40,7 +40,7 @@ func RegisterSignal(sig ...os.Signal) { // Adds handler for execution after signal is received by bringauto_process package. Returns handler // remover which should be deferred by caller. -func AddHandler(handler func() error) func() { +func SignalHandlerAddHandler(handler func() error) func() { lock.Lock() defer lock.Unlock() handlers = append(handlers, handler) From 9184f7ab7be8082cdd85bf635a3dd4e44dc44f8f Mon Sep 17 00:00:00 2001 From: Mikulas Date: Fri, 27 Sep 2024 15:32:14 +0300 Subject: [PATCH 59/91] Better error message --- modules/bringauto_sysroot/Sysroot.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/bringauto_sysroot/Sysroot.go b/modules/bringauto_sysroot/Sysroot.go index 648a13c..6114196 100644 --- a/modules/bringauto_sysroot/Sysroot.go +++ b/modules/bringauto_sysroot/Sysroot.go @@ -70,7 +70,7 @@ func (sysroot *Sysroot) anyFileAlreadyExistsInSysroot(source string) bool { return fmt.Errorf("file already exists in sysroot") } } - + return nil }) @@ -121,7 +121,7 @@ func (sysroot *Sysroot) IsSysrootDirectoryEmpty() bool { if err == io.EOF { // The directory exists, but is empty return true } else if err != nil { - bringauto_log.GetLogger().Warn("Cannot read in sysroot directory: %s", err) + bringauto_log.GetLogger().Warn("Sysroot directory is not readable: %s", err) } return false From 0ec198cd002ce9d471da81f93d263d40dbd25a7b Mon Sep 17 00:00:00 2001 From: Mikulas Date: Fri, 27 Sep 2024 17:22:33 +0300 Subject: [PATCH 60/91] Doc improvements --- modules/bringauto_process/SignalHandler.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/bringauto_process/SignalHandler.go b/modules/bringauto_process/SignalHandler.go index 119fa64..3ece021 100644 --- a/modules/bringauto_process/SignalHandler.go +++ b/modules/bringauto_process/SignalHandler.go @@ -23,6 +23,7 @@ import ( var lock sync.Mutex var handlers []func() error +// SignalHandlerRegisterSignal // Registers handling of specified signals to bringauto_process package func SignalHandlerRegisterSignal(sig ...os.Signal) { sigs := make(chan os.Signal, 1) @@ -38,8 +39,11 @@ func SignalHandlerRegisterSignal(sig ...os.Signal) { }() } +// SignalHandlerAddHandler // Adds handler for execution after signal is received by bringauto_process package. Returns handler -// remover which should be deferred by caller. +// remover which should be deferred by caller. So it should be used as this: +// handlerRemover := SignalHandlerAddHandler(my_handler) +// defer handlerRemover() func SignalHandlerAddHandler(handler func() error) func() { lock.Lock() defer lock.Unlock() From 917b97665d770e4d7c47c2dab9b80c7279a0354f Mon Sep 17 00:00:00 2001 From: Mikulas Date: Fri, 27 Sep 2024 17:27:44 +0300 Subject: [PATCH 61/91] Move buffer size to constant --- modules/bringauto_ssh/SFTP.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/bringauto_ssh/SFTP.go b/modules/bringauto_ssh/SFTP.go index 241b93a..5e2de96 100644 --- a/modules/bringauto_ssh/SFTP.go +++ b/modules/bringauto_ssh/SFTP.go @@ -15,6 +15,8 @@ import ( const ( archiveName string = "install_arch.tar" archiveNameSep string = string(os.PathSeparator) + archiveName + // Size of the buffer used by bufio module + bufferSize = 1024*1024 ) type SFTP struct { @@ -130,8 +132,8 @@ func (sftpd *SFTP) copyFile(sftpClient *sftp.Client, remoteFile string, localDir } func copyIOFile(sourceFile *sftp.File, destFile *os.File) { - sourceFileBuff := bufio.NewReaderSize(sourceFile, 1024*1024) - destFileBuff := bufio.NewWriterSize(destFile, 1027*1024) + sourceFileBuff := bufio.NewReaderSize(sourceFile, bufferSize) + destFileBuff := bufio.NewWriterSize(destFile, bufferSize) var err error _, err = io.Copy(destFileBuff, sourceFileBuff) From 7b5a097afcd0c236cf3aaf657c80bb4c531be055 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Fri, 27 Sep 2024 19:12:42 +0300 Subject: [PATCH 62/91] Error print improvement - overwriting sysroot The first 10 problematic files are now printed in error when trying to ovewrite files in sysroot. --- modules/bringauto_sysroot/Sysroot.go | 33 +++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/modules/bringauto_sysroot/Sysroot.go b/modules/bringauto_sysroot/Sysroot.go index 6114196..2f9cd5e 100644 --- a/modules/bringauto_sysroot/Sysroot.go +++ b/modules/bringauto_sysroot/Sysroot.go @@ -15,6 +15,9 @@ import ( const ( sysrootDirectoryName = "install_sysroot" + // Constant for number of problematic files which will be printed when trying to ovewrite files + // in sysroot + listFilesCount = 10 ) // Sysroot represents a standard Linux sysroot with all needed libraries installed. @@ -43,8 +46,10 @@ func (sysroot *Sysroot) CheckPrerequisites(args *bringauto_prerequisites.Args) e // CopyToSysroot copy source to a sysroot func (sysroot *Sysroot) CopyToSysroot(source string) error { - if sysroot.anyFileAlreadyExistsInSysroot(source) { - return fmt.Errorf("trying to ovewrite files in sysroot - sysroot consistency interrupted") + existingFiles := sysroot.getExistingFilesInSysroot(source) + if len(existingFiles) > 0 { + printOverwriteFilesError(existingFiles, listFilesCount) + return fmt.Errorf("trying to overwrite files in sysroot") } var err error copyOptions := copy.Options{ @@ -59,22 +64,38 @@ func (sysroot *Sysroot) CopyToSysroot(source string) error { return nil } -func (sysroot *Sysroot) anyFileAlreadyExistsInSysroot(source string) bool { +// printOverwriteFilesError +// Prints error for overwriting files in sysroot. Lists first n files in problematic_files. +func printOverwriteFilesError(problematicFiles []string, n int) { + logger := bringauto_log.GetLogger() + logger.Error("Trying to overwrite files in sysroot - sysroot consistency interrupted.") + logger.Error("Listing first %d of problematic files:", n) + for i, filePath := range problematicFiles { + logger.ErrorIndent(filePath) + if i == n - 1 { + break + } + } +} + +func (sysroot *Sysroot) getExistingFilesInSysroot(source string) []string { sysrootPath := sysroot.GetSysrootPath() - err := filepath.WalkDir(source, func(path string, d fs.DirEntry, err error) error { + var existingFiles []string + + filepath.WalkDir(source, func(path string, d fs.DirEntry, err error) error { if !d.IsDir() { filePath := strings.TrimPrefix(path, source) _, err := os.Stat(sysrootPath + filePath) if err == nil { - return fmt.Errorf("file already exists in sysroot") + existingFiles = append(existingFiles, sysrootPath + filePath) } } return nil }) - return err != nil + return existingFiles } // GetSysrootPath returns absolute path ot the sysroot From 566e30fe0d008dcd7f4c9f1d6476eda80f21c0c4 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Fri, 27 Sep 2024 19:57:20 +0300 Subject: [PATCH 63/91] Add check for existing files in sysroot Now the error for overwriting files in sysroot is not printed when trying to copy files which are not already there. Before if there was any file in sysroot, the build have failed. The files printed with error are also just the files which were actually tried to copy into sysroot, not all files in sysroot. --- modules/bringauto_sysroot/Sysroot.go | 77 ++++++++++++++++++---------- 1 file changed, 49 insertions(+), 28 deletions(-) diff --git a/modules/bringauto_sysroot/Sysroot.go b/modules/bringauto_sysroot/Sysroot.go index 2f9cd5e..3cb50fd 100644 --- a/modules/bringauto_sysroot/Sysroot.go +++ b/modules/bringauto_sysroot/Sysroot.go @@ -46,12 +46,10 @@ func (sysroot *Sysroot) CheckPrerequisites(args *bringauto_prerequisites.Args) e // CopyToSysroot copy source to a sysroot func (sysroot *Sysroot) CopyToSysroot(source string) error { - existingFiles := sysroot.getExistingFilesInSysroot(source) - if len(existingFiles) > 0 { - printOverwriteFilesError(existingFiles, listFilesCount) - return fmt.Errorf("trying to overwrite files in sysroot") + err := sysroot.checkForOverwritingFiles(source) + if err != nil { + return err } - var err error copyOptions := copy.Options{ OnSymlink: onSymlink, PreserveOwner: true, @@ -64,40 +62,42 @@ func (sysroot *Sysroot) CopyToSysroot(source string) error { return nil } +// checkForOverwritingFiles +// Checks if in dirPath directory are not files which are also in sysroot directory. If there are +// some, then prints Error with listing problematic files and returns non nil error. Else returns +// nil error without printing anything. +func (sysroot *Sysroot) checkForOverwritingFiles(dirPath string) error { + filesToCopy := getExistingFilesInDir(dirPath) + filesInSysroot := getExistingFilesInDir(sysroot.GetSysrootPath()) + var intersection []string + for _, fileToCopy := range filesToCopy { + for _, fileInSysroot := range filesInSysroot { + if fileToCopy == fileInSysroot { + intersection = append(intersection, fileToCopy) + } + } + } + if len(intersection) > 0 { + sysroot.printOverwriteFilesError(intersection, listFilesCount) + return fmt.Errorf("trying to overwrite files in sysroot") + } + return nil +} + // printOverwriteFilesError // Prints error for overwriting files in sysroot. Lists first n files in problematic_files. -func printOverwriteFilesError(problematicFiles []string, n int) { +func (sysroot *Sysroot) printOverwriteFilesError(problematicFiles []string, n int) { logger := bringauto_log.GetLogger() logger.Error("Trying to overwrite files in sysroot - sysroot consistency interrupted.") - logger.Error("Listing first %d of problematic files:", n) + logger.Error("Listing first %d problematic files:", n) for i, filePath := range problematicFiles { - logger.ErrorIndent(filePath) + logger.ErrorIndent(sysrootDirectoryName + "/" + sysroot.PlatformString.Serialize() + filePath) if i == n - 1 { break } } } -func (sysroot *Sysroot) getExistingFilesInSysroot(source string) []string { - sysrootPath := sysroot.GetSysrootPath() - - var existingFiles []string - - filepath.WalkDir(source, func(path string, d fs.DirEntry, err error) error { - if !d.IsDir() { - filePath := strings.TrimPrefix(path, source) - _, err := os.Stat(sysrootPath + filePath) - if err == nil { - existingFiles = append(existingFiles, sysrootPath + filePath) - } - } - - return nil - }) - - return existingFiles -} - // GetSysrootPath returns absolute path ot the sysroot func (sysroot *Sysroot) GetSysrootPath() string { workingDir, err := os.Getwd() @@ -151,3 +151,24 @@ func (sysroot *Sysroot) IsSysrootDirectoryEmpty() bool { func onSymlink(src string) copy.SymlinkAction { return copy.Shallow } + +// getExistingFilesInDir +// Returns array of string which contains all file paths existing in dirPath directory. The +// returned paths are without dirPath prefix. +func getExistingFilesInDir(dirPath string) []string { + var existingFiles []string + + filepath.WalkDir(dirPath, func(path string, d fs.DirEntry, err error) error { + if !d.IsDir() { + filePath := strings.TrimPrefix(path, dirPath) + _, err := os.Stat(path) + if err == nil { + existingFiles = append(existingFiles, filePath) + } + } + + return nil + }) + + return existingFiles +} From 8197f030860e166170be369493196f752b0007c9 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 30 Sep 2024 12:53:29 +0300 Subject: [PATCH 64/91] Refactor check for default port availability Add handling error returned by docker command to IsDefaultPortAvailable. Check for default port availability moved from other components code to Docker CheckPrerequisities. --- modules/bringauto_build/Build.go | 5 ----- modules/bringauto_docker/Docker.go | 8 +++++++- modules/bringauto_docker/DockerTools.go | 12 ++++++++---- modules/bringauto_package/PlatformString.go | 5 ----- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/modules/bringauto_build/Build.go b/modules/bringauto_build/Build.go index 9f0cb42..b7876f2 100644 --- a/modules/bringauto_build/Build.go +++ b/modules/bringauto_build/Build.go @@ -127,11 +127,6 @@ func (build *Build) RunBuild() error { StdOut: file, } - availablePort := bringauto_docker.IsDefaultPortAvailable() - if !availablePort { - return fmt.Errorf("default port not available") - } - err = bringauto_prerequisites.Initialize(build.Docker) if err != nil { return err diff --git a/modules/bringauto_docker/Docker.go b/modules/bringauto_docker/Docker.go index e74a24c..e54935b 100644 --- a/modules/bringauto_docker/Docker.go +++ b/modules/bringauto_docker/Docker.go @@ -48,6 +48,12 @@ func (docker *Docker) FillDynamic(*bringauto_prerequisites.Args) error { // It checks if the docker is installed and can be run by given user. // Function returns nil if Docker installation is ok, not nil of the problem is recognized func (docker *Docker) CheckPrerequisites(*bringauto_prerequisites.Args) error { + portAvailable, err := IsDefaultPortAvailable() + if err != nil { + return err + } else if !portAvailable { + return fmt.Errorf("default port not available") + } process := bringauto_process.Process{ CommandAbsolutePath: DockerExecutablePathConst, Args: bringauto_process.ProcessArgs{ @@ -56,7 +62,7 @@ func (docker *Docker) CheckPrerequisites(*bringauto_prerequisites.Args) error { }, }, } - err := process.Run() + err = process.Run() if err != nil { return err } diff --git a/modules/bringauto_docker/DockerTools.go b/modules/bringauto_docker/DockerTools.go index 68417ad..ad2fe1e 100644 --- a/modules/bringauto_docker/DockerTools.go +++ b/modules/bringauto_docker/DockerTools.go @@ -1,13 +1,14 @@ package bringauto_docker import ( - "bringauto/modules/bringauto_process" "bringauto/modules/bringauto_const" + "bringauto/modules/bringauto_process" "bytes" + "fmt" "strconv" ) -func IsDefaultPortAvailable() bool { +func IsDefaultPortAvailable() (bool, error) { var outBuff, errBuff bytes.Buffer process := bringauto_process.Process{ @@ -26,7 +27,10 @@ func IsDefaultPortAvailable() bool { StdErr: &errBuff, } - process.Run() + err := process.Run() + if err != nil { + return false, fmt.Errorf(errBuff.String()) + } - return outBuff.Len() == 0 + return outBuff.Len() == 0, nil } diff --git a/modules/bringauto_package/PlatformString.go b/modules/bringauto_package/PlatformString.go index fc24347..91a32ed 100644 --- a/modules/bringauto_package/PlatformString.go +++ b/modules/bringauto_package/PlatformString.go @@ -118,11 +118,6 @@ func (pstr *PlatformString) determinePlatformString(credentials bringauto_ssh.SS panic(fmt.Errorf("cannot determine PlatformString for explicit mode")) } - availablePort := bringauto_docker.IsDefaultPortAvailable() - if !availablePort { - return fmt.Errorf("default port not available") - } - dockerRun := (*bringauto_docker.DockerRun)(docker) err := dockerRun.Run() if err != nil { From 2da225ab0fc23720871380089b38827abd97dca3 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 30 Sep 2024 13:00:54 +0300 Subject: [PATCH 65/91] Log improvements --- bap-builder/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bap-builder/main.go b/bap-builder/main.go index 124d715..31e19ba 100644 --- a/bap-builder/main.go +++ b/bap-builder/main.go @@ -25,7 +25,7 @@ func main() { if args.BuildImage { err = BuildDockerImage(&args.BuildImagesArgs, *args.Context) if err != nil { - logger.Error(err.Error()) + logger.Error("Failed to build Docker image: %s", err) return } return @@ -34,7 +34,7 @@ func main() { if args.BuildPackage { err = BuildPackage(&args.BuildPackageArgs, *args.Context) if err != nil { - logger.Error(err.Error()) + logger.Error("Failed to build package: %s", err) return } return From d8967c2d09499d8838706a85c67a1fc7c0097ced Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 30 Sep 2024 13:19:42 +0300 Subject: [PATCH 66/91] More effective cmd line construction of Tar command --- modules/bringauto_ssh/Tar.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/modules/bringauto_ssh/Tar.go b/modules/bringauto_ssh/Tar.go index ec8c7e5..6595053 100644 --- a/modules/bringauto_ssh/Tar.go +++ b/modules/bringauto_ssh/Tar.go @@ -2,8 +2,8 @@ package bringauto_ssh import ( "bringauto/modules/bringauto_prerequisites" + "path/filepath" "strings" - "os" ) // Tar @@ -32,13 +32,12 @@ func (tar *Tar) CheckPrerequisites(*bringauto_prerequisites.Args) error { // ConstructCMDLine // Constructs command for tar tool. func (tar *Tar) ConstructCMDLine() []string { - var cmdLine []string - cmdLine = append(cmdLine, "tar") - cmdLine = append(cmdLine, "cvf") - cmdLine = append(cmdLine, tar.SourceDir + string(os.PathSeparator) + tar.ArchiveName) - cmdLine = append(cmdLine, "-C") - cmdLine = append(cmdLine, tar.SourceDir) - cmdLine = append(cmdLine, ".") + var cmdBuilder strings.Builder + cmdBuilder.WriteString("tar cvf ") + cmdBuilder.WriteString(filepath.Join(tar.SourceDir, tar.ArchiveName)) + cmdBuilder.WriteString(" -C ") + cmdBuilder.WriteString(tar.SourceDir) + cmdBuilder.WriteString(" .") - return []string{strings.Join(cmdLine, " ")} + return []string{cmdBuilder.String()} } From 32f0db4e15281f9ef14b2c82411ff72ac2e7710e Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 30 Sep 2024 13:33:28 +0300 Subject: [PATCH 67/91] Add prerequisites functionality to Tar, add check prerequisites --- modules/bringauto_ssh/SFTP.go | 6 ++---- modules/bringauto_ssh/Tar.go | 18 +++++++++++++++++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/modules/bringauto_ssh/SFTP.go b/modules/bringauto_ssh/SFTP.go index 5e2de96..310c573 100644 --- a/modules/bringauto_ssh/SFTP.go +++ b/modules/bringauto_ssh/SFTP.go @@ -2,6 +2,7 @@ package bringauto_ssh import ( "bringauto/modules/bringauto_const" + "bringauto/modules/bringauto_prerequisites" "bufio" "fmt" "io" @@ -35,10 +36,7 @@ type SFTP struct { func (sftpd *SFTP) DownloadDirectory() error { var err error - tar := Tar{ - ArchiveName: archiveName, - SourceDir: bringauto_const.DockerInstallDirConst, - } + tar := bringauto_prerequisites.CreateAndInitialize[Tar](archiveName, bringauto_const.DockerInstallDirConst) shellEvaluator := ShellEvaluator{ Commands: tar.ConstructCMDLine(), diff --git a/modules/bringauto_ssh/Tar.go b/modules/bringauto_ssh/Tar.go index 6595053..a56e7cd 100644 --- a/modules/bringauto_ssh/Tar.go +++ b/modules/bringauto_ssh/Tar.go @@ -2,6 +2,7 @@ package bringauto_ssh import ( "bringauto/modules/bringauto_prerequisites" + "fmt" "path/filepath" "strings" ) @@ -15,17 +16,32 @@ type Tar struct { SourceDir string } +type tarInitArgs struct { + ArchiveName string + SourceDir string +} + func (tar *Tar) FillDefault(*bringauto_prerequisites.Args) error { tar.ArchiveName = "" tar.SourceDir = "" return nil } -func (tar *Tar) FillDynamic(*bringauto_prerequisites.Args) error { +func (tar *Tar) FillDynamic(args *bringauto_prerequisites.Args) error { + var argsStruct tarInitArgs + bringauto_prerequisites.GetArgs(args, &argsStruct) + tar.ArchiveName = argsStruct.ArchiveName + tar.SourceDir = argsStruct.SourceDir return nil } func (tar *Tar) CheckPrerequisites(*bringauto_prerequisites.Args) error { + if tar.ArchiveName == "" { + return fmt.Errorf("empty archive name") + } + if tar.SourceDir == "" { + return fmt.Errorf("empty source directory name") + } return nil } From d8be850f11f7500292f2c4becd9d2e7702d17ad9 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 30 Sep 2024 13:52:39 +0300 Subject: [PATCH 68/91] Doc and log improvements --- modules/bringauto_docker/DockerBuild.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/bringauto_docker/DockerBuild.go b/modules/bringauto_docker/DockerBuild.go index b61c1c3..d62f1fb 100644 --- a/modules/bringauto_docker/DockerBuild.go +++ b/modules/bringauto_docker/DockerBuild.go @@ -23,11 +23,14 @@ func (dockerBuild *DockerBuild) Build() error { var ok = dockerBuild.prepareAndRun(prepareBuildArgs) if !ok { - return fmt.Errorf("DockerBuild build error") + return fmt.Errorf("DockerBuild build error for %s image", dockerBuild.Tag) } return nil } +// prepareAndRun +// Runs docker build command. Returns bool value which indicates if the command was succesfull or +// not. func (dockerBuild *DockerBuild) prepareAndRun(f func(build *DockerBuild) []string) bool { logger := bringauto_log.GetLogger() contextLogger := logger.CreateContextLogger(dockerBuild.Tag, "", bringauto_log.ImageBuildContext) From 5fa84d9d13b0d27efff7235a5dd63dd022bd7b15 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 30 Sep 2024 14:03:16 +0300 Subject: [PATCH 69/91] Add port number to error message --- modules/bringauto_docker/Docker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/bringauto_docker/Docker.go b/modules/bringauto_docker/Docker.go index e54935b..4a2de11 100644 --- a/modules/bringauto_docker/Docker.go +++ b/modules/bringauto_docker/Docker.go @@ -52,7 +52,7 @@ func (docker *Docker) CheckPrerequisites(*bringauto_prerequisites.Args) error { if err != nil { return err } else if !portAvailable { - return fmt.Errorf("default port not available") + return fmt.Errorf("default port %d not available", bringauto_const.DefaultSSHPort) } process := bringauto_process.Process{ CommandAbsolutePath: DockerExecutablePathConst, From cf1e570d124f9085e029ac0d5789b7444d14f18f Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 30 Sep 2024 14:22:41 +0300 Subject: [PATCH 70/91] Refactor and optimize Handle function --- modules/bringauto_log/Handler.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/modules/bringauto_log/Handler.go b/modules/bringauto_log/Handler.go index 38da8b0..661a5b8 100644 --- a/modules/bringauto_log/Handler.go +++ b/modules/bringauto_log/Handler.go @@ -6,6 +6,7 @@ import ( "io" "fmt" "strconv" + "strings" ) // Color codes constants @@ -56,17 +57,21 @@ func colorizeLevel(level slog.Level) string { // Handle // Mandatory function which sets style as specified by slog module. func (handler *Handler) Handle(_ context.Context, r slog.Record) error { - buf := make([]byte, 0, 1024) - formated := r.Time.Format("2006-01-02 15:04:05") + var stringBuilder strings.Builder + + // Append timestamp if !r.Time.IsZero() { - buf = fmt.Append(buf, formated) + stringBuilder.WriteString(r.Time.Format("2006-01-02 15:04:05")) + stringBuilder.WriteByte(' ') } - buf = fmt.Appendf(buf, " %s", colorizeLevel(r.Level)) + // Append colored level + stringBuilder.WriteString(colorizeLevel(r.Level)) - buf = fmt.Appendf(buf, " %s\n", r.Message) + // Append message with attributes + stringBuilder.WriteString(fmt.Sprintf(" %s\n", r.Message)) - _, err := handler.writer.Write(buf) + _, err := io.WriteString(handler.writer, stringBuilder.String()) return err } From dee41b3d310d112fd9852cdd68ae0e5a6272ca23 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 30 Sep 2024 14:28:27 +0300 Subject: [PATCH 71/91] Remove TODO section from README --- README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/README.md b/README.md index aaed58c..7e771f1 100644 --- a/README.md +++ b/README.md @@ -121,8 +121,3 @@ In this case: - Check if there are no running Docker containers that are instances of one of the Docker images in the context directory. - Check if there are no other containers using port `1122`. - -## TODO - -- Improve SFTP copy and package creation speed (~20 minutes for Boost). -- Refactor error handling (use a logging library, etc.) and error messages. From 509d59b239ea679b8d06d221007858680e1fe664 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 30 Sep 2024 14:34:33 +0300 Subject: [PATCH 72/91] filepath.Join use --- modules/bringauto_log/ContextLogger.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/bringauto_log/ContextLogger.go b/modules/bringauto_log/ContextLogger.go index f1d93a6..37fd6d2 100644 --- a/modules/bringauto_log/ContextLogger.go +++ b/modules/bringauto_log/ContextLogger.go @@ -3,6 +3,7 @@ package bringauto_log import ( "bringauto/modules/bringauto_prerequisites" "os" + "path/filepath" ) const ( @@ -72,5 +73,6 @@ func (logger *ContextLogger) CheckPrerequisites(*bringauto_prerequisites.Args) e // Returns writable file for writing logs for specified context of a package. The file must be // closed by the caller. func (logger *ContextLogger) GetFile() (*os.File, error) { - return os.OpenFile(logger.logDirPath + "/" + logger.logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + filepath := filepath.Join(logger.logDirPath, logger.logFileName) + return os.OpenFile(filepath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) } From 373b7054b4337bbdd3ca5544a17d93fba57c0b5f Mon Sep 17 00:00:00 2001 From: Mikulas Date: Mon, 30 Sep 2024 14:45:57 +0300 Subject: [PATCH 73/91] Doc improvements --- modules/bringauto_process/SignalHandler.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/modules/bringauto_process/SignalHandler.go b/modules/bringauto_process/SignalHandler.go index 3ece021..a5333e2 100644 --- a/modules/bringauto_process/SignalHandler.go +++ b/modules/bringauto_process/SignalHandler.go @@ -1,11 +1,12 @@ // Functions for global signal handling // -// The SignalHandlerRegisterSignal() must be called to start handling signal by this module. Then -// SignalHandlerAddHandler() can be called to add handlers to execute when the registered signal is -// received. The method returns function which should be deferred after SignalHandlerAddHandler() -// call by caller. This returned function basically removes handler from execution after signal is -// received. When registered signal is received, all added handlers (and not removed yet) will be -// executed in reverse order and then the program exits with status code 1. +// The SignalHandlerRegisterSignal() function must be called to start handling signals with this +// module. Afterward, SignalHandlerAddHandler() can be used to add handlers that will execute when +// a registered signal is received. The method returns a function which should be deferred +// immediately after the SignalHandlerAddHandler() call by the caller. This returned function +// removes the handler from execution if called before the signal is received. When a registered +// signal is received, all added (and not yet removed) handlers will be executed in reverse order, +// and then the program exits with status code 1. // // Note: Do not use with concurrent programming. Can behave unexpectedly! From 9f7353a5e1547fe01c5f85df0a5932f771b4b0ed Mon Sep 17 00:00:00 2001 From: Mikulas Date: Tue, 1 Oct 2024 13:22:49 +0300 Subject: [PATCH 74/91] Remove not needed error value --- modules/bringauto_process/SignalHandler.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/bringauto_process/SignalHandler.go b/modules/bringauto_process/SignalHandler.go index a5333e2..1063746 100644 --- a/modules/bringauto_process/SignalHandler.go +++ b/modules/bringauto_process/SignalHandler.go @@ -14,7 +14,6 @@ package bringauto_process import ( "bringauto/modules/bringauto_log" - "fmt" "os" "sync" "os/signal" @@ -56,12 +55,11 @@ func SignalHandlerAddHandler(handler func() error) func() { } } -func removeLastHandler() error { +func removeLastHandler() { if len(handlers) == 0 { - return fmt.Errorf("no handler to remove") + return } handlers = handlers[:len(handlers) - 1] - return nil } func executeAllHandlers() { From c6155c5ff0d3bf24bbee726dee2aa9c66de3f727 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Tue, 1 Oct 2024 13:30:14 +0300 Subject: [PATCH 75/91] Add check for nil --- modules/bringauto_config/Config.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/bringauto_config/Config.go b/modules/bringauto_config/Config.go index 628777e..8f44db7 100644 --- a/modules/bringauto_config/Config.go +++ b/modules/bringauto_config/Config.go @@ -123,7 +123,9 @@ func (config *Config) fillBuildStructure(dockerImageName string, platformString tmpPackage := config.Package err = bringauto_prerequisites.Initialize(&tmpPackage) - tmpPackage.PlatformString = *platformString + if platformString != nil { + tmpPackage.PlatformString = *platformString + } if err != nil { panic(err) } From 5928ac729b86873522c6f88b94775e8d3a624cd0 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Tue, 1 Oct 2024 13:47:02 +0300 Subject: [PATCH 76/91] Refactor and optimization --- modules/bringauto_ssh/SFTP.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/modules/bringauto_ssh/SFTP.go b/modules/bringauto_ssh/SFTP.go index 310c573..095e64c 100644 --- a/modules/bringauto_ssh/SFTP.go +++ b/modules/bringauto_ssh/SFTP.go @@ -97,7 +97,7 @@ func (sftpd *SFTP) DownloadDirectory() error { err = os.Remove(localArchivePath) if err != nil { - return fmt.Errorf("cannot remove local dir %s", err) + return fmt.Errorf("cannot remove local archive %s: %s", localArchivePath, err) } return nil @@ -105,16 +105,14 @@ func (sftpd *SFTP) DownloadDirectory() error { func (sftpd *SFTP) copyFile(sftpClient *sftp.Client, remoteFile string, localDir string) error { var err error - _, err = sftpClient.Lstat(remoteFile) + remotePathStat, err := sftpClient.Lstat(remoteFile) if os.IsNotExist(err) { return fmt.Errorf("requested remote file %s does not exist", remoteFile) + } else if err != nil { + return fmt.Errorf("error retrieving %s remote file info: %s", remoteFile, err) } - normalizedLocalDir, _ := normalizePath(localDir) - remotePathStat, err := sftpClient.Lstat(remoteFile) - if err != nil { - return fmt.Errorf("cannot get Lstat if remote %s", remoteFile) - } + normalizedLocalDir, _ := normalizePath(localDir) sourceFile, err := sftpClient.Open(remoteFile) if err != nil { return fmt.Errorf("cannot open file for read - %s,%s", remoteFile, err) From bfdc1742dadf7298ded44741f14159f07d0761fa Mon Sep 17 00:00:00 2001 From: Mikulas Date: Tue, 1 Oct 2024 14:00:02 +0300 Subject: [PATCH 77/91] Optimize checking for existing files in sysroot with map --- modules/bringauto_sysroot/Sysroot.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/modules/bringauto_sysroot/Sysroot.go b/modules/bringauto_sysroot/Sysroot.go index 3cb50fd..1f66e16 100644 --- a/modules/bringauto_sysroot/Sysroot.go +++ b/modules/bringauto_sysroot/Sysroot.go @@ -15,7 +15,7 @@ import ( const ( sysrootDirectoryName = "install_sysroot" - // Constant for number of problematic files which will be printed when trying to ovewrite files + // Constant for number of problematic files which will be printed when trying to overwrite files // in sysroot listFilesCount = 10 ) @@ -68,13 +68,15 @@ func (sysroot *Sysroot) CopyToSysroot(source string) error { // nil error without printing anything. func (sysroot *Sysroot) checkForOverwritingFiles(dirPath string) error { filesToCopy := getExistingFilesInDir(dirPath) - filesInSysroot := getExistingFilesInDir(sysroot.GetSysrootPath()) + filesInSysrootMap := make(map[string]struct{}) + for _, file := range getExistingFilesInDir(sysroot.GetSysrootPath()) { + filesInSysrootMap[file] = struct{}{} + } var intersection []string for _, fileToCopy := range filesToCopy { - for _, fileInSysroot := range filesInSysroot { - if fileToCopy == fileInSysroot { - intersection = append(intersection, fileToCopy) - } + _, exists := filesInSysrootMap[fileToCopy] + if exists { + intersection = append(intersection, fileToCopy) } } if len(intersection) > 0 { From 071657fabcb704f8d1c1a107927fd6637704a205 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Tue, 1 Oct 2024 14:03:59 +0300 Subject: [PATCH 78/91] Add err handling --- modules/bringauto_sysroot/Sysroot.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/bringauto_sysroot/Sysroot.go b/modules/bringauto_sysroot/Sysroot.go index 1f66e16..aa6b6f5 100644 --- a/modules/bringauto_sysroot/Sysroot.go +++ b/modules/bringauto_sysroot/Sysroot.go @@ -161,6 +161,9 @@ func getExistingFilesInDir(dirPath string) []string { var existingFiles []string filepath.WalkDir(dirPath, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } if !d.IsDir() { filePath := strings.TrimPrefix(path, dirPath) _, err := os.Stat(path) From 78b1b22003ec8e66904785d931433a442474ccdc Mon Sep 17 00:00:00 2001 From: Mikulas Date: Tue, 1 Oct 2024 14:22:59 +0300 Subject: [PATCH 79/91] Refactor --- bap-builder/PackageMode.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index 7a6e0a5..7dbe03b 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -129,7 +129,10 @@ func BuildPackage(cmdLine *BuildPackageCmdLineArgs, contextPath string) error { if err != nil { return err } - checkSysrootDirs(platformString) + err = checkSysrootDirs(platformString) + if err != nil { + return err + } buildAll := cmdLine.All if *buildAll { return buildAllPackages(cmdLine, contextPath, platformString) @@ -228,11 +231,11 @@ func buildSinglePackage(cmdLine *BuildPackageCmdLineArgs, contextPath string, pl // addConfigsToDefsMap // Adds all configs in packageJsonPathList to defsMap. func addConfigsToDefsMap(defsMap *ConfigMapType, packageJsonPathList []string) { + logger := bringauto_log.GetLogger() for _, packageJsonPath := range packageJsonPathList { var config bringauto_config.Config err := config.LoadJSONConfig(packageJsonPath) if err != nil { - logger := bringauto_log.GetLogger() logger.Error("Couldn't load JSON config from %s path - %s", packageJsonPath, err) continue } From de42ca7f935a02bdf63549e709e6e56ca91a5bd6 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Tue, 1 Oct 2024 14:25:28 +0300 Subject: [PATCH 80/91] Fix log --- bap-builder/ContextManager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bap-builder/ContextManager.go b/bap-builder/ContextManager.go index 6fcfc79..057a3e0 100644 --- a/bap-builder/ContextManager.go +++ b/bap-builder/ContextManager.go @@ -106,7 +106,7 @@ func (context *ContextManager) getAllDepsJsonPaths(packageJsonPath string) ([]st for _, packageDepJsonPath := range packageDepsJsonPaths { err := depConfig.LoadJSONConfig(packageDepJsonPath) if err != nil { - return []string{}, fmt.Errorf("couldn't load JSON config from %s path - %s", packageJsonPath, err) + return []string{}, fmt.Errorf("couldn't load JSON config from %s path - %s", packageDepJsonPath, err) } if depConfig.Package.IsDebug != config.Package.IsDebug { continue From 1702e3080743999c3d66d83cdaf88cd1cbb7e727 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Thu, 3 Oct 2024 10:36:21 +0300 Subject: [PATCH 81/91] Infinite loop when circular dependency between packages fix --- bap-builder/ContextManager.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/bap-builder/ContextManager.go b/bap-builder/ContextManager.go index 057a3e0..c9d3e1c 100644 --- a/bap-builder/ContextManager.go +++ b/bap-builder/ContextManager.go @@ -90,12 +90,13 @@ func (context *ContextManager) GetPackageJsonDefPaths(packageName string) ([]str // getAllDepsJsonPaths // returns all json defintions paths recursively for given package specified by its json definition path -func (context *ContextManager) getAllDepsJsonPaths(packageJsonPath string) ([]string, error) { +func (context *ContextManager) getAllDepsJsonPaths(packageJsonPath string, visited map[string]struct{}) ([]string, error) { var config bringauto_config.Config err := config.LoadJSONConfig(packageJsonPath) if err != nil { return []string{}, fmt.Errorf("couldn't load JSON config from %s path - %s", packageJsonPath, err) } + visited[packageJsonPath] = struct{}{} var jsonPathListWithDeps []string for _, packageDep := range config.DependsOn { packageDepsJsonPaths, err := context.GetPackageJsonDefPaths(packageDep) @@ -104,6 +105,10 @@ func (context *ContextManager) getAllDepsJsonPaths(packageJsonPath string) ([]st } var depConfig bringauto_config.Config for _, packageDepJsonPath := range packageDepsJsonPaths { + _, packageVisited := visited[packageDepJsonPath] + if packageVisited { + continue + } err := depConfig.LoadJSONConfig(packageDepJsonPath) if err != nil { return []string{}, fmt.Errorf("couldn't load JSON config from %s path - %s", packageDepJsonPath, err) @@ -112,7 +117,7 @@ func (context *ContextManager) getAllDepsJsonPaths(packageJsonPath string) ([]st continue } jsonPathListWithDeps = append(jsonPathListWithDeps, packageDepJsonPath) - jsonPathListWithDepsTmp, err := context.getAllDepsJsonPaths(packageDepJsonPath) + jsonPathListWithDepsTmp, err := context.getAllDepsJsonPaths(packageDepJsonPath, visited) if err != nil { return []string{}, err } @@ -131,8 +136,9 @@ func (context *ContextManager) GetPackageWithDepsJsonDefPaths(packageName string return []string{}, fmt.Errorf("cannot get config paths for package '%s' - %s", packageName, err) } var packageDeps []string + visitedPackages := make(map[string]struct{}) for _, packageDef := range packageDefs { - packageDepsTmp, err := context.getAllDepsJsonPaths(packageDef) + packageDepsTmp, err := context.getAllDepsJsonPaths(packageDef, visitedPackages) if err != nil { return []string{}, err } From 0a8fdc39cdcdf83aae69a444b332f97c774b0a07 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Thu, 3 Oct 2024 10:36:46 +0300 Subject: [PATCH 82/91] Add check for circular dependencies between packages --- bap-builder/PackageMode.go | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index 7dbe03b..e5759d4 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -40,13 +40,31 @@ func removeDuplicates(configList *[]*bringauto_config.Config) []*bringauto_confi return newConfigList } -func (list *buildDepList) TopologicalSort(buildMap ConfigMapType) []*bringauto_config.Config { +// checkForCircularDependency +// Checks for circular dependency in defsMap. If there is one, returns error with message +// and problematic packages, else returns nil. +func checkForCircularDependency(dependsMap map[string]*map[string]bool) error { + for packageName, depsMap := range dependsMap { + for depPackageName, _ := range *depsMap { + if (*dependsMap[depPackageName])[packageName] { + return fmt.Errorf("circular dependency between packages %s and %s", packageName, depPackageName) + } + } + } + return nil +} + +func (list *buildDepList) TopologicalSort(buildMap ConfigMapType) ([]*bringauto_config.Config, error) { // Map represents 'PackageName: []DependsOnPackageNames' var dependsMap map[string]*map[string]bool var allDependencies map[string]bool dependsMap, allDependencies = list.createDependsMap(&buildMap) + err := checkForCircularDependency(dependsMap) + if err != nil { + return []*bringauto_config.Config{}, err + } dependsMapCopy := make(map[string]*map[string]bool, len(dependsMap)) for key, value := range dependsMap { @@ -76,7 +94,7 @@ func (list *buildDepList) TopologicalSort(buildMap ConfigMapType) []*bringauto_c sortedDependenciesConfig = append(sortedDependenciesConfig, buildMap[packageName]...) } - return removeDuplicates(&sortedDependenciesConfig) + return removeDuplicates(&sortedDependenciesConfig), nil } func (list *buildDepList) createDependsMap(buildMap *ConfigMapType) (dependsMapType, allDependenciesType) { @@ -157,7 +175,10 @@ func buildAllPackages(cmdLine *BuildPackageCmdLineArgs, contextPath string, plat addConfigsToDefsMap(&defsMap, packageJsonPathList) } depsList := buildDepList{} - configList := depsList.TopologicalSort(defsMap) + configList, err := depsList.TopologicalSort(defsMap) + if err != nil { + return err + } logger := bringauto_log.GetLogger() @@ -201,7 +222,10 @@ func buildSinglePackage(cmdLine *BuildPackageCmdLineArgs, contextPath string, pl defsMap := make(ConfigMapType) addConfigsToDefsMap(&defsMap, packageJsonPathList) depList := buildDepList{} - configList = depList.TopologicalSort(defsMap) + configList, err = depList.TopologicalSort(defsMap) + if err != nil { + return err + } } else { packageJsonPathList, err := contextManager.GetPackageJsonDefPaths(packageName) if err != nil { From 6c0dca8f85a36c5ff798ec3865851f06211dd867 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Thu, 3 Oct 2024 11:31:33 +0300 Subject: [PATCH 83/91] Better error handling --- modules/bringauto_ssh/SFTP.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/modules/bringauto_ssh/SFTP.go b/modules/bringauto_ssh/SFTP.go index 095e64c..5bd0410 100644 --- a/modules/bringauto_ssh/SFTP.go +++ b/modules/bringauto_ssh/SFTP.go @@ -122,31 +122,33 @@ func (sftpd *SFTP) copyFile(sftpClient *sftp.Client, remoteFile string, localDir return err } - copyIOFile(sourceFile, destFile) - - return nil + return copyIOFile(sourceFile, destFile) } -func copyIOFile(sourceFile *sftp.File, destFile *os.File) { +func copyIOFile(sourceFile *sftp.File, destFile *os.File) error { sourceFileBuff := bufio.NewReaderSize(sourceFile, bufferSize) destFileBuff := bufio.NewWriterSize(destFile, bufferSize) var err error _, err = io.Copy(destFileBuff, sourceFileBuff) if err != nil { - panic(fmt.Errorf("cannot copy remote IO files")) + return fmt.Errorf("cannot copy remote IO files: %s", err) } - _ = destFileBuff.Flush() + err = destFileBuff.Flush() + if err != nil { + return fmt.Errorf("cannot flush destination buffer: %s", err) + } err = destFile.Close() if err != nil { - panic(fmt.Errorf("cannot close destFile: %s", err)) + return fmt.Errorf("cannot close destination file: %s", err) } err = sourceFile.Close() if err != nil { - panic(fmt.Errorf("cannot close sourceFile: %s", err)) + return fmt.Errorf("cannot close source file: %s", err) } + return nil } // normalizePath From ac096684f50d375eddc3d8811aac8ddc3bd714b9 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Thu, 3 Oct 2024 13:13:10 +0300 Subject: [PATCH 84/91] Add check for package context directory consistency --- bap-builder/PackageMode.go | 42 ++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index e5759d4..7752108 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -1,17 +1,19 @@ package main import ( - "bringauto/modules/bringauto_log" "bringauto/modules/bringauto_build" "bringauto/modules/bringauto_config" - "bringauto/modules/bringauto_package" "bringauto/modules/bringauto_docker" - "bringauto/modules/bringauto_ssh" + "bringauto/modules/bringauto_log" + "bringauto/modules/bringauto_package" "bringauto/modules/bringauto_prerequisites" + "bringauto/modules/bringauto_process" "bringauto/modules/bringauto_repository" + "bringauto/modules/bringauto_ssh" "bringauto/modules/bringauto_sysroot" - "bringauto/modules/bringauto_process" "fmt" + "io/fs" + "path/filepath" "strconv" ) @@ -140,6 +142,34 @@ func (list *buildDepList) sortDependencies(rootName string, dependsMap *map[stri return &sorted } + +// checkContextDirConsistency +// Checks if all directories in contextPath have same name as Package names from JSON definitions +// inside this directory. If not, returns error with description, else returns nil. Also returns error +// if the Package JSON definition can't be loaded. +func checkContextDirConsistency(contextPath string) error { + packageContextPath := filepath.Join(contextPath, PackageDirectoryNameConst) + err := filepath.WalkDir(packageContextPath, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + var config bringauto_config.Config + err = config.LoadJSONConfig(path) + if err != nil { + return fmt.Errorf("couldn't load JSON config from %s path", path) + } + dirName := filepath.Base(filepath.Dir(path)) + if config.Package.Name != dirName { + return fmt.Errorf("directory name (%s) is different from package name (%s)", dirName, config.Package.Name) + } + } + return nil + }) + + return err +} + // BuildPackage // process Package mode of the program func BuildPackage(cmdLine *BuildPackageCmdLineArgs, contextPath string) error { @@ -151,6 +181,10 @@ func BuildPackage(cmdLine *BuildPackageCmdLineArgs, contextPath string) error { if err != nil { return err } + err = checkContextDirConsistency(contextPath) + if err != nil { + return fmt.Errorf("package context directory consistency check failed: %s", err) + } buildAll := cmdLine.All if *buildAll { return buildAllPackages(cmdLine, contextPath, platformString) From 1b76148ccf6e744fa350bf7bd487834540b054c1 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Thu, 3 Oct 2024 15:55:28 +0300 Subject: [PATCH 85/91] Change SignalHandlerAddHandler semantics The function now returns function which also executes handler. Updated function calls and removed not executing handler when errors occurs during build. --- bap-builder/PackageMode.go | 14 ++++++++++---- modules/bringauto_build/Build.go | 1 - modules/bringauto_process/SignalHandler.go | 18 +++++++++++------- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index 7752108..2cd53c2 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -322,6 +322,8 @@ func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_bu if err != nil { return err } + var removeHandler func() + logger := bringauto_log.GetLogger() for _, buildConfig := range *build { @@ -342,27 +344,31 @@ func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_bu logger.InfoIndent("Copying to Git repository") - removeHandler := bringauto_process.SignalHandlerAddHandler(buildConfig.CleanUp) + removeHandler = bringauto_process.SignalHandlerAddHandler(buildConfig.CleanUp) err = repo.CopyToRepository(*buildConfig.Package, buildConfig.GetLocalInstallDirPath()) if err != nil { - return err + break } logger.InfoIndent("Copying to local sysroot directory") err = sysroot.CopyToSysroot(buildConfig.GetLocalInstallDirPath()) if err != nil { - return err + break } err = buildConfig.CleanUp() removeHandler() + removeHandler = nil if err != nil { return err } logger.InfoIndent("Build OK") } - return nil + if removeHandler != nil { + removeHandler() + } + return err } // determinePlatformString diff --git a/modules/bringauto_build/Build.go b/modules/bringauto_build/Build.go index b7876f2..298a654 100644 --- a/modules/bringauto_build/Build.go +++ b/modules/bringauto_build/Build.go @@ -140,7 +140,6 @@ func (build *Build) RunBuild() error { if err != nil { return err } - defer build.stopAndRemoveContainer() err = shellEvaluator.RunOverSSH(*build.SSHCredentials) if err != nil { diff --git a/modules/bringauto_process/SignalHandler.go b/modules/bringauto_process/SignalHandler.go index 1063746..50232f4 100644 --- a/modules/bringauto_process/SignalHandler.go +++ b/modules/bringauto_process/SignalHandler.go @@ -2,11 +2,10 @@ // // The SignalHandlerRegisterSignal() function must be called to start handling signals with this // module. Afterward, SignalHandlerAddHandler() can be used to add handlers that will execute when -// a registered signal is received. The method returns a function which should be deferred -// immediately after the SignalHandlerAddHandler() call by the caller. This returned function -// removes the handler from execution if called before the signal is received. When a registered -// signal is received, all added (and not yet removed) handlers will be executed in reverse order, -// and then the program exits with status code 1. +// a registered signal is received. The method returns a function which should be deffered or +// called later in program to execute handler and remove it from handling by this module. When a +// registered signal is received, all added (and not yet removed) handlers will be executed in +// reverse order and then the program exits with status code 1. // // Note: Do not use with concurrent programming. Can behave unexpectedly! @@ -40,8 +39,9 @@ func SignalHandlerRegisterSignal(sig ...os.Signal) { } // SignalHandlerAddHandler -// Adds handler for execution after signal is received by bringauto_process package. Returns handler -// remover which should be deferred by caller. So it should be used as this: +// Adds handler for execution after signal is received by bringauto_process package. Returns +// function, which executes handler and removes it from handling by bringauto_process module. +// The returned function should be deferred by caller. It should be used as this: // handlerRemover := SignalHandlerAddHandler(my_handler) // defer handlerRemover() func SignalHandlerAddHandler(handler func() error) func() { @@ -51,6 +51,10 @@ func SignalHandlerAddHandler(handler func() error) func() { return func() { lock.Lock() defer lock.Unlock() + err := handler() + if err != nil { + bringauto_log.GetLogger().Error("Handler returned error - %s", err) + } removeLastHandler() } } From f31dab3395c632a90c73872802b33defec75b1cc Mon Sep 17 00:00:00 2001 From: Mikulas Date: Thu, 3 Oct 2024 17:55:00 +0300 Subject: [PATCH 86/91] Address also indirect circular dependencies in check --- bap-builder/PackageMode.go | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index 2cd53c2..13aa273 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -46,16 +46,44 @@ func removeDuplicates(configList *[]*bringauto_config.Config) []*bringauto_confi // Checks for circular dependency in defsMap. If there is one, returns error with message // and problematic packages, else returns nil. func checkForCircularDependency(dependsMap map[string]*map[string]bool) error { - for packageName, depsMap := range dependsMap { - for depPackageName, _ := range *depsMap { - if (*dependsMap[depPackageName])[packageName] { - return fmt.Errorf("circular dependency between packages %s and %s", packageName, depPackageName) - } + visited := make(map[string]bool) + + for packageName := range dependsMap { + cycleDetected, cycleString := detectCycle(packageName, dependsMap, visited) + if cycleDetected { + return fmt.Errorf("circular dependency detected - %s", packageName + " -> " + cycleString) + } + // Clearing recursion stack after one path through graph was checked + for visitedPackage := range visited { + visited[visitedPackage] = false } } return nil } +// detectCycle +// Detects cycle between package dependencies in one path through graph. visited is current +// recursion stack and dependsMap is whole graph representation. packageName is root node where +// cycle detection should start. +func detectCycle(packageName string, dependsMap map[string]*map[string]bool, visited map[string]bool) (bool, string) { + visited[packageName] = true + depsMap, found := dependsMap[packageName] + if found { + for depPackageName := range *depsMap { + if visited[depPackageName] { + return true, depPackageName + } else { + cycleDetected, cycleString := detectCycle(depPackageName, dependsMap, visited) + if cycleDetected { + return cycleDetected, depPackageName + " -> " + cycleString + } + } + } + } + visited[packageName] = false + return false, "" +} + func (list *buildDepList) TopologicalSort(buildMap ConfigMapType) ([]*bringauto_config.Config, error) { // Map represents 'PackageName: []DependsOnPackageNames' From 531128f28aed960d243b8d7bdb760c249775677c Mon Sep 17 00:00:00 2001 From: Mikulas Date: Thu, 3 Oct 2024 18:53:49 +0300 Subject: [PATCH 87/91] Forced context directory consistency --- .../fleet-http-client-shared/fleet_http_client_debug.json | 2 +- .../fleet-http-client-shared/fleet_http_client_release.json | 2 +- .../package/fleet-protocol-cpp/fleet_protocol_cpp_debug.json | 2 +- .../package/fleet-protocol-cpp/fleet_protocol_cpp_release.json | 2 +- .../fleet-protocol-internal-client/internal_client_debug.json | 2 +- .../fleet-protocol-internal-client/internal_client_release.json | 2 +- example/package/libosmium/libosmium_debug.json | 2 +- example/package/libosmium/libosmium_release.json | 2 +- .../nlohmann_json_debug.json} | 0 .../nlohmann_json_release.json} | 0 example/package/pahomqttcpp/README.md | 0 11 files changed, 8 insertions(+), 8 deletions(-) rename example/package/{nlohmannjson/nlohmannjson_debug.json => nlohmann-json/nlohmann_json_debug.json} (100%) rename example/package/{nlohmannjson/nlohmannjson_release.json => nlohmann-json/nlohmann_json_release.json} (100%) delete mode 100644 example/package/pahomqttcpp/README.md diff --git a/example/package/fleet-http-client-shared/fleet_http_client_debug.json b/example/package/fleet-http-client-shared/fleet_http_client_debug.json index 13a66fb..a18ef71 100644 --- a/example/package/fleet-http-client-shared/fleet_http_client_debug.json +++ b/example/package/fleet-http-client-shared/fleet_http_client_debug.json @@ -4,7 +4,7 @@ "zlib", "boost", "fleet-protocol-interface", - "fleet-protocol-cxx-helpers-static", + "fleet-protocol-cpp", "cpprestsdk" ], "Git": { diff --git a/example/package/fleet-http-client-shared/fleet_http_client_release.json b/example/package/fleet-http-client-shared/fleet_http_client_release.json index 3a40061..364e8a4 100644 --- a/example/package/fleet-http-client-shared/fleet_http_client_release.json +++ b/example/package/fleet-http-client-shared/fleet_http_client_release.json @@ -4,7 +4,7 @@ "zlib", "boost", "fleet-protocol-interface", - "fleet-protocol-cxx-helpers-static", + "fleet-protocol-cpp", "cpprestsdk" ], "Git": { diff --git a/example/package/fleet-protocol-cpp/fleet_protocol_cpp_debug.json b/example/package/fleet-protocol-cpp/fleet_protocol_cpp_debug.json index 101d7e9..201ce42 100644 --- a/example/package/fleet-protocol-cpp/fleet_protocol_cpp_debug.json +++ b/example/package/fleet-protocol-cpp/fleet_protocol_cpp_debug.json @@ -19,7 +19,7 @@ } }, "Package": { - "Name": "fleet-protocol-cxx-helpers-static", + "Name": "fleet-protocol-cpp", "VersionTag": "v1.1.1", "PlatformString": { "Mode": "auto" diff --git a/example/package/fleet-protocol-cpp/fleet_protocol_cpp_release.json b/example/package/fleet-protocol-cpp/fleet_protocol_cpp_release.json index de12659..82ef37a 100644 --- a/example/package/fleet-protocol-cpp/fleet_protocol_cpp_release.json +++ b/example/package/fleet-protocol-cpp/fleet_protocol_cpp_release.json @@ -19,7 +19,7 @@ } }, "Package": { - "Name": "fleet-protocol-cxx-helpers-static", + "Name": "fleet-protocol-cpp", "VersionTag": "v1.1.1", "PlatformString": { "Mode": "auto" diff --git a/example/package/fleet-protocol-internal-client/internal_client_debug.json b/example/package/fleet-protocol-internal-client/internal_client_debug.json index 77595c3..2657bdf 100644 --- a/example/package/fleet-protocol-internal-client/internal_client_debug.json +++ b/example/package/fleet-protocol-internal-client/internal_client_debug.json @@ -20,7 +20,7 @@ } }, "Package": { - "Name": "internal-client", + "Name": "fleet-protocol-internal-client", "VersionTag": "v1.1.1", "PlatformString": { "Mode": "auto" diff --git a/example/package/fleet-protocol-internal-client/internal_client_release.json b/example/package/fleet-protocol-internal-client/internal_client_release.json index 7f57d5b..ccab0bb 100644 --- a/example/package/fleet-protocol-internal-client/internal_client_release.json +++ b/example/package/fleet-protocol-internal-client/internal_client_release.json @@ -20,7 +20,7 @@ } }, "Package": { - "Name": "internal-client", + "Name": "fleet-protocol-internal-client", "VersionTag": "v1.1.1", "PlatformString": { "Mode": "auto" diff --git a/example/package/libosmium/libosmium_debug.json b/example/package/libosmium/libosmium_debug.json index ca4922e..85e4573 100644 --- a/example/package/libosmium/libosmium_debug.json +++ b/example/package/libosmium/libosmium_debug.json @@ -23,7 +23,7 @@ } }, "Package": { - "Name": "osmium", + "Name": "libosmium", "VersionTag": "v2.17.3", "PlatformString": { "Mode": "auto" diff --git a/example/package/libosmium/libosmium_release.json b/example/package/libosmium/libosmium_release.json index 84de50e..1be0ed6 100644 --- a/example/package/libosmium/libosmium_release.json +++ b/example/package/libosmium/libosmium_release.json @@ -23,7 +23,7 @@ } }, "Package": { - "Name": "osmium", + "Name": "libosmium", "VersionTag": "v2.17.3", "PlatformString": { "Mode": "auto" diff --git a/example/package/nlohmannjson/nlohmannjson_debug.json b/example/package/nlohmann-json/nlohmann_json_debug.json similarity index 100% rename from example/package/nlohmannjson/nlohmannjson_debug.json rename to example/package/nlohmann-json/nlohmann_json_debug.json diff --git a/example/package/nlohmannjson/nlohmannjson_release.json b/example/package/nlohmann-json/nlohmann_json_release.json similarity index 100% rename from example/package/nlohmannjson/nlohmannjson_release.json rename to example/package/nlohmann-json/nlohmann_json_release.json diff --git a/example/package/pahomqttcpp/README.md b/example/package/pahomqttcpp/README.md deleted file mode 100644 index e69de29..0000000 From e2369cfcac829ab555d9ef024c48f5c0a435ce84 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Thu, 3 Oct 2024 19:21:42 +0300 Subject: [PATCH 88/91] Move AddHandler to correct location local install directory is created earlier --- bap-builder/PackageMode.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bap-builder/PackageMode.go b/bap-builder/PackageMode.go index 13aa273..380f70a 100644 --- a/bap-builder/PackageMode.go +++ b/bap-builder/PackageMode.go @@ -365,6 +365,7 @@ func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_bu buildConfig.SetSysroot(&sysroot) logger.InfoIndent("Run build inside container") + removeHandler = bringauto_process.SignalHandlerAddHandler(buildConfig.CleanUp) err = buildConfig.RunBuild() if err != nil { return err @@ -372,8 +373,6 @@ func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_bu logger.InfoIndent("Copying to Git repository") - removeHandler = bringauto_process.SignalHandlerAddHandler(buildConfig.CleanUp) - err = repo.CopyToRepository(*buildConfig.Package, buildConfig.GetLocalInstallDirPath()) if err != nil { break @@ -385,7 +384,6 @@ func buildAndCopyPackage(cmdLine *BuildPackageCmdLineArgs, build *[]bringauto_bu break } - err = buildConfig.CleanUp() removeHandler() removeHandler = nil if err != nil { From b55a7d76e8aed9b38df1f2fd1278744407c30226 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Thu, 3 Oct 2024 19:23:05 +0300 Subject: [PATCH 89/91] Add wait for docker run command when CTRL+C received --- modules/bringauto_build/Build.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/bringauto_build/Build.go b/modules/bringauto_build/Build.go index 298a654..d9705a7 100644 --- a/modules/bringauto_build/Build.go +++ b/modules/bringauto_build/Build.go @@ -13,6 +13,7 @@ import ( "fmt" "os" "path/filepath" + "time" ) type Build struct { @@ -133,7 +134,11 @@ func (build *Build) RunBuild() error { } dockerRun := (*bringauto_docker.DockerRun)(build.Docker) - removeHandler := bringauto_process.SignalHandlerAddHandler(build.stopAndRemoveContainer) + removeHandler := bringauto_process.SignalHandlerAddHandler(func() error { + // Waiting for docker run command to get container id + time.Sleep(300 * time.Millisecond) + return build.stopAndRemoveContainer() + }) defer removeHandler() err = dockerRun.Run() From b07b8c16ec5c6a7dd8c3ab2054af9dbfa76fe547 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Thu, 3 Oct 2024 19:32:44 +0300 Subject: [PATCH 90/91] Add CTRL+C handling to PlatformString --- modules/bringauto_package/PlatformString.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/modules/bringauto_package/PlatformString.go b/modules/bringauto_package/PlatformString.go index 91a32ed..d26f8f0 100644 --- a/modules/bringauto_package/PlatformString.go +++ b/modules/bringauto_package/PlatformString.go @@ -4,9 +4,11 @@ import ( "bringauto/modules/bringauto_docker" "bringauto/modules/bringauto_prerequisites" "bringauto/modules/bringauto_ssh" + "bringauto/modules/bringauto_process" "fmt" "regexp" "strings" + "time" ) // PlatformStringMode is a fill-up mode of the platform-string. @@ -119,16 +121,20 @@ func (pstr *PlatformString) determinePlatformString(credentials bringauto_ssh.SS } dockerRun := (*bringauto_docker.DockerRun)(docker) + removeHandler := bringauto_process.SignalHandlerAddHandler(func() error { + dockerStop := (*bringauto_docker.DockerStop)(docker) + dockerRm := (*bringauto_docker.DockerRm)(docker) + // Waiting for docker run command to get container id + time.Sleep(200 * time.Millisecond) + dockerStop.Stop() + return dockerRm.RemoveContainer() + }) + defer removeHandler() + err := dockerRun.Run() if err != nil { return err } - defer func() { - dockerStop := (*bringauto_docker.DockerStop)(docker) - dockerRm := (*bringauto_docker.DockerRm)(docker) - dockerStop.Stop() - dockerRm.RemoveContainer() - }() pstr.String.DistroName = getDistributionName(credentials) pstr.String.DistroRelease = getReleaseVersion(credentials) From d7059def5672c96d59913424f3e674958c6a0a96 Mon Sep 17 00:00:00 2001 From: Mikulas Date: Wed, 16 Oct 2024 10:21:30 +0300 Subject: [PATCH 91/91] Typos and documentation --- bap-builder/DockerMode.go | 5 ++--- modules/bringauto_docker/DockerTools.go | 3 +++ modules/bringauto_process/SignalHandler.go | 8 ++++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/bap-builder/DockerMode.go b/bap-builder/DockerMode.go index 53488a6..b0a3949 100644 --- a/bap-builder/DockerMode.go +++ b/bap-builder/DockerMode.go @@ -22,8 +22,7 @@ func BuildDockerImage(cmdLine *BuildImageCmdLineArgs, contextPath string) error if err != nil { return err } - buildSingleDockerImage(*cmdLine.Name, dockerfilePath) - return nil + return buildSingleDockerImage(*cmdLine.Name, dockerfilePath) } // buildAllDockerImages @@ -42,7 +41,7 @@ func buildAllDockerImages(contextManager ContextManager) error { logger.Warn("Bug: multiple Dockerfile present for same image name %s", imageName) continue } - buildSingleDockerImage(imageName, dockerfilePath[0]) + return buildSingleDockerImage(imageName, dockerfilePath[0]) } return nil } diff --git a/modules/bringauto_docker/DockerTools.go b/modules/bringauto_docker/DockerTools.go index ad2fe1e..a8473c8 100644 --- a/modules/bringauto_docker/DockerTools.go +++ b/modules/bringauto_docker/DockerTools.go @@ -8,6 +8,9 @@ import ( "strconv" ) +// IsDefaultPortAvailable +// Returns true if default port for docker is available, else returns false. +// When false is returned, the error contains message from the docker command. func IsDefaultPortAvailable() (bool, error) { var outBuff, errBuff bytes.Buffer diff --git a/modules/bringauto_process/SignalHandler.go b/modules/bringauto_process/SignalHandler.go index 50232f4..f751938 100644 --- a/modules/bringauto_process/SignalHandler.go +++ b/modules/bringauto_process/SignalHandler.go @@ -68,9 +68,9 @@ func removeLastHandler() { func executeAllHandlers() { for i := len(handlers)-1; i >= 0; i-- { - err := handlers[i]() - if err != nil { - bringauto_log.GetLogger().Error("Handler returned error - %s", err) - } + err := handlers[i]() + if err != nil { + bringauto_log.GetLogger().Error("Handler returned error - %s", err) + } } }