Skip to content

Commit e29cba2

Browse files
author
Federico Fissore
committed
Parallel compiling: arduino-builder will leverage your multi core pc. If
-jobs X param is specified and X is greater than 0, arduino-builder won't spawn more than X processes. Fixes #17 Signed-off-by: Federico Fissore <f.fissore@arduino.cc>
1 parent 08f5cf3 commit e29cba2

File tree

4 files changed

+85
-5
lines changed

4 files changed

+85
-5
lines changed

‎main.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ import (
4343
"io/ioutil"
4444
"os"
4545
"os/exec"
46+
"runtime"
4647
"strings"
4748
"syscall"
4849
)
@@ -74,6 +75,7 @@ const FLAG_LOGGER_HUMAN = "human"
7475
const FLAG_LOGGER_MACHINE = "machine"
7576
const FLAG_VERSION = "version"
7677
const FLAG_VID_PID = "vid-pid"
78+
const FLAG_JOBS = "jobs"
7779

7880
type slice []string
7981

@@ -116,6 +118,7 @@ var warningsLevelFlag *string
116118
var loggerFlag *string
117119
var versionFlag *bool
118120
var vidPidFlag *string
121+
var jobsFlag *int
119122

120123
func init() {
121124
compileFlag = flag.Bool(FLAG_ACTION_COMPILE, false, "compiles the given sketch")
@@ -137,6 +140,7 @@ func init() {
137140
loggerFlag = flag.String(FLAG_LOGGER, FLAG_LOGGER_HUMAN, "Sets type of logger. Available values are '"+FLAG_LOGGER_HUMAN+"', '"+FLAG_LOGGER_MACHINE+"'")
138141
versionFlag = flag.Bool(FLAG_VERSION, false, "prints version and exits")
139142
vidPidFlag = flag.String(FLAG_VID_PID, "", "specify to use vid/pid specific build properties, as defined in boards.txt")
143+
jobsFlag = flag.Int(FLAG_JOBS, 0, "specify how many concurrent gcc processes should run at the same time. Defaults to the number of available cores on the running machine")
140144
}
141145

142146
func main() {
@@ -152,6 +156,12 @@ func main() {
152156
return
153157
}
154158

159+
if *jobsFlag > 0 {
160+
runtime.GOMAXPROCS(*jobsFlag)
161+
} else {
162+
runtime.GOMAXPROCS(runtime.NumCPU())
163+
}
164+
155165
context := make(map[string]interface{})
156166

157167
buildOptions := make(map[string]string)

‎src/arduino.cc/builder/builder_utils/utils.go

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ import (
4040
"os/exec"
4141
"path/filepath"
4242
"strings"
43+
"sync"
4344
)
4445

4546
func CompileFilesRecursive(objectFiles []string, sourcePath string, buildPath string, buildProperties map[string]string, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) {
@@ -116,15 +117,47 @@ func findFilesInFolder(sourcePath string, extension string, recurse bool) ([]str
116117
}
117118

118119
func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties map[string]string, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) {
120+
if len(sources) == 0 {
121+
return objectFiles, nil
122+
}
123+
objectFilesChan := make(chan string)
124+
errorsChan := make(chan error)
125+
doneChan := make(chan struct{})
126+
127+
var wg sync.WaitGroup
128+
wg.Add(len(sources))
129+
119130
for _, source := range sources {
120-
objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger)
121-
if err != nil {
131+
go func(source string) {
132+
defer wg.Done()
133+
objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger)
134+
if err != nil {
135+
errorsChan <- err
136+
} else {
137+
objectFilesChan <- objectFile
138+
}
139+
}(source)
140+
}
141+
142+
go func() {
143+
wg.Wait()
144+
doneChan <- struct{}{}
145+
}()
146+
147+
for {
148+
select {
149+
case objectFile := <-objectFilesChan:
150+
objectFiles = append(objectFiles, objectFile)
151+
case err := <-errorsChan:
122152
return nil, utils.WrapError(err)
153+
case <-doneChan:
154+
close(objectFilesChan)
155+
for objectFile := range objectFilesChan {
156+
objectFiles = append(objectFiles, objectFile)
157+
}
158+
return objectFiles, nil
123159
}
124-
125-
objectFiles = append(objectFiles, objectFile)
126160
}
127-
return objectFiles, nil
128161
}
129162

130163
func compileFileWithRecipe(sourcePath string, source string, buildPath string, buildProperties map[string]string, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) (string, error) {

‎src/arduino.cc/builder/test/builder_test.go

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,13 @@ import (
3636
"os"
3737
"os/exec"
3838
"path/filepath"
39+
"runtime"
3940
"testing"
4041
)
4142

4243
func TestBuilderEmptySketch(t *testing.T) {
4344
DownloadCoresAndToolsAndLibraries(t)
45+
runtime.GOMAXPROCS(runtime.NumCPU())
4446

4547
context := make(map[string]interface{})
4648

@@ -75,6 +77,7 @@ func TestBuilderEmptySketch(t *testing.T) {
7577

7678
func TestBuilderBridge(t *testing.T) {
7779
DownloadCoresAndToolsAndLibraries(t)
80+
runtime.GOMAXPROCS(runtime.NumCPU())
7881

7982
context := make(map[string]interface{})
8083

@@ -110,6 +113,7 @@ func TestBuilderBridge(t *testing.T) {
110113

111114
func TestBuilderSketchWithConfig(t *testing.T) {
112115
DownloadCoresAndToolsAndLibraries(t)
116+
runtime.GOMAXPROCS(runtime.NumCPU())
113117

114118
context := make(map[string]interface{})
115119

@@ -144,6 +148,7 @@ func TestBuilderSketchWithConfig(t *testing.T) {
144148

145149
func TestBuilderBridgeTwice(t *testing.T) {
146150
DownloadCoresAndToolsAndLibraries(t)
151+
runtime.GOMAXPROCS(runtime.NumCPU())
147152

148153
context := make(map[string]interface{})
149154

@@ -182,6 +187,7 @@ func TestBuilderBridgeTwice(t *testing.T) {
182187

183188
func TestBuilderBridgeSAM(t *testing.T) {
184189
DownloadCoresAndToolsAndLibraries(t)
190+
runtime.GOMAXPROCS(runtime.NumCPU())
185191

186192
context := make(map[string]interface{})
187193

@@ -226,6 +232,7 @@ func TestBuilderBridgeSAM(t *testing.T) {
226232

227233
func TestBuilderBridgeRedBearLab(t *testing.T) {
228234
DownloadCoresAndToolsAndLibraries(t)
235+
runtime.GOMAXPROCS(runtime.NumCPU())
229236

230237
context := make(map[string]interface{})
231238

@@ -260,6 +267,7 @@ func TestBuilderBridgeRedBearLab(t *testing.T) {
260267

261268
func TestBuilderSketchNoFunctions(t *testing.T) {
262269
DownloadCoresAndToolsAndLibraries(t)
270+
runtime.GOMAXPROCS(runtime.NumCPU())
263271

264272
context := make(map[string]interface{})
265273

@@ -281,6 +289,7 @@ func TestBuilderSketchNoFunctions(t *testing.T) {
281289

282290
func TestBuilderSketchWithBackup(t *testing.T) {
283291
DownloadCoresAndToolsAndLibraries(t)
292+
runtime.GOMAXPROCS(runtime.NumCPU())
284293

285294
context := make(map[string]interface{})
286295

@@ -302,6 +311,7 @@ func TestBuilderSketchWithBackup(t *testing.T) {
302311

303312
func TestBuilderSketchWithOldLib(t *testing.T) {
304313
DownloadCoresAndToolsAndLibraries(t)
314+
runtime.GOMAXPROCS(runtime.NumCPU())
305315

306316
context := make(map[string]interface{})
307317

@@ -323,6 +333,7 @@ func TestBuilderSketchWithOldLib(t *testing.T) {
323333

324334
func TestBuilderSketchWithSubfolders(t *testing.T) {
325335
DownloadCoresAndToolsAndLibraries(t)
336+
runtime.GOMAXPROCS(runtime.NumCPU())
326337

327338
context := make(map[string]interface{})
328339

@@ -341,3 +352,26 @@ func TestBuilderSketchWithSubfolders(t *testing.T) {
341352
err := command.Run(context)
342353
NoError(t, err)
343354
}
355+
356+
func TestBuilderSketchWithSyntaxError(t *testing.T) {
357+
DownloadCoresAndToolsAndLibraries(t)
358+
runtime.GOMAXPROCS(runtime.NumCPU())
359+
360+
context := make(map[string]interface{})
361+
362+
buildPath := SetupBuildPath(t, context)
363+
defer os.RemoveAll(buildPath)
364+
365+
context[constants.CTX_HARDWARE_FOLDERS] = []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"}
366+
context[constants.CTX_TOOLS_FOLDERS] = []string{"downloaded_tools"}
367+
context[constants.CTX_FQBN] = "arduino:avr:uno"
368+
context[constants.CTX_SKETCH_LOCATION] = filepath.Join("sketch_with_syntax_error", "sketch.ino")
369+
context[constants.CTX_BUILT_IN_LIBRARIES_FOLDERS] = []string{"downloaded_libraries"}
370+
context[constants.CTX_OTHER_LIBRARIES_FOLDERS] = []string{"libraries"}
371+
context[constants.CTX_BUILD_PROPERTIES_RUNTIME_IDE_VERSION] = "10600"
372+
context[constants.CTX_VERBOSE] = true
373+
374+
command := builder.Builder{}
375+
err := command.Run(context)
376+
require.Error(t, err)
377+
}

‎src/arduino.cc/builder/test/sketch_with_syntax_error/sketch.ino

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
void setup() {
2+
3+
void loop() {}

0 commit comments

Comments
 (0)