Skip to content
This repository was archived by the owner on Feb 20, 2023. It is now read-only.

Commit d770eb9

Browse files
bialesdanielapavlo
bialesdaniel
and
apavlo
authored
Common Jenkinsfile Build Function (#1368)
Co-authored-by: Andy Pavlo <pavlo@cs.brown.edu>
1 parent 8855886 commit d770eb9

File tree

10 files changed

+289
-163
lines changed

10 files changed

+289
-163
lines changed

‎Jenkinsfile

+84-100
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
def utils // common build functions are loaded from Jenkinsfile-utils into this object
2+
String utilsFileName = 'Jenkinsfile-utils'
3+
14
pipeline {
25
agent none
36
options {
@@ -99,14 +102,12 @@ pipeline {
99102
}
100103
steps {
101104
sh 'echo $NODE_NAME'
102-
sh script: 'echo y | sudo ./script/installation/packages.sh all', label: 'Installing packages'
103-
104-
sh script: '''
105-
mkdir build
106-
cd build
107-
cmake -GNinja -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_TEST_PARALLELISM=$(nproc) -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_USE_ASAN=ON -DNOISEPAGE_BUILD_BENCHMARKS=OFF -DNOISEPAGE_USE_JUMBOTESTS=ON ..
108-
ninja''', label: 'Compiling'
109-
105+
106+
script{
107+
utils = utils ?: load(utilsFileName)
108+
utils.noisePageBuild(useASAN:true, isJumboTest:true)
109+
}
110+
110111
sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15721', label: 'Kill PID(15721)'
111112
sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15722', label: 'Kill PID(15722)'
112113
sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15723', label: 'Kill PID(15723)'
@@ -142,13 +143,11 @@ pipeline {
142143
}
143144
steps {
144145
sh 'echo $NODE_NAME'
145-
sh 'echo y | sudo ./script/installation/packages.sh all'
146-
147-
sh script: '''
148-
mkdir build
149-
cd build
150-
cmake -GNinja -DNOISEPAGE_UNITY_BUILD=OFF -DNOISEPAGE_TEST_PARALLELISM=1 -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_USE_ASAN=OFF -DNOISEPAGE_BUILD_BENCHMARKS=OFF -DNOISEPAGE_GENERATE_COVERAGE=ON ..
151-
ninja''', label: 'Compiling'
146+
147+
script{
148+
utils = utils ?: load(utilsFileName)
149+
utils.noisePageBuild(isCodeCoverage:true)
150+
}
152151

153152
sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15721', label: 'Kill PID(15721)'
154153
sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15722', label: 'Kill PID(15722)'
@@ -197,13 +196,11 @@ pipeline {
197196
}
198197
steps {
199198
sh 'echo $NODE_NAME'
200-
sh 'echo y | sudo ./script/installation/packages.sh all'
201199

202-
sh script: '''
203-
mkdir build
204-
cd build
205-
cmake -GNinja -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_TEST_PARALLELISM=$(nproc) -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_USE_ASAN=ON -DNOISEPAGE_BUILD_BENCHMARKS=OFF -DNOISEPAGE_USE_JUMBOTESTS=ON ..
206-
ninja''', label: 'Compiling'
200+
script{
201+
utils = utils ?: load(utilsFileName)
202+
utils.noisePageBuild(useASAN:true, isJumboTest:true)
203+
}
207204

208205
sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15721', label: 'Kill PID(15721)'
209206
sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15722', label: 'Kill PID(15722)'
@@ -236,13 +233,11 @@ pipeline {
236233
}
237234
steps {
238235
sh 'echo $NODE_NAME'
239-
sh 'echo y | sudo ./script/installation/packages.sh all'
240236

241-
sh script: '''
242-
mkdir build
243-
cd build
244-
cmake -GNinja -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_TEST_PARALLELISM=$(nproc) -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_BUILD_TYPE=Release -DNOISEPAGE_USE_ASAN=OFF -DNOISEPAGE_BUILD_BENCHMARKS=OFF -DNOISEPAGE_USE_JUMBOTESTS=ON ..
245-
ninja''', label: 'Compiling'
237+
script{
238+
utils = utils ?: load(utilsFileName)
239+
utils.noisePageBuild(buildType:utils.RELEASE_BUILD, isJumboTest:true)
240+
}
246241

247242
sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15721', label: 'Kill PID(15721)'
248243
sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15722', label: 'Kill PID(15722)'
@@ -279,13 +274,11 @@ pipeline {
279274
}
280275
steps {
281276
sh 'echo $NODE_NAME'
282-
sh 'echo y | sudo ./script/installation/packages.sh all'
283277

284-
sh script: '''
285-
mkdir build
286-
cd build
287-
cmake -GNinja -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_TEST_PARALLELISM=$(nproc) -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_BUILD_TYPE=Release -DNOISEPAGE_USE_ASAN=OFF -DNOISEPAGE_BUILD_BENCHMARKS=OFF -DNOISEPAGE_USE_JUMBOTESTS=ON ..
288-
ninja''', label: 'Compiling'
278+
script{
279+
utils = utils ?: load(utilsFileName)
280+
utils.noisePageBuild(buildType:utils.RELEASE_BUILD, isJumboTest:true)
281+
}
289282

290283
sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15721', label: 'Kill PID(15721)'
291284
sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15722', label: 'Kill PID(15722)'
@@ -311,9 +304,9 @@ pipeline {
311304
}
312305
}
313306

314-
stage('End-to-End Debug') {
315-
parallel{
316-
stage('ubuntu-20.04/gcc-9.3 (Debug/e2etest/oltpbench)') {
307+
stage('End-to-End') {
308+
parallel {
309+
stage('Debug') {
317310
agent {
318311
docker {
319312
image 'noisepage:focal'
@@ -322,13 +315,11 @@ pipeline {
322315
}
323316
steps {
324317
sh 'echo $NODE_NAME'
325-
sh script: 'echo y | sudo ./script/installation/packages.sh all', label: 'Installing pacakges'
326318

327-
sh script: '''
328-
mkdir build
329-
cd build
330-
cmake -GNinja -DNOISEPAGE_UNITY_BUILD=ON -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_USE_ASAN=ON ..
331-
ninja noisepage''', label: 'Compiling'
319+
script{
320+
utils = utils ?: load(utilsFileName)
321+
utils.noisePageBuild(useASAN:true, isBuildTests:false)
322+
}
332323

333324
sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
334325

@@ -376,60 +367,57 @@ pipeline {
376367
}
377368
}
378369
}
379-
}
380-
}
381-
stage('End-to-End Performance') {
382-
agent { label 'benchmark' }
383-
steps {
384-
sh 'echo $NODE_NAME'
385-
sh script:'echo y | sudo ./script/installation/packages.sh all', label:'Installing packages'
386-
387-
sh script:'''
388-
mkdir build
389-
cd build
390-
cmake -GNinja -DNOISEPAGE_UNITY_BUILD=ON -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_BUILD_TYPE=Release -DNOISEPAGE_USE_ASAN=OFF -DNOISEPAGE_USE_JEMALLOC=ON ..
391-
ninja noisepage''', label: 'Compiling'
392-
393-
sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
370+
stage('Performance') {
371+
agent { label 'benchmark' }
372+
steps {
373+
sh 'echo $NODE_NAME'
374+
375+
script{
376+
utils = utils ?: load(utilsFileName)
377+
utils.noisePageBuild(buildType:utils.RELEASE_BUILD, isBuildTests:false)
378+
}
394379

395-
sh script:'''
396-
cd build
397-
timeout 10m python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/end_to_end_performance/tatp.json --build-type=release
398-
''', label: 'OLTPBench (TATP)'
380+
sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
399381

400-
sh script:'''
401-
cd build
402-
timeout 10m python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/end_to_end_performance/tatp_wal_disabled.json --build-type=release
403-
''', label: 'OLTPBench (TATP No WAL)'
382+
sh script:'''
383+
cd build
384+
timeout 10m python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/end_to_end_performance/tatp.json --build-type=release
385+
''', label: 'OLTPBench (TATP)'
404386

405-
sh script:'''
406-
cd build
407-
timeout 10m python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/end_to_end_performance/tatp_wal_ramdisk.json --build-type=release
408-
''', label: 'OLTPBench (TATP RamDisk WAL)'
387+
sh script:'''
388+
cd build
389+
timeout 10m python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/end_to_end_performance/tatp_wal_disabled.json --build-type=release
390+
''', label: 'OLTPBench (TATP No WAL)'
409391

410-
sh script:'''
411-
cd build
412-
timeout 30m python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/end_to_end_performance/tpcc.json --build-type=release
413-
''', label: 'OLTPBench (TPCC HDD WAL)'
392+
sh script:'''
393+
cd build
394+
timeout 10m python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/end_to_end_performance/tatp_wal_ramdisk.json --build-type=release
395+
''', label: 'OLTPBench (TATP RamDisk WAL)'
414396

415-
sh script:'''
416-
cd build
417-
timeout 30m python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/end_to_end_performance/tpcc_wal_disabled.json --build-type=release
418-
''', label: 'OLTPBench (TPCC No WAL)'
397+
sh script:'''
398+
cd build
399+
timeout 30m python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/end_to_end_performance/tpcc.json --build-type=release
400+
''', label: 'OLTPBench (TPCC HDD WAL)'
419401

420-
sh script:'''
421-
cd build
422-
timeout 30m python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/end_to_end_performance/tpcc_wal_ramdisk.json --build-type=release
423-
''', label: 'OLTPBench (TPCC RamDisk WAL)'
402+
sh script:'''
403+
cd build
404+
timeout 30m python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/end_to_end_performance/tpcc_wal_disabled.json --build-type=release
405+
''', label: 'OLTPBench (TPCC No WAL)'
424406

425-
sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
407+
sh script:'''
408+
cd build
409+
timeout 30m python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/end_to_end_performance/tpcc_wal_ramdisk.json --build-type=release
410+
''', label: 'OLTPBench (TPCC RamDisk WAL)'
411+
}
412+
post {
413+
cleanup {
414+
deleteDir()
415+
}
416+
}
417+
}
426418
}
427-
post {
428-
cleanup {
429-
deleteDir()
430-
}
431-
}
432419
}
420+
433421
stage('Self-Driving End-to-End Test') {
434422
agent {
435423
docker {
@@ -439,13 +427,11 @@ pipeline {
439427
}
440428
steps {
441429
sh 'echo $NODE_NAME'
442-
sh script: 'echo y | sudo ./script/installation/packages.sh all', label: 'Installing packages'
443430

444-
sh script: '''
445-
mkdir build
446-
cd build
447-
cmake -GNinja -DNOISEPAGE_UNITY_BUILD=ON -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_BUILD_TYPE=Release -DNOISEPAGE_USE_ASAN=OFF -DNOISEPAGE_USE_JEMALLOC=ON -DNOISEPAGE_BUILD_TESTS=OFF -DNOISEPAGE_BUILD_SELF_DRIVING_TESTS=ON ..
448-
ninja mini_runners''', label: 'Self-driving tests (Compile mini_runners)'
431+
script{
432+
utils = utils ?: load(utilsFileName)
433+
utils.noisePageBuild(buildType:utils.RELEASE_BUILD, isBuildTests:false, isBuildSelfDrivingTests: true)
434+
}
449435

450436
// The parameters to the mini_runners target are (arbitrarily picked to complete tests within a reasonable time / picked to exercise all OUs).
451437
// Specifically, the parameters chosen are:
@@ -478,13 +464,11 @@ pipeline {
478464
agent { label 'benchmark' }
479465
steps {
480466
sh 'echo $NODE_NAME'
481-
sh script: 'echo y | sudo ./script/installation/packages.sh all', label: 'Installing packages'
482467

483-
sh script: '''
484-
mkdir build
485-
cd build
486-
cmake -GNinja -DNOISEPAGE_UNITY_BUILD=ON -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_BUILD_TYPE=Release -DNOISEPAGE_USE_ASAN=OFF -DNOISEPAGE_USE_JEMALLOC=ON -DNOISEPAGE_BUILD_TESTS=OFF ..
487-
ninja all''', label: 'Microbenchmark (Compile)'
468+
script{
469+
utils = utils ?: load(utilsFileName)
470+
utils.noisePageBuild(isBuildTests:false, isBuildBenchmarks:true)
471+
}
488472
}
489473
post {
490474
cleanup {
@@ -493,4 +477,4 @@ pipeline {
493477
}
494478
}
495479
}
496-
}
480+
}

0 commit comments

Comments
 (0)