Skip to content

Commit

Permalink
Property to limit test parallelisation (airbytehq#23870)
Browse files Browse the repository at this point in the history
* Adds new functionality on integration test pipeline and test pipeline in order to reduce parallelization.
It's done by creating a `gradle.properties` file in the connector root folder.

Extra: removes test concatenation for performance tests.

* rename parameter

* fix names

* Reduce threads
  • Loading branch information
sergio-ropero authored and danielduckworth committed Mar 13, 2023
1 parent a1a5e78 commit 4915581
Show file tree
Hide file tree
Showing 4 changed files with 10 additions and 3 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
numberThreads=2
5 changes: 4 additions & 1 deletion build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,7 @@ subprojects { subproj ->
// make tag accessible in submodules.
ext {
cloudStorageTestTagName = 'cloud-storage'
numberThreads = project.hasProperty('numberThreads') ? project.getProperty('numberThreads') : Runtime.runtime.availableProcessors() ?: 1
}

spotbugs {
Expand All @@ -354,7 +355,9 @@ subprojects { subproj ->
}

test {
maxParallelForks = Runtime.runtime.availableProcessors() ?: 1
//This allows to set up a `gradle.properties` file inside the connector folder to reduce number of threads and reduce parallelization.
//Specially useful for connectors that shares resources (like Redshift or Snowflake).
maxParallelForks = numberThreads

jacoco {
enabled = true
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,11 @@ class AirbyteIntegrationTestJavaPlugin implements Plugin<Project> {
dependsOn project.airbyteDocker
}

//This allows to set up a `gradle.properties` file inside the connector folder to reduce number of threads and reduce parallelization.
//Specially useful for connectors that shares resources (like Redshift or Snowflake).
ext.numberThreads = project.hasProperty('numberThreads') ? project.getProperty('numberThreads') : Runtime.runtime.availableProcessors() ?: 1
maxHeapSize = '3g'
maxParallelForks = Runtime.runtime.availableProcessors() ?: 1
maxParallelForks = numberThreads

// This is needed to make the destination-snowflake tests succeed - https://github.com/snowflakedb/snowflake-jdbc/issues/589#issuecomment-983944767
jvmArgs = ["--add-opens=java.base/java.nio=ALL-UNNAMED"]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ class AirbytePerformanceTestJavaPlugin implements Plugin<Project> {
}

project.task('performanceTestJava', type: Test) {
testClassesDirs += project.sourceSets.performanceTestJava.output.classesDirs
testClassesDirs = project.sourceSets.performanceTestJava.output.classesDirs
classpath += project.sourceSets.performanceTestJava.runtimeClasspath

systemProperty "cpuLimit", System.getProperty("cpuLimit")
Expand Down

0 comments on commit 4915581

Please sign in to comment.