diff --git a/.github/workflows/chart_update_on_merge.yml b/.github/workflows/chart_update_on_merge.yml index a2b14f2ec65..a17dd971436 100644 --- a/.github/workflows/chart_update_on_merge.yml +++ b/.github/workflows/chart_update_on_merge.yml @@ -60,7 +60,7 @@ jobs: run: | set -e cd cromwell - sbt -Dproject.isSnapshot=false -Dproject.isRelease=false dockerBuildAndPush + sbt -Dproject.isSnapshot=false dockerBuildAndPush - name: Deploy to dev and board release train (Cromwell) uses: broadinstitute/repository-dispatch@master with: diff --git a/.github/workflows/docker_build_test.yml b/.github/workflows/docker_build_test.yml index 01c2ea502c9..b4d373f330e 100644 --- a/.github/workflows/docker_build_test.yml +++ b/.github/workflows/docker_build_test.yml @@ -34,4 +34,10 @@ jobs: run: | set -e cd cromwell - sbt -Dproject.isSnapshot=false -Dproject.isRelease=false docker + sbt -Dproject.isSnapshot=false docker + # Rarely used but we really want it always working for emergencies + - name: Build Cromwell Debug Docker + run: | + set -e + cd cromwell + sbt -Dproject.isDebug=true docker diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index b005da65041..d353eae4e58 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -12,7 +12,6 @@ jobs: project: - cromiam - cromwell-drs-localizer - - perf - server - womtool diff --git a/build.sbt b/build.sbt index a3029d0ab09..8e480907c5e 100644 --- a/build.sbt +++ b/build.sbt @@ -387,11 +387,6 @@ lazy val `cloud-nio-impl-drs` = (project in cloudNio / "cloud-nio-impl-drs") .dependsOn(common) .dependsOn(common % "test->test") -lazy val perf = project - .withExecutableSettings("perf", dependencies = perfDependencies, pushDocker = false) - .dependsOn(common) - .dependsOn(common % "test->test") - lazy val `cromwell-drs-localizer` = project .withExecutableSettings("cromwell-drs-localizer", drsLocalizerDependencies, drsLocalizerSettings) .dependsOn(`cloud-nio-impl-drs`) @@ -453,7 +448,6 @@ lazy val root = (project in file(".")) .aggregate(googleBatch) .aggregate(httpFileSystem) .aggregate(languageFactoryCore) - .aggregate(perf) .aggregate(server) .aggregate(services) .aggregate(sfsBackend) diff --git a/docs/developers/Building.md b/docs/developers/Building.md index 5c5cc21eaf7..08ffe7a3ddd 100644 --- a/docs/developers/Building.md +++ b/docs/developers/Building.md @@ -8,13 +8,6 @@ features or fixes, the following are required to build Cromwell from source: * [AdoptOpenJDK 11 HotSpot](https://adoptopenjdk.net/) * [Git](https://git-scm.com/) -You can also use the [development image](https://github.com/broadinstitute/cromwell/tree/develop/scripts/docker-develop), and build a development container to work inside: - -```bash -$ docker build -t cromwell-dev . -$ docker run -it cromwell-dev bash -``` - First start by cloning the Cromwell repository from GitHub: ```bash @@ -35,15 +28,15 @@ Finally build the Cromwell jar: $ sbt assembly ``` -NOTE: This command will run for a long time the first time. -NOTE: Compiling will not succeed on directories encrypted with ecryptfs (ubuntu encrypted home dirs for example), due to long file paths. - `sbt assembly` will build the runnable Cromwell JAR in `server/target/scala-2.13/` with a name like `cromwell-.jar`. It will also build a runnable Womtool JAR in `womtool/target/scala-2.13/` with a name like `womtool-.jar`. -To build a [Docker](https://www.docker.com/) image, run: +## Docker -```bash -$ sbt server/docker -``` +The following Docker build configurations are supported. Most users will want Snapshot, resulting in an image like `broadinstitute/cromwell:-SNAP`. -This will build and tag a Docker image with a name like `broadinstitute/cromwell:-SNAP`. +| Command | Build Type | Debug Tools | Description | +|------------------------------------------------|------------|-------------|--------------------------------------| +| `sbt server/docker` | Snapshot | No | Most common local build | +| `sbt -Dproject.isDebug=true server/docker` | Debug | Yes | Local build with debugging/profiling | +| `sbt -Dproject.isSnapshot=false server/docker` | Standard | No | Reserved for CI: commit on `develop` | +| `sbt -Dproject.isRelease=true server/docker` | Release | No | Reserved for CI: numbered release | diff --git a/perf/src/main/resources/logback.xml b/perf/src/main/resources/logback.xml deleted file mode 100644 index 8a90f3ba0ab..00000000000 --- a/perf/src/main/resources/logback.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - INFO - ACCEPT - DENY - - - %msg%n - - - - - - ERROR - - - %red(%msg%n) - - - - - - - - - diff --git a/perf/src/main/scala/cromwell/perf/Call.scala b/perf/src/main/scala/cromwell/perf/Call.scala deleted file mode 100644 index b8e91589126..00000000000 --- a/perf/src/main/scala/cromwell/perf/Call.scala +++ /dev/null @@ -1,90 +0,0 @@ -package cromwell.perf - -import java.time.{Duration, OffsetDateTime} - -import io.circe.JsonObject - -case class CallCaching(hit: Option[Boolean], - result: Option[String], - hitFailures: Option[Seq[Map[String, Seq[JsonObject]]]] -) - -case class ExecutionEvents(startTime: OffsetDateTime, description: String, endTime: OffsetDateTime) - -case class Call(shardIndex: Int, - start: OffsetDateTime, - end: OffsetDateTime, - callCaching: Option[CallCaching], - executionEvents: Seq[ExecutionEvents] -) { - val callCachingEventStates = - List("CheckingCallCache", "FetchingCachedOutputsFromDatabase", "BackendIsCopyingCachedOutputs") - val jobPreparationEventStates = - List("Pending", "RequestingExecutionToken", "WaitingForValueStore", "PreparingJob", "CheckingJobStore") - val cacheCopyingEventStates = List("FetchingCachedOutputsFromDatabase", "BackendIsCopyingCachedOutputs") - - /** - * @return Cache copy retries before getting a successful hit or running the job - */ - val cacheCopyRetries: Int = { - val numFailures: Option[Int] = for { - cachingObject <- callCaching - failuresMap <- cachingObject.hitFailures - } yield failuresMap.size - - numFailures.getOrElse(0) - } - - /*** - * @return Time (in Duration) job spent in just CheckingCallCache state - */ - val timeInCheckingCallCacheState: Duration = { - val eventsRelatedToCC = executionEvents.collect { - case event if event.description.equalsIgnoreCase("CheckingCallCache") => - Duration.between(event.startTime, event.endTime) - } - - if (eventsRelatedToCC.nonEmpty) eventsRelatedToCC.reduce(_ plus _) else Duration.ZERO - } - - /** - * @return Time (in Duration) job spent in Call Caching states - */ - val timeInCallCachingState: Duration = { - val eventsRelatedToCC = - executionEvents.filter(event => callCachingEventStates.exists(state => state.equalsIgnoreCase(event.description))) - - if (eventsRelatedToCC.nonEmpty) { - val eventsSortedByStartTime = eventsRelatedToCC.sortBy(_.startTime) - Duration.between(eventsSortedByStartTime.head.startTime, eventsSortedByStartTime.last.endTime) - } else Duration.ZERO - } - - /** - * @return Time (in Duration) job spent in job preparation state i.e time between job submission and running it - * (this doesn't consider the time it spent in call caching states - */ - val timeInJobPreparation: Duration = { - val durationOfEventsInPreparationState = executionEvents.collect { - case event if jobPreparationEventStates.exists(_.equalsIgnoreCase(event.description)) => - Duration.between(event.startTime, event.endTime) - } - - if (durationOfEventsInPreparationState.nonEmpty) durationOfEventsInPreparationState.reduce(_ plus _) - else Duration.ZERO - } - - /** - * - * @return Time (in Duration) job spent in fetching and copying cache hit(s) - */ - val timeForFetchingAndCopyingCacheHit: Duration = { - val durationOfEventsInCacheCopyingState = executionEvents.collect { - case event if cacheCopyingEventStates.exists(_.equalsIgnoreCase(event.description)) => - Duration.between(event.startTime, event.endTime) - } - - if (durationOfEventsInCacheCopyingState.nonEmpty) durationOfEventsInCacheCopyingState.reduce(_ plus _) - else Duration.ZERO - } -} diff --git a/perf/src/main/scala/cromwell/perf/CompareMetadata.scala b/perf/src/main/scala/cromwell/perf/CompareMetadata.scala deleted file mode 100644 index 64e49d93807..00000000000 --- a/perf/src/main/scala/cromwell/perf/CompareMetadata.scala +++ /dev/null @@ -1,153 +0,0 @@ -package cromwell.perf - -import java.io.FileInputStream -import java.time.{Duration, OffsetDateTime} - -import better.files.File -import cats.data.Validated.{Invalid, Valid} -import cats.syntax.all._ -import cats.instances.list._ -import com.google.auth.oauth2.GoogleCredentials -import com.google.cloud.storage.StorageOptions -import com.typesafe.scalalogging.StrictLogging -import common.validation.ErrorOr._ -import io.circe._ -import io.circe.generic.semiauto._ -import io.circe.parser._ - -object CompareMetadata extends App with StrictLogging { - private val REGRESSION_CONST = 1.1 - - // If removed, IntelliJ (Oct '18) thinks the import isn't used. - // Later the compiler fails to find a decoder for OffsetDateTime. - lazy val anchorOffsetDateTime: Decoder[OffsetDateTime] = implicitly - implicit lazy val decodeCall: Decoder[Call] = deriveDecoder - implicit lazy val decodeCallCaching: Decoder[CallCaching] = deriveDecoder - implicit lazy val decodeExecutionEvents: Decoder[ExecutionEvents] = deriveDecoder - implicit lazy val decodeMetadata: Decoder[Metadata] = deriveDecoder - - def parseMetadataFromLocalFile(filePath: String): Either[Error, Metadata] = { - val metadataFile = File(filePath) - val metadataFileContent = metadataFile.contentAsString - - decode[Metadata](metadataFileContent) - } - - def parseMetadataFromGcsFile(gcsUrl: String, pathToServiceAccount: String): Either[Error, Metadata] = { - val gcsUrlArray = gcsUrl.replace("gs://", "").split("/", 2) - val Array(gcsBucket, fileToBeLocalized) = gcsUrlArray - - val credentials = GoogleCredentials.fromStream(new FileInputStream(pathToServiceAccount)) - val storage = StorageOptions.newBuilder().setCredentials(credentials).build().getService - val blob = storage.get(gcsBucket, fileToBeLocalized) - val metadataFileContent = blob.getContent().map(_.toChar).mkString - - decode[Metadata](metadataFileContent) - } - - def parseMetadata(inputFile: String, pathToServiceAccount: String): Either[Error, Metadata] = - if (inputFile.startsWith("gs://")) - parseMetadataFromGcsFile(inputFile, pathToServiceAccount) - else parseMetadataFromLocalFile(inputFile) - - def displayComputedMetrics(metadata: Metadata, displayMsg: String): Unit = { - logger.info(displayMsg) - - logger.info(s"Workflow started after: ${metadata.workflowStartedAfter}") - logger.info(s"Workflow Running time: ${metadata.workflowRunningTime}") - logger.info(s"Total jobs per root workflow: ${metadata.totalJobsPerRootWf}") - logger.info(s"Avg cache copy retries: ${metadata.avgCacheRetries}") - logger.info(s"Avg time job spent in Call Caching state: ${metadata.avgTimeInCallCachingState}") - logger.info(s"Avg time job spent in just CheckingCallCache state: ${metadata.avgTimeInCheckingCallCacheState}") - logger.info(s"Avg time job spent in Job Preparation state: ${metadata.avgTimeInJobPreparation}") - logger.info( - s"Avg time job spent in fetching and copying cache hit(s) state: ${metadata.avgTimeForFetchingAndCopyingCacheHit}" - ) - } - - /*** - * Compares the metrics in Metadata which have type 'Duration' (mostly the ones related to time metrics) - */ - def compareDurationMetrics(metadataOld: Metadata, - metadataNew: Metadata, - metricFunc: Metadata => Duration, - metricName: String - ): ErrorOr[String] = - if (metricFunc(metadataNew).toMillis > REGRESSION_CONST * metricFunc(metadataOld).toMillis) { - (s"$metricName of new metadata is greater than 10% of old metadata. " + - s"New metric value: ${metricFunc(metadataNew)}. " + - s"Old metric value: ${metricFunc(metadataOld)}.").invalidNel[String] - } else s"$metricName hasn't regressed yet".valid - - /*** - * Compares the metrics in Metadata which have type 'Int' (mostly the ones related to size metrics) - */ - def compareIntMetrics(metadataOld: Metadata, - metadataNew: Metadata, - metricFunc: Metadata => Int, - metricName: String - ): ErrorOr[String] = - if (metricFunc(metadataNew) > REGRESSION_CONST * metricFunc(metadataOld)) { - (s"$metricName of new metadata is greater than 10% of old metadata. " + - s"New metric value: ${metricFunc(metadataNew)}. " + - s"Old metric value: ${metricFunc(metadataOld)}.").invalidNel[String] - } else s"$metricName hasn't regressed yet".valid - - def compareMetadataMetrics(metadataOld: Metadata, metadataNew: Metadata): ErrorOr[List[String]] = { - val durationFuncList: List[(Metadata => Duration, String)] = List( - (_.workflowStartedAfter, "workflowStartedAfter"), - (_.workflowRunningTime, "workflowRunningTime"), - (_.avgTimeInCallCachingState, "avgTimeInCallCachingState"), - (_.avgTimeInCheckingCallCacheState, "avgTimeInCheckingCallCacheState"), - (_.avgTimeInJobPreparation, "avgTimeInJobPreparation"), - (_.avgTimeForFetchingAndCopyingCacheHit, "avgTimeForFetchingAndCopyingCacheHit") - ) - - val intFuncList: List[(Metadata => Int, String)] = - List((_.totalJobsPerRootWf, "totalJobsPerRootWf"), (_.avgCacheRetries, "avgCacheRetries")) - - val durationMetricsComp = durationFuncList.map(x => compareDurationMetrics(metadataOld, metadataNew, x._1, x._2)) - val intMetricsComp = intFuncList.map(x => compareIntMetrics(metadataOld, metadataNew, x._1, x._2)) - (durationMetricsComp ::: intMetricsComp).sequence[ErrorOr, String] - } - - def printParseErrorToConsoleAndExit(metadataFile: String, error: Error, systemExit: Boolean): Unit = { - logger.error(s"Something went wrong while parsing $metadataFile. Error: ${error.getLocalizedMessage}") - if (systemExit) System.exit(1) - } - - def generateAndCompareMetrics(metadataOldEither: Either[Error, Metadata], - metadataNewEither: Either[Error, Metadata] - ): Unit = - (metadataOldEither, metadataNewEither) match { - case (Right(metadataOld), Right(metadataNew)) => - val metadataOldMsg = s"Metrics for metadata generated from ${args(0)}" - val metadataNewMsg = s"\nMetrics for metadata generated from ${args(1)}" - displayComputedMetrics(metadataOld, metadataOldMsg) - displayComputedMetrics(metadataNew, metadataNewMsg) - compareMetadataMetrics(metadataOld, metadataNew) match { - case Valid(_) => logger.info("\nYAY!! Metrics from new metadata json haven't regressed!") - case Invalid(listOfErrors) => - logger.error("\nBelow metadata metrics have regressed:") - logger.error(listOfErrors.toList.mkString("\n")) - System.exit(1) - } - case (Right(_), Left(e)) => printParseErrorToConsoleAndExit(args(1), e, systemExit = true) - case (Left(e), Right(_)) => printParseErrorToConsoleAndExit(args(0), e, systemExit = true) - case (Left(e1), Left(e2)) => - printParseErrorToConsoleAndExit(args(0), e1, systemExit = false) - printParseErrorToConsoleAndExit(args(1), e2, systemExit = true) - } - - args.length match { - case 2 => - if (args(0).startsWith("gs://") || args(1).startsWith("gs://")) { - logger.error("Path to service account is needed to download GCS file. Please pass it as 3rd argument.") - System.exit(1) - } else generateAndCompareMetrics(parseMetadataFromLocalFile(args(0)), parseMetadataFromLocalFile(args(1))) - case 3 => generateAndCompareMetrics(parseMetadata(args(0), args(2)), parseMetadata(args(1), args(2))) - case _ => - logger.error("Please pass in 2 file paths!") - System.exit(1) - } -} diff --git a/perf/src/main/scala/cromwell/perf/Metadata.scala b/perf/src/main/scala/cromwell/perf/Metadata.scala deleted file mode 100644 index b61e88c795a..00000000000 --- a/perf/src/main/scala/cromwell/perf/Metadata.scala +++ /dev/null @@ -1,74 +0,0 @@ -package cromwell.perf - -import java.time.{Duration, OffsetDateTime} - -case class Metadata(id: String, - workflowName: String, - submission: OffsetDateTime, - start: OffsetDateTime, - end: OffsetDateTime, - status: String, - calls: Option[Map[String, Seq[Call]]] -) { - - private def addInt(x: Int, y: Int): Int = x + y - - private def addDuration(x: Duration, y: Duration): Duration = x.plus(y) - - private def sumElementsInOptionSeq[A](listOption: Option[Iterable[A]], op: (A, A) => A, default: A): A = - listOption match { - case Some(list) => list.reduce[A]((a, b) => op(a, b)) - case None => default - } - - /** - * @return Time between the submission and start of workflow - */ - val workflowStartedAfter: Duration = Duration.between(submission, start) - - val workflowRunningTime: Duration = Duration.between(start, end) - - val totalJobsPerRootWf: Int = - sumElementsInOptionSeq(calls.map(taskMap => taskMap.map(callsPerTask => callsPerTask._2.size)), addInt, 0) - - val avgCacheRetries: Int = - if (totalJobsPerRootWf > 0) { - val cacheRetriesList = - calls.map(taskMap => taskMap.flatMap(callsPerTask => callsPerTask._2.map(call => call.cacheCopyRetries))) - sumElementsInOptionSeq(cacheRetriesList, addInt, 0) / totalJobsPerRootWf - } else 0 - - val avgTimeInCallCachingState: Duration = - if (totalJobsPerRootWf > 0) { - val timeInCallCachingStateList = - calls.map(taskMap => taskMap.flatMap(callsPerTask => callsPerTask._2.map(call => call.timeInCallCachingState))) - sumElementsInOptionSeq(timeInCallCachingStateList, addDuration, Duration.ZERO).dividedBy( - totalJobsPerRootWf.toLong - ) - } else Duration.ZERO - - val avgTimeInCheckingCallCacheState: Duration = - if (totalJobsPerRootWf > 0) { - val timeInCheckingCallCacheStateList = calls.map(taskMap => - taskMap.flatMap(callsPerTask => callsPerTask._2.map(call => call.timeInCheckingCallCacheState)) - ) - sumElementsInOptionSeq(timeInCheckingCallCacheStateList, addDuration, Duration.ZERO).dividedBy( - totalJobsPerRootWf.toLong - ) - } else Duration.ZERO - - val avgTimeInJobPreparation: Duration = - if (totalJobsPerRootWf > 0) { - val timeInJobPreparationList = - calls.map(taskMap => taskMap.flatMap(callsPerTask => callsPerTask._2.map(call => call.timeInJobPreparation))) - sumElementsInOptionSeq(timeInJobPreparationList, addDuration, Duration.ZERO).dividedBy(totalJobsPerRootWf.toLong) - } else Duration.ZERO - - val avgTimeForFetchingAndCopyingCacheHit: Duration = - if (totalJobsPerRootWf > 0) { - val timeInCopyingList = calls.map(taskMap => - taskMap.flatMap(callsPerTask => callsPerTask._2.map(call => call.timeForFetchingAndCopyingCacheHit)) - ) - sumElementsInOptionSeq(timeInCopyingList, addDuration, Duration.ZERO).dividedBy(totalJobsPerRootWf.toLong) - } else Duration.ZERO -} diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 8a12d0e0cc2..fc9beae1f12 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -643,9 +643,6 @@ object Dependencies { // Version of the swagger UI to write into config files val swaggerUiVersion: String = swaggerUiV - val perfDependencies: List[ModuleID] = circeDependencies ++ betterFilesDependencies ++ commonDependencies ++ - googleApiClientDependencies ++ googleCloudDependencies - val drsLocalizerDependencies: List[ModuleID] = List( "com.google.auth" % "google-auth-library-oauth2-http" % googleOauth2V, "com.google.cloud" % "google-cloud-storage" % googleCloudStorageV, @@ -675,7 +672,6 @@ object Dependencies { implDrsDependencies ++ implFtpDependencies ++ languageFactoryDependencies ++ - perfDependencies ++ serverDependencies ++ sfsBackendDependencies ++ spiDependencies ++ diff --git a/project/Publishing.scala b/project/Publishing.scala index 6dd1b281700..4abc5cc3258 100644 --- a/project/Publishing.scala +++ b/project/Publishing.scala @@ -1,4 +1,4 @@ -import Version.cromwellVersion +import Version.{Debug, Release, Snapshot, Standard, cromwellVersion} import org.apache.ivy.Ivy import org.apache.ivy.core.IvyPatternHelper import org.apache.ivy.core.module.descriptor.{DefaultModuleDescriptor, MDArtifact} @@ -36,20 +36,26 @@ object Publishing { ArrayBuffer(broadinstitute/cromwell:dev, broadinstitute/cromwell:develop) */ dockerTags := { - val versionsCsv = if (Version.isSnapshot) { - // Tag looks like `85-443a6fc-SNAP` - version.value - } else { - if (Version.isRelease) { - // Tags look like `85`, `85-443a6fc` - s"$cromwellVersion,${version.value}" - } else { - // Tag looks like `85-443a6fc`, `latest` - s"${version.value},latest" - } + val tags = Version.buildType match { + case Snapshot => + // Ordinary local build + // Looks like `85-443a6fc-SNAP` + Seq(version.value) + case Release => + // Looks like `85`, `85-443a6fc` + Seq(cromwellVersion, version.value) + case Debug => + // Ordinary local build with debug stuff + // Looks like `85-443a6fc-DEBUG` + Seq(version.value) + case Standard => + // Merge to `develop` + // Looks like `85-443a6fc`, `latest`, `develop` + // TODO: once we automate releases, `latest` should move to `Release` + Seq(version.value, "latest", "develop") } - // Travis applies (as of 10/22) the `dev` and `develop` tags on merge to `develop` + val versionsCsv = tags.mkString(",") sys.env.getOrElse("CROMWELL_SBT_DOCKER_TAGS", versionsCsv).split(",") }, docker / imageNames := dockerTags.value map { tag => @@ -68,6 +74,11 @@ object Publishing { add(artifact, artifactTargetPath) runRaw(s"ln -s $artifactTargetPath /app/$projectName.jar") + // Extra tools in debug mode only + if (Version.buildType == Debug) { + addInstruction(installDebugFacilities) + } + /* If you use the 'exec' form for an entry point, shell processing is not performed and environment variable substitution does not occur. Thus we have to /bin/bash here @@ -116,6 +127,44 @@ object Publishing { ) ) + /** + * Install packages needed for debugging when shelled in to a running image. + * + * This includes: + * - The JDK, which includes tools like `jstack` not present in the JRE + * - The YourKit Java Profiler + * - Various Linux system & development utilities + * + * @return Instruction to run in the build + */ + def installDebugFacilities: Instruction = { + import sbtdocker.Instructions + + // It is optimal to use a single `Run` instruction to minimize the number of layers in the image. + // Do not be tempted to install the default JDK in the repositories, it's from Oracle. + // + // Documentation: + // - https://www.yourkit.com/docs/java-profiler/2024.3/help/docker_broker.jsp#setup + // - https://adoptium.net/installation/linux/#_deb_installation_on_debian_or_ubuntu + Instructions.Run( + """apt-get update -qq && \ + |apt-get install -qq --no-install-recommends file gpg htop jq less nload unzip vim && \ + |wget -qO - https://packages.adoptium.net/artifactory/api/gpg/key/public | gpg --dearmor | \ + | tee /etc/apt/trusted.gpg.d/adoptium.gpg > /dev/null && \ + |echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | \ + | tee /etc/apt/sources.list.d/adoptium.list && \ + |apt-get update -qq && \ + |apt-get install -qq temurin-11-jdk && \ + |rm -rf /var/lib/apt/lists/* && \ + |wget -q https://www.yourkit.com/download/docker/YourKit-JavaProfiler-2024.3-docker.zip -P /tmp/docker-build-cache/ && \ + |unzip /tmp/docker-build-cache/YourKit-JavaProfiler-2024.3-docker.zip -d /tmp/docker-build-cache && \ + |mkdir -p /usr/local/YourKit-JavaProfiler-2024.3/bin/ && \ + |cp -R /tmp/docker-build-cache/YourKit-JavaProfiler-2024.3/bin/linux-x86-64/ /usr/local/YourKit-JavaProfiler-2024.3/bin/linux-x86-64/ && \ + |rm -rf /tmp/docker-build-cache + |""".stripMargin + ) + } + def dockerPushSettings(pushEnabled: Boolean): Seq[Setting[_]] = if (pushEnabled) { List( diff --git a/project/Version.scala b/project/Version.scala index e530e54e62e..26b0057eb10 100644 --- a/project/Version.scala +++ b/project/Version.scala @@ -8,24 +8,44 @@ object Version { // Upcoming release, or current if we're on a master / hotfix branch val cromwellVersion = "87" + sealed trait BuildType + case object Snapshot extends BuildType + case object Debug extends BuildType + case object Release extends BuildType + case object Standard extends BuildType + + def buildType: BuildType = { + if (isDebug) Debug + else if (isRelease) Release + else if (isSnapshot) Snapshot + else Standard + } + /** * Returns true if this project should be considered a snapshot. * * The value is read in directly from the system property `project.isSnapshot` as there were confusing issues with * the multi-project and sbt.Keys#isSnapshot(). * - * Default `true`. + * This is the default if no arguments are provided. */ - val isSnapshot: Boolean = sys.props.get("project.isSnapshot").forall(_.toBoolean) + private lazy val isSnapshot: Boolean = getPropOrDefault("project.isSnapshot", default = true) /** - * Returns `true` if this project should tag a release like `85` in addition to a hash like `85-443a6fc`. - * - * Has no effect when `isSnapshot` is `true`. + * Returns true if this project should be built in the debugging configuration. * - * Default `true`. + * Note that this image is much larger than the default build! */ - val isRelease: Boolean = sys.props.get("project.isRelease").forall(_.toBoolean) + private lazy val isDebug: Boolean = getPropOrDefault("project.isDebug") + + /** + * Returns `true` if this project should tag a release like `85` in addition to a hash like `85-443a6fc`. + */ + private lazy val isRelease: Boolean = getPropOrDefault("project.isRelease") + + private def getPropOrDefault(prop: String, default: Boolean = false): Boolean = { + sys.props.get(prop).map(_.toBoolean).getOrElse(default) + } // Adapted from SbtGit.versionWithGit def cromwellVersionWithGit: Seq[Setting[_]] = @@ -90,6 +110,11 @@ object Version { val version = overrideVersion orElse commitVersion getOrElse unknownVersion // For now, obfuscate SNAPSHOTs from sbt's developers: https://github.com/sbt/sbt/issues/2687#issuecomment-236586241 - if (isSnapshot) s"$version-SNAP" else version + // (by calling it `SNAP` instead of `SNAPSHOT`) + buildType match { + case Snapshot => s"$version-SNAP" + case Debug => s"$version-DEBUG" + case _ => version + } } } diff --git a/scripts/perf/clone_database/clone_database.sh b/scripts/perf/clone_database/clone_database.sh deleted file mode 100755 index 99240996ff4..00000000000 --- a/scripts/perf/clone_database/clone_database.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -set -euo pipefail - -source scripts/perf/helper.inc.sh - -mkdir -p mnt - -# Read the DB password from vault: -DB_PASS=$(read_path_from_vault_json "secret/dsp/cromwell/perf" '.data.db_pass') - -# Read the service account credentials from vault: -read_service_account_from_vault - -# Clone the CloudSQL DB -# Note: Cloning the same database in parallel doesn't work. -# By doing it here we can run the jenkins jobs sequentially and ensure the database is cloned once at a time as well -SQL_OPERATION=$(gcloud_run_as_service_account "perf_sql_create_gcloud_${BUILD_NUMBER}" "gcloud --project broad-dsde-cromwell-perf sql instances clone --async ${CLOUD_SQL_INSTANCE_TO_CLONE} ${CLOUD_SQL_INSTANCE_NEW_NAME} --format='value(name)'" ) - -gcloud_run_as_service_account "perf_sql_await_gcloud_${BUILD_NUMBER}" "gcloud beta sql operations wait --timeout=900 --project broad-dsde-cromwell-perf ${SQL_OPERATION}" - -gcloud_run_as_service_account "perf_sql_user_gcloud_${BUILD_NUMBER}" "gcloud --project broad-dsde-cromwell-perf sql users create cromwell --instance=${CLOUD_SQL_INSTANCE_NEW_NAME} --password=${DB_PASS}" diff --git a/scripts/perf/delete_database/delete_database.sh b/scripts/perf/delete_database/delete_database.sh deleted file mode 100755 index a393cac5eb2..00000000000 --- a/scripts/perf/delete_database/delete_database.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -set -euo pipefail - -source scripts/perf/helper.inc.sh - -mkdir -p mnt - -# Read the service account credentials from vault: -read_service_account_from_vault - -gcloud_run_as_service_account "perf_sql_delete_gcloud_${BUILD_NUMBER}" " \ - gcloud \ - --verbosity info \ - --project broad-dsde-cromwell-perf \ - sql instances delete ${CLOUD_SQL_INSTANCE} -q" diff --git a/scripts/perf/delete_horicromtal/delete_horicromtal.sh b/scripts/perf/delete_horicromtal/delete_horicromtal.sh deleted file mode 100755 index 807b9c239b0..00000000000 --- a/scripts/perf/delete_horicromtal/delete_horicromtal.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -set -euo pipefail - -source scripts/perf/helper.inc.sh - -mkdir -p mnt - -# Read the service account credentials from vault: -read_service_account_from_vault - -for instance in runner101 runner102 reader101 -do - gcloud_delete_instance \ - "perf_delete_gcloud_instance_${BUILD_NUMBER}-${instance}" \ - "${CROMWELL_INSTANCE_NAME}-${instance}" -done diff --git a/scripts/perf/delete_instance/delete_instance.sh b/scripts/perf/delete_instance/delete_instance.sh deleted file mode 100755 index 29d40ce7fb6..00000000000 --- a/scripts/perf/delete_instance/delete_instance.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -set -euo pipefail - -source scripts/perf/helper.inc.sh - -mkdir -p mnt - -# Read the service account credentials from vault: -read_service_account_from_vault - -gcloud_run_as_service_account "perf_delete_gcloud_instance_${BUILD_NUMBER}" " \ - gcloud \ - --verbosity info \ - --project broad-dsde-cromwell-perf \ - compute instances delete ${CROMWELL_INSTANCE_NAME} --zone=us-central1-c -q" diff --git a/scripts/perf/deploy_horicromtal/deploy_horicromtal.sh b/scripts/perf/deploy_horicromtal/deploy_horicromtal.sh deleted file mode 100755 index 6d060996b77..00000000000 --- a/scripts/perf/deploy_horicromtal/deploy_horicromtal.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash -set -euo pipefail - -source scripts/perf/helper.inc.sh - -DB_PASS=$(read_path_from_vault_json "secret/dsp/cromwell/perf" '.data.db_pass') - -read_service_account_from_vault - -runner_metadata=( - "BUILD_NUMBER=$BUILD_NUMBER" - "BUILD_TAG=$BUILD_TAG" - "CLEAN_UP=false" - "CLOUD_SQL_INSTANCE=$CLOUD_SQL_INSTANCE" - "CROMWELL_DB_USER=cromwell" - "CROMWELL_DB_PASS=$DB_PASS" - "CROMWELL_DOCKER_IMAGE=$CROMWELL_DOCKER_IMAGE" - "CROMWELL_PROJECT=broad-dsde-cromwell-perf" - "CROMWELL_BUCKET=${CROMWELL_EXECUTION_BUCKET}" - "CROMWELL_STATSD_HOST=10.128.0.4" - "CROMWELL_STATSD_PORT=8125" - "CROMWELL_PERF_SCRIPTS_BRANCH=${REPO_BRANCH}" - "GCS_REPORT_PATH=${GCS_REPORT_PATH}" - "GCS_REPORT_BUCKET=${GCS_REPORT_BUCKET}" - "TEST_CASE_DIRECTORY=${TEST_CASE_DIRECTORY}" - "CONFIG_OVERRIDE"="-Dservices.MetadataService.config.metadata-summary-refresh-interval = \"Inf\"" -) - -reader_metadata=( - "BUILD_NUMBER=$BUILD_NUMBER" - "BUILD_TAG=$BUILD_TAG" - "CLEAN_UP=false" - "CLOUD_SQL_INSTANCE=$CLOUD_SQL_INSTANCE" - "CROMWELL_DB_USER=cromwell" - "CROMWELL_DB_PASS=$DB_PASS" - "CROMWELL_DOCKER_IMAGE=$CROMWELL_DOCKER_IMAGE" - "CROMWELL_PROJECT=broad-dsde-cromwell-perf" - "CROMWELL_BUCKET=${CROMWELL_EXECUTION_BUCKET}" - "CROMWELL_STATSD_HOST=10.128.0.4" - "CROMWELL_STATSD_PORT=8125" - "CROMWELL_PERF_SCRIPTS_BRANCH=${REPO_BRANCH}" - "GCS_REPORT_PATH=${GCS_REPORT_PATH}" - "GCS_REPORT_BUCKET=${GCS_REPORT_BUCKET}" - "TEST_CASE_DIRECTORY=reader" - "CONFIG_OVERRIDE"="" -) - -cp scripts/perf/deploy_instance/run_on_instance.sh mnt/ - -gcloud_deploy_instance \ - "perf_deploy_instance_${BUILD_NUMBER}-runner101" \ - "${CROMWELL_INSTANCE_NAME}-runner101" \ - "${INSTANCE_TEMPLATE}" \ - "$(join ${runner_metadata[@]})" | tee runner1.txt - -gcloud_deploy_instance \ - "perf_deploy_instance_${BUILD_NUMBER}-runner102" \ - "${CROMWELL_INSTANCE_NAME}-runner102" \ - "${INSTANCE_TEMPLATE}" \ - "$(join ${runner_metadata[@]})" | tee runner2.txt - -# Note: The reader_metadata currently allows the default summarizer to continue running... -# ... that's fine as long as there's only one reader, but if we add more, we should make the "summarizer" a separate VM. - -gcloud_deploy_instance \ - "perf_deploy_instance_${BUILD_NUMBER}-reader101" \ - "${CROMWELL_INSTANCE_NAME}-reader101" \ - "${INSTANCE_TEMPLATE}" \ - "$(join ${reader_metadata[@]})" | tee reader1.txt - -typeset RUNNER_UNDER_TEST_1=$(cat runner1.txt | tail -n1 | awk '{print $5}') -typeset RUNNER_UNDER_TEST_2=$(cat runner2.txt | tail -n1 | awk '{print $5}') - -typeset READER_UNDER_TEST_1=$(cat reader1.txt | tail -n1 | awk '{print $5}') -typeset CROMWELL_VALUE_TO_RETURN=$(cat reader1.txt | tail -n1 | awk "{print \$${FIELD_NO_TO_RETURN}}" ) - -if test -z "${RUNNER_UNDER_TEST_1}" -o -z "${RUNNER_UNDER_TEST_2}" -o -z "${READER_UNDER_TEST_1}" -then - echo "One of RUNNER_UNDER_TEST_1 or RUNNER_UNDER_TEST_2 or READER_UNDER_TEST_1 are empty ('${RUNNER_UNDER_TEST_1}', '${RUNNER_UNDER_TEST_2}, ${READER_UNDER_TEST_1}')" - exit 1 -else - echo "Determined that RUNNER_UNDER_TEST_1=${RUNNER_UNDER_TEST_1}, RUNNER_UNDER_TEST_2=${RUNNER_UNDER_TEST_2}, READER_UNDER_TEST_1=${READER_UNDER_TEST_1}" -fi - -custom_wait_for_cromwell "${RUNNER_UNDER_TEST_1}" -custom_wait_for_cromwell "${RUNNER_UNDER_TEST_2}" -custom_wait_for_cromwell "${READER_UNDER_TEST_1}" - -mkdir -p output -echo "CROMWELL_UNDER_TEST=${CROMWELL_VALUE_TO_RETURN}" > output/cromwell.properties diff --git a/scripts/perf/deploy_horicromtal/run_on_instance.sh b/scripts/perf/deploy_horicromtal/run_on_instance.sh deleted file mode 100644 index c9420a68ae3..00000000000 --- a/scripts/perf/deploy_horicromtal/run_on_instance.sh +++ /dev/null @@ -1,78 +0,0 @@ -#! /bin/bash - -set -x - -### /!\ This script assumes docker and docker compose are already installed on the host - -# Utility function to extract values from instance metadata -extract_metadata() { - curl "http://metadata.google.internal/computeMetadata/v1/instance/attributes/$1" -H "Metadata-Flavor: Google" -} - -# Exports an env var and also adds it to the root bashrc. This way if there is a need to ssh onto the machine -# for debugging one will have the env variables already set when using root -addVar() { - export $1 - echo "export $1" >> /root/.bashrc -} - -# Make sure ip forwarding is enabled by default so that docker doesn't lose connectivity -echo "net.ipv4.ip_forward = 1" > /etc/sysctl.conf - -# Set up env variables -addVar CROMWELL_BRANCH=$(extract_metadata CROMWELL_PERF_SCRIPTS_BRANCH) -addVar CLOUD_SQL_DB_USER=$(extract_metadata CROMWELL_DB_USER) -addVar CLOUD_SQL_DB_PASSWORD=$(extract_metadata CROMWELL_DB_PASS) -addVar CLOUD_SQL_INSTANCE=$(extract_metadata CLOUD_SQL_INSTANCE) -addVar CROMWELL_DOCKER_IMAGE=$(extract_metadata CROMWELL_DOCKER_IMAGE) -addVar CROMWELL_PROJECT=$(extract_metadata CROMWELL_PROJECT) -addVar CROMWELL_EXECUTION_ROOT=$(extract_metadata CROMWELL_BUCKET) -addVar CROMWELL_STATSD_HOST=$(extract_metadata CROMWELL_STATSD_HOST) -addVar CROMWELL_STATSD_PORT=$(extract_metadata CROMWELL_STATSD_PORT) -addVar BUILD_ID=$(extract_metadata BUILD_TAG) -addVar GCS_REPORT_BUCKET=$(extract_metadata GCS_REPORT_BUCKET) -addVar GCS_REPORT_PATH=$(extract_metadata GCS_REPORT_PATH) -addVar TEST_CASE_DIRECTORY=$(extract_metadata TEST_CASE_DIRECTORY) -addVar CONFIG_OVERRIDE=$(extract_metadata CONFIG_OVERRIDE) - -# Use the instance name as statsd prefix to avoid metrics collisions -addVar CROMWELL_STATSD_PREFIX=${BUILD_ID} - -addVar CROMWELL_ROOT=/app -addVar PERF_ROOT=${CROMWELL_ROOT}/scripts/perf -addVar TEST_WORKFLOW_ROOT=${PERF_ROOT}/test_cases - -# Clone cromwell to get the perf scripts. Use https to avoid ssh fingerprint prompt when the script runs -git clone -b ${CROMWELL_BRANCH} --depth 1 --single-branch https://github.com/broadinstitute/cromwell.git ${CROMWELL_ROOT} - -source ${PERF_ROOT}/helper.inc.sh - -if [ -f "${PERF_ROOT}/test_cases/${TEST_CASE_DIRECTORY}/cromwell.conf" ] -then - echo "Using ${PERF_ROOT}/test_cases/${TEST_CASE_DIRECTORY} as the Cromwell configuration directory" - addVar CROMWELL_CONF_DIR="${PERF_ROOT}/test_cases/${TEST_CASE_DIRECTORY}" - - # Also copy the default one next to it so we can include it - cp ${PERF_ROOT}/vm_scripts/cromwell/cromwell.conf ${CROMWELL_CONF_DIR}/cromwell_default.conf -else - echo "Using ${PERF_ROOT}/vm_scripts/cromwell as the Cromwell configuration directory" - addVar CROMWELL_CONF_DIR="${PERF_ROOT}/vm_scripts/cromwell" -fi - -if test -n "${CONFIG_OVERRIDE}" -then - echo "${CONFIG_OVERRIDE}" >> "${CROMWELL_CONF_DIR}/cromwell.conf" -fi - -addVar REPORT_URL="$(strip_trailing_slash "gs://${GCS_REPORT_BUCKET}/${GCS_REPORT_PATH}")" - -# Start cromwell and cloud sql proxy -prepare_statsd_proxy -docker-compose -f ${PERF_ROOT}/vm_scripts/docker-compose.yml up -d - -# Intermittently upload Cromwell logs to the bucket -while true -do - export_cromwell_logs - sleep 60 -done diff --git a/scripts/perf/deploy_instance/deploy_instance.sh b/scripts/perf/deploy_instance/deploy_instance.sh deleted file mode 100755 index c1775b8d4b0..00000000000 --- a/scripts/perf/deploy_instance/deploy_instance.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -set -euo pipefail - -source scripts/perf/helper.inc.sh - -DB_PASS=$(read_path_from_vault_json "secret/dsp/cromwell/perf" '.data.db_pass') - -read_service_account_from_vault - -function join() { local IFS=","; echo "$*"; } - -metadata=( - "BUILD_NUMBER=$BUILD_NUMBER" - "BUILD_TAG=$BUILD_TAG" - "CLEAN_UP=false" - "CLOUD_SQL_INSTANCE=$CLOUD_SQL_INSTANCE" - "CROMWELL_DB_USER=cromwell" - "CROMWELL_DB_PASS=$DB_PASS" - "CROMWELL_DOCKER_IMAGE=$CROMWELL_DOCKER_IMAGE" - "CROMWELL_PROJECT=broad-dsde-cromwell-perf" - "CROMWELL_BUCKET=${CROMWELL_EXECUTION_BUCKET}" - "CROMWELL_STATSD_HOST=10.128.0.4" - "CROMWELL_STATSD_PORT=8125" - "CROMWELL_PERF_SCRIPTS_BRANCH=${REPO_BRANCH}" - "GCS_REPORT_PATH=${GCS_REPORT_PATH}" - "GCS_REPORT_BUCKET=${GCS_REPORT_BUCKET}" - "TEST_CASE_DIRECTORY=${TEST_CASE_DIRECTORY}" -) - -cp scripts/perf/deploy_instance/run_on_instance.sh mnt/ - -gcloud_run_as_service_account "perf_deploy_instance_${BUILD_NUMBER}" \ - "gcloud \ - --verbosity info \ - --project broad-dsde-cromwell-perf \ - compute \ - instances \ - create ${CROMWELL_INSTANCE_NAME} \ - --zone us-central1-c \ - --source-instance-template $INSTANCE_TEMPLATE \ - --metadata-from-file startup-script=$DOCKER_ETC_PATH/run_on_instance.sh \ - --metadata \ - $(join ${metadata[@]})" | tee dockerResult.txt - -typeset CROMWELL_UNDER_TEST=$(cat dockerResult.txt | tail -n1 | awk '{print $5}' ) -typeset CROMWELL_VALUE_TO_RETURN=$(cat dockerResult.txt | tail -n1 | awk "{print \$${FIELD_NO_TO_RETURN}}" ) - -if test -z "CROMWELL_UNDER_TEST" -o -z "CROMWELL_VALUE_TO_RETURN" -then - echo "One of CROMWELL_UNDER_TEST or CROMWELL_VALUE_TO_RETURN are empty ('${CROMWELL_UNDER_TEST}', '${CROMWELL_VALUE_TO_RETURN}')" - exit 1 -else - echo "Determined that CROMWELL_UNDER_TEST=${CROMWELL_UNDER_TEST}, CROMWELL_VALUE_TO_RETURN=${CROMWELL_VALUE_TO_RETURN}" -fi - -custom_wait_for_cromwell "${CROMWELL_UNDER_TEST}" - -mkdir -p output -echo "CROMWELL_UNDER_TEST=${CROMWELL_VALUE_TO_RETURN}" > output/cromwell.properties diff --git a/scripts/perf/deploy_instance/run_on_instance.sh b/scripts/perf/deploy_instance/run_on_instance.sh deleted file mode 100644 index 3c5d1e85ed4..00000000000 --- a/scripts/perf/deploy_instance/run_on_instance.sh +++ /dev/null @@ -1,72 +0,0 @@ -#! /bin/bash - -set -x - -### /!\ This script assumes docker and docker compose are already installed on the host - -# Utility function to extract values from instance metadata -extract_metadata() { - curl "http://metadata.google.internal/computeMetadata/v1/instance/attributes/$1" -H "Metadata-Flavor: Google" -} - -# Exports an env var and also adds it to the root bashrc. This way if there is a need to ssh onto the machine -# for debugging one will have the env variables already set when using root -addVar() { - export $1 - echo "export $1" >> /root/.bashrc -} - -# Make sure ip forwarding is enabled by default so that docker doesn't lose connectivity -echo "net.ipv4.ip_forward = 1" > /etc/sysctl.conf - -# Set up env variables -addVar CROMWELL_BRANCH=$(extract_metadata CROMWELL_PERF_SCRIPTS_BRANCH) -addVar CLOUD_SQL_DB_USER=$(extract_metadata CROMWELL_DB_USER) -addVar CLOUD_SQL_DB_PASSWORD=$(extract_metadata CROMWELL_DB_PASS) -addVar CLOUD_SQL_INSTANCE=$(extract_metadata CLOUD_SQL_INSTANCE) -addVar CROMWELL_DOCKER_IMAGE=$(extract_metadata CROMWELL_DOCKER_IMAGE) -addVar CROMWELL_PROJECT=$(extract_metadata CROMWELL_PROJECT) -addVar CROMWELL_EXECUTION_ROOT=$(extract_metadata CROMWELL_BUCKET) -addVar CROMWELL_STATSD_HOST=$(extract_metadata CROMWELL_STATSD_HOST) -addVar CROMWELL_STATSD_PORT=$(extract_metadata CROMWELL_STATSD_PORT) -addVar BUILD_ID=$(extract_metadata BUILD_TAG) -addVar GCS_REPORT_BUCKET=$(extract_metadata GCS_REPORT_BUCKET) -addVar GCS_REPORT_PATH=$(extract_metadata GCS_REPORT_PATH) -addVar TEST_CASE_DIRECTORY=$(extract_metadata TEST_CASE_DIRECTORY) - -# Use the instance name as statsd prefix to avoid metrics collisions -addVar CROMWELL_STATSD_PREFIX=${BUILD_ID} - -addVar CROMWELL_ROOT=/app -addVar PERF_ROOT=${CROMWELL_ROOT}/scripts/perf -addVar TEST_WORKFLOW_ROOT=${PERF_ROOT}/test_cases - -# Clone cromwell to get the perf scripts. Use https to avoid ssh fingerprint prompt when the script runs -git clone -b ${CROMWELL_BRANCH} --depth 1 --single-branch https://github.com/broadinstitute/cromwell.git ${CROMWELL_ROOT} - -source ${PERF_ROOT}/helper.inc.sh - -if [ -f "${PERF_ROOT}/test_cases/${TEST_CASE_DIRECTORY}/cromwell.conf" ] -then - echo "Using ${PERF_ROOT}/test_cases/${TEST_CASE_DIRECTORY} as the Cromwell configuration directory" - addVar CROMWELL_CONF_DIR="${PERF_ROOT}/test_cases/${TEST_CASE_DIRECTORY}" - - # Also copy the default one next to it so we can include it - cp ${PERF_ROOT}/vm_scripts/cromwell/cromwell.conf ${CROMWELL_CONF_DIR}/cromwell_default.conf -else - echo "Using ${PERF_ROOT}/vm_scripts/cromwell as the Cromwell configuration directory" - addVar CROMWELL_CONF_DIR="${PERF_ROOT}/vm_scripts/cromwell" -fi - -addVar REPORT_URL="$(strip_trailing_slash "gs://${GCS_REPORT_BUCKET}/${GCS_REPORT_PATH}")" - -# Start cromwell and cloud sql proxy -prepare_statsd_proxy -docker-compose -f ${PERF_ROOT}/vm_scripts/docker-compose.yml up -d - -# Intermittently upload Cromwell logs to the bucket -while true -do - export_cromwell_logs - sleep 60 -done diff --git a/scripts/perf/helper.inc.sh b/scripts/perf/helper.inc.sh deleted file mode 100644 index aa9e9818c7b..00000000000 --- a/scripts/perf/helper.inc.sh +++ /dev/null @@ -1,279 +0,0 @@ -#!/usr/bin/env bash - -# Set some global constants: -typeset CROMWELL_EXECUTION_BUCKET=gs://cromwell-perf-test/ - -typeset DOCKER_ETC_PATH="/usr/share/etc" -typeset CROMWELL_PERF_PROJECT=broad-dsde-cromwell-perf - -if test -f "/etc/vault-token-dsde" -then - typeset VAULT_TOKEN=$(cat /etc/vault-token-dsde) -fi - -wait_for_cromwell() { - git clone https://github.com/vishnubob/wait-for-it.git /wait-for-it - chmod u+x /wait-for-it/wait-for-it.sh - # Give 5 minutes to cromwell to be online - - echo "[$(date)] Waiting for http://${CROMWELL_UNDER_TEST}:8000/engine/v1/version to appear..." - /wait-for-it/wait-for-it.sh ${CROMWELL_UNDER_TEST}:8000 -t 300 - READY=$? - if [ ${READY} -eq 0 ] - then - # Just wait a bit longer - no rush, this is a chill VM - this is because cromwell responds to requests before being really ready... - sleep 30 - - CROMWELL_VERSION=$(curl --silent -X GET "http://${CROMWELL_UNDER_TEST}:8000/engine/v1/version" -H "accept: application/json" | jq -r '.cromwell') - if [ -z ${CROMWELL_VERSION} ] - then - echo "Cromwell was up but failed to return its version, so something went wrong, shutting down" - shutdown - fi - export CROMWELL_VERSION - else - echo "Cromwell was not ready after 5 minutes, shutting down" - shutdown - fi -} - -# Waits for Cromwell. Works even from *outside* a google instance -custom_wait_for_cromwell() { - - set +e - - local cromwell_under_test=$1 - - RESULT=1 - ATTEMPTS=0 - # increase max tries from 20 to 60 to allow more time for cromwell instance to deploy - MAX_ATTEMPTS=60 - - while [ "${ATTEMPTS}" -le "${MAX_ATTEMPTS}" -a "${RESULT}" -gt "0" ] - do - echo "[$(date)] Waiting for Cromwell (http://${cromwell_under_test}:8000/engine/v1/version) to come up (tried ${ATTEMPTS} times so far)" - sleep 30 - ATTEMPTS=$((ATTEMPTS + 1)) - - CROMWELL_VERSION_JSON=$(curl --silent -X GET "http://${cromwell_under_test}:8000/engine/v1/version" -H "accept: application/json") - RESULT=$? - - CROMWELL_VERSION=$(echo "${CROMWELL_VERSION_JSON}" | jq -r '.cromwell') - - if [ "${RESULT}" -eq "0" -a -z "${CROMWELL_VERSION}" ] - then - echo "Cromwell was up but failed to return its version, maybe something went wrong? Continuing waiting..." - RESULT=1 - fi - done - - if [ "${RESULT}" -gt "0" ] - then - echo "Cromwell never came up after ${ATTEMPTS} attempts" - exit 1 - else - echo "Cromwell came up after ${ATTEMPTS} attempts" - fi -} - -export_centaur_logs() { - echo "Exporting centaur logs to ${REPORT_URL}" - - # Copy centaur log - gsutil -h "Content-Type:text/plain" cp "${CROMWELL_ROOT}/centaur.log" "${REPORT_URL}/centaur.log" || true - - # Copy test_rc.log - echo "${TEST_RC}" > test_rc.txt && gsutil -h "Content-Type:text/plain" cp "test_rc.txt" "${REPORT_URL}/test_rc.txt" || true -} - -export_cromwell_logs() { - echo "Exporting Cromwell logs to ${REPORT_URL} as $(hostname)" - - # Copy the cromwell container logs - gsutil -h "Content-Type:text/plain" cp "$(docker inspect --format='{{.LogPath}}' cromwell)" "${REPORT_URL}/cromwell_$(hostname).log" || true - # Copy the docker daemon logs - gsutil -h "Content-Type:text/plain" cp "/var/log/daemon.log" "${REPORT_URL}/daemon_$(hostname).log" || true -} - - -export_logs() { - export REPORT_URL="gs://${GCS_REPORT_BUCKET}/${GCS_REPORT_PATH}" - - export_cromwell_logs - export_centaur_logs -} - -clean_up() { - if [ "${CLEAN_UP}" = true ] - then - gcloud sql instances delete ${CLOUD_SQL_INSTANCE} - self_destruct_instance - fi -} - -self_destruct_instance() { - gcloud compute instances delete $(curl --silent "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google") --zone=us-central1-c -q -} - -run_test() { - cd ${CROMWELL_ROOT} - - if [ -z ${GCS_REPORT_PATH} ] - then - export GCS_REPORT_PATH="${TEST_CASE_DIRECTORY}/${CROMWELL_VERSION}/${BUILD_ID}" - fi - - sbt \ - --error \ - -Dconfig.file=${CENTAUR_CONF_DIR}/centaur.conf \ - -Dcentaur.cromwell.mode="url" \ - -Dcentaur.cromwell.url="http://${CROMWELL_UNDER_TEST}:8000" \ - "centaur/it:testOnly centaur.ExternalTestCaseSpec" | tee centaur.log - export TEST_RC=$? -} - -shutdown() { - export_logs - docker-compose -f ${PERF_ROOT}/vm_scripts/docker-compose.yml down - clean_up -} - -function join() { local IFS=","; echo "$*"; } - -read_path_from_vault_json() { - local path=$1 - local field=$2 - - docker run --rm -e VAULT_TOKEN=$VAULT_TOKEN broadinstitute/dsde-toolbox vault read -format=json "${path}" | jq --exit-status -r "${field}" -} - - -read_service_account_from_vault() { - mkdir -p mnt - read_path_from_vault_json "secret/dsp/cromwell/perf/service-account-deployer" '.data.service_account' > mnt/sa.json -} - -gcloud_run_as_service_account() { - local name="$1" - local command="$2" - docker run --name ${name} -v "$(pwd)"/mnt:${DOCKER_ETC_PATH} --rm gcr.io/google.com/cloudsdktool/cloud-sdk:slim \ - /bin/bash -c "\ - gcloud auth activate-service-account --key-file ${DOCKER_ETC_PATH}/sa.json 2> /dev/null &&\ - ${command}" -} - -gcloud_get_field_from_instance_describe() { - local instance="$1" - local field="$2" - - INSTANCE_METADATA=$(gcloud_run_as_service_account "perf_read_field_${field}" \ - "gcloud \ - --project broad-dsde-cromwell-perf \ - compute instances describe \ - --zone us-central1-c \ - ${instance} - ") - - echo "${INSTANCE_METADATA}" | grep "${field}" | awk '{print $2}' -} - -gcloud_deploy_instance() { - local taskname=$1 - local instance_name=$2 - local instance_template=$3 - local joined_metadata_string=$4 - - gcloud_run_as_service_account "${taskname}" \ - "gcloud \ - --verbosity info \ - --project broad-dsde-cromwell-perf \ - compute instances create ${instance_name} \ - --zone us-central1-c \ - --source-instance-template ${instance_template} \ - --metadata-from-file startup-script=$DOCKER_ETC_PATH/run_on_instance.sh \ - --metadata ${joined_metadata_string}" -} - -gcloud_delete_instance() { - local taskname=$1 - local instance_name=$2 - - gcloud_run_as_service_account "${taskname}" " \ - gcloud \ - --verbosity info \ - --project broad-dsde-cromwell-perf \ - compute instances delete ${instance_name} \ - --zone=us-central1-c -q" -} - -# Waits for a remote machine to start, and then stop, responding to PINGs -wait_for_ping_to_start_then_stop() { - local address="$1" - - # Turn off "fail on exit code" and "debug logging": - set +e - set +x - - RESULT=1 - ATTEMPTS=0 - MAX_ATTEMPTS=20 - - while [ "${ATTEMPTS}" -le "${MAX_ATTEMPTS}" -a "${RESULT}" -gt "0" ] - do - echo "[$(date)] Waiting for ${address} to respond to pings (tried ${ATTEMPTS} times so far)" - sleep 30 - ATTEMPTS=$((ATTEMPTS + 1)) - - ping -c 1 ${address} &> /dev/null - RESULT=$? - done - - if [ "${RESULT}" -gt "0" ] - then - echo "[$(date)] ${address} never came up after ${ATTEMPTS} attempts" - exit 1 - else - echo "[$(date)] ${address} came up after ${ATTEMPTS} attempts" - fi - - while ping -c 1 ${address} &> /dev/null - do - echo "[$(date)] ${address} is still responding to pings. Pausing before next ping" - sleep 30 - done - - echo "[$(date)] ${address} has stopped responding to pings." - - set -e -} - -strip_trailing_slash() { - local path=$1 - echo "${path}" | sed 's/\/$//' -} - -# Patch the activation policy of the Cloud SQL Instance referenced by $CLOUD_SQL_INSTANCE. -# In practice used mostly to turn a Cloud SQL instance on or off. -# -# Arguments: -# `op` - a descriptive term that will appear in the Docker container that runs gcloud. -# `policy` - the Cloud SQL activation policy, should be one of ALWAYS, NEVER or ON_DEMAND. -# For more details: https://cloud.google.com/sql/docs/mysql/start-stop-restart-instance -private::patch_cloud_sql_activation_policy() { - op="$1" - policy="$2" - - mkdir -p mnt - # Read the service account credentials from vault: - read_service_account_from_vault - - gcloud_run_as_service_account "perf_sql_${op}_gcloud_${BUILD_NUMBER}" "gcloud -q --project broad-dsde-cromwell-perf sql instances patch ${CLOUD_SQL_INSTANCE} --activation-policy ${policy}" -} - -start_cloud_sql() { - private::patch_cloud_sql_activation_policy "start" "ALWAYS" -} - -stop_cloud_sql() { - private::patch_cloud_sql_activation_policy "stop" "NEVER" -} diff --git a/scripts/perf/run_centaur_blocking/create_runner_instance.sh b/scripts/perf/run_centaur_blocking/create_runner_instance.sh deleted file mode 100755 index 9c0603f5dba..00000000000 --- a/scripts/perf/run_centaur_blocking/create_runner_instance.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -source scripts/perf/helper.inc.sh - -# Prepare the startup script to run on new instance -mkdir -p mnt -cp scripts/perf/run_centaur_blocking/test_runner_script.sh mnt/ - - -read_service_account_from_vault - -metadata=( - "BUILD_NUMBER=${BUILD_NUMBER}" - "BUILD_TAG=${BUILD_TAG}" - "CLEAN_UP=true" - "CROMWELL_PROJECT=${CROMWELL_PERF_PROJECT}" - "CROMWELL_BUCKET=${CROMWELL_EXECUTION_BUCKET}" - "CROMWELL_PERF_SCRIPTS_BRANCH=${CROMWELL_BRANCH}" - "CROMWELL_UNDER_TEST=${CROMWELL_UNDER_TEST}" - "GCS_REPORT_PATH=${GCS_REPORT_PATH}" - "GCS_REPORT_BUCKET=${GCS_REPORT_BUCKET}" - "TEST_CASE_DIRECTORY=${TEST_CASE_DIRECTORY}" -) - -# NB: The 'cromwell-perf' tag is required to give cromwell the 'perf' firewall rules - -gcloud_run_as_service_account "perf_centaur_test_runner_${BUILD_NUMBER}" \ - "gcloud \ - --verbosity info \ - --project ${CROMWELL_PERF_PROJECT} \ - compute \ - instances \ - create ${BUILD_TAG} \ - --zone us-central1-c \ - --source-instance-template ${INSTANCE_TEMPLATE} \ - --tags cromwell-perf \ - --metadata-from-file startup-script=${DOCKER_ETC_PATH}/test_runner_script.sh \ - --metadata \ - $(join ${metadata[@]})" | tee startupResult.txt - -typeset CENTAUR_TEST_RUNNER=$(cat startupResult.txt | tail -n1 | awk '{print $5}' ) - -echo "Determined that CENTAUR_TEST_RUNNER=${CENTAUR_TEST_RUNNER}" - -wait_for_ping_to_start_then_stop ${CENTAUR_TEST_RUNNER} diff --git a/scripts/perf/run_centaur_blocking/test_runner_script.sh b/scripts/perf/run_centaur_blocking/test_runner_script.sh deleted file mode 100644 index 0c6baf9a688..00000000000 --- a/scripts/perf/run_centaur_blocking/test_runner_script.sh +++ /dev/null @@ -1,88 +0,0 @@ -#! /bin/bash - -# Deliberately don't set these options. The script must continue to run (and clean itself up) even if the centaur tests -# fail. -#set -euo pipefail -set +x - -### /!\ This script assumes docker and docker compose are already installed on the host - -# Utility function to extract values from instance metadata -extract_metadata() { - curl "http://metadata.google.internal/computeMetadata/v1/instance/attributes/$1" -H "Metadata-Flavor: Google" -} - -# Exports an env var and also adds it to the root bashrc. This way if there is a need to ssh onto the machine -# for debugging one will have the env variables already set when using root -addVar() { - export $1 - echo "export $1" >> /root/.bashrc -} - -# Make sure ip forwarding is enabled by default so that docker doesn't loses connectivity -echo "net.ipv4.ip_forward = 1" > /etc/sysctl.conf - -# Extract env variables from instance metadata: -addVar TEST_CASE_DIRECTORY=$(extract_metadata TEST_CASE_DIRECTORY) -addVar CROMWELL_UNDER_TEST=$(extract_metadata CROMWELL_UNDER_TEST) -addVar CROMWELL_PROJECT=$(extract_metadata CROMWELL_PROJECT) -addVar GCS_REPORT_BUCKET=$(extract_metadata GCS_REPORT_BUCKET) -addVar GCS_REPORT_PATH=$(extract_metadata GCS_REPORT_PATH) -addVar BUILD_ID=$(extract_metadata BUILD_TAG) -addVar CROMWELL_PERF_SCRIPTS_BRANCH=$(extract_metadata CROMWELL_PERF_SCRIPTS_BRANCH) - -addVar CROMWELL_ROOT=/app -addVar PERF_ROOT=${CROMWELL_ROOT}/scripts/perf - -# Clone cromwell to get the perf scripts. Use https to avoid ssh fingerprint prompt when the script runs -git clone -b ${CROMWELL_PERF_SCRIPTS_BRANCH} --depth 1 --single-branch https://github.com/broadinstitute/cromwell.git ${CROMWELL_ROOT} - -source ${PERF_ROOT}/helper.inc.sh - -addVar CLEAN_UP=true - -addVar REPORT_BUCKET="cromwell-perf-test-reporting" -addVar TEST_WORKFLOW_ROOT="${PERF_ROOT}/test_cases" -addVar REPORT_URL="$(strip_trailing_slash "gs://${GCS_REPORT_BUCKET}/${GCS_REPORT_PATH}")" - -if [ -n "${TEST_CASE_DIRECTORY}" ] -then - # Must be a directory - TEST_WORKFLOW_DIR=${TEST_WORKFLOW_ROOT}/${TEST_CASE_DIRECTORY} - - # If it contains a custom cromwell configuration, use that - if [ -f "${TEST_WORKFLOW_DIR}/cromwell.conf" ] - then - addVar CROMWELL_CONF_DIR=${TEST_WORKFLOW_DIR} - # copy the default one next to it so we can include it - cp ${PERF_ROOT}/vm_scripts/cromwell/cromwell.conf ${CROMWELL_CONF_DIR}/cromwell_default.conf - else - # Otherwise use the default one - addVar CROMWELL_CONF_DIR=${PERF_ROOT}/vm_scripts/cromwell - fi - - # If it contains a custom centaur configuration, use that - if [ -f "${TEST_WORKFLOW_DIR}/centaur.conf" ] - then - addVar CENTAUR_CONF_DIR=${TEST_WORKFLOW_DIR} - # copy the default one next to it so we can include it - cp ${PERF_ROOT}/vm_scripts/centaur/centaur.conf ${CENTAUR_CONF_DIR}/centaur_default.conf - else - # Otherwise use the default one - addVar CENTAUR_CONF_DIR=${PERF_ROOT}/vm_scripts/centaur - fi - - addVar CENTAUR_TEST_FILE=$(ls ${TEST_WORKFLOW_DIR}/*.test | head) - sed -i "s/\$BRANCH/${CROMWELL_PERF_SCRIPTS_BRANCH}/" ${CENTAUR_TEST_FILE} -fi - -if [ -n "${CENTAUR_TEST_FILE}" ] -then - wait_for_cromwell - run_test -else - echo "No workflow provided, shutting down" -fi - -export_centaur_logs -self_destruct_instance diff --git a/scripts/perf/start_database/start_database.sh b/scripts/perf/start_database/start_database.sh deleted file mode 100644 index 779c6350e77..00000000000 --- a/scripts/perf/start_database/start_database.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set -euo pipefail - -source scripts/perf/helper.inc.sh - -start_cloud_sql diff --git a/scripts/perf/stop_database/stop_database.sh b/scripts/perf/stop_database/stop_database.sh deleted file mode 100644 index 8772f793fa5..00000000000 --- a/scripts/perf/stop_database/stop_database.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set -euo pipefail - -source scripts/perf/helper.inc.sh - -stop_cloud_sql diff --git a/scripts/perf/test_cases/blacklist_call_caching_stress/blacklist_call_caching_stress.test b/scripts/perf/test_cases/blacklist_call_caching_stress/blacklist_call_caching_stress.test deleted file mode 100644 index a00e1fdff3f..00000000000 --- a/scripts/perf/test_cases/blacklist_call_caching_stress/blacklist_call_caching_stress.test +++ /dev/null @@ -1,12 +0,0 @@ -name: blacklist_call_caching_stress -testFormat: workflowsuccess - -files { - workflow: "https://raw.githubusercontent.com/broadinstitute/cromwell/$BRANCH/scripts/perf/test_cases/call_caching_stress_workflow/callCachingStress.wdl" - inputs: "https://raw.githubusercontent.com/broadinstitute/cromwell/$BRANCH/scripts/perf/test_cases/call_caching_stress_workflow/callCachingStress.inputs" - options: "gs://cromwell-perf-test/workflow_related_files/call_caching_sa2_workflow_options.json" -} - -metadata { - status: Succeeded -} diff --git a/scripts/perf/test_cases/blacklist_call_caching_stress/centaur.conf b/scripts/perf/test_cases/blacklist_call_caching_stress/centaur.conf deleted file mode 100644 index 2b7ce0eed62..00000000000 --- a/scripts/perf/test_cases/blacklist_call_caching_stress/centaur.conf +++ /dev/null @@ -1,18 +0,0 @@ -include required("centaur_default.conf") - -centaur { - metadata-args { - includeKey = [ - "id", - "%shardIndex%", - "%hitFailures%", - "%executionEvents%", - "status", - "workflowName", - "submission", - "start", - "end", - "failures" - ] - } -} diff --git a/scripts/perf/test_cases/blacklist_call_caching_stress/cromwell.conf b/scripts/perf/test_cases/blacklist_call_caching_stress/cromwell.conf deleted file mode 100644 index 710bfed0fa8..00000000000 --- a/scripts/perf/test_cases/blacklist_call_caching_stress/cromwell.conf +++ /dev/null @@ -1,9 +0,0 @@ -include required("cromwell_default.conf") - -engine.filesystems.gcs.auth = "user-service-account" -backend.providers.Papi.config.genomics.auth = "user-service-account" -backend.providers.Papi.config.filesystems.gcs.auth = "user-service-account" - - -call-caching.blacklist-cache.enabled = true -call-caching.blacklist-cache.ttl = "3 days" diff --git a/scripts/perf/test_cases/call_caching_lots_of_inputs/call_caching_stress_lots_of_inputs.test b/scripts/perf/test_cases/call_caching_lots_of_inputs/call_caching_stress_lots_of_inputs.test deleted file mode 100644 index 8210853b91e..00000000000 --- a/scripts/perf/test_cases/call_caching_lots_of_inputs/call_caching_stress_lots_of_inputs.test +++ /dev/null @@ -1,11 +0,0 @@ -name: call_caching_lots_of_inputs -testFormat: workflowsuccess - -files { - workflow: "https://raw.githubusercontent.com/broadinstitute/cromwell/$BRANCH/scripts/perf/test_cases/call_caching_lots_of_inputs/lots_of_inputs.wdl" - inputs: "https://raw.githubusercontent.com/broadinstitute/cromwell/$BRANCH/scripts/perf/test_cases/call_caching_lots_of_inputs/lots_of_inputs.inputs" -} - -metadata { - status: Succeeded -} diff --git a/scripts/perf/test_cases/call_caching_lots_of_inputs/lots_of_inputs.inputs b/scripts/perf/test_cases/call_caching_lots_of_inputs/lots_of_inputs.inputs deleted file mode 100644 index 84d05f5842f..00000000000 --- a/scripts/perf/test_cases/call_caching_lots_of_inputs/lots_of_inputs.inputs +++ /dev/null @@ -1,804 +0,0 @@ -{ - "call_cache_read_input_files_perf_test.files" : [ - "gs://cromwell-perf-testing/lots_of_input_files/input-1", - "gs://cromwell-perf-testing/lots_of_input_files/input-2", - "gs://cromwell-perf-testing/lots_of_input_files/input-3", - "gs://cromwell-perf-testing/lots_of_input_files/input-4", - "gs://cromwell-perf-testing/lots_of_input_files/input-5", - "gs://cromwell-perf-testing/lots_of_input_files/input-6", - "gs://cromwell-perf-testing/lots_of_input_files/input-7", - "gs://cromwell-perf-testing/lots_of_input_files/input-8", - "gs://cromwell-perf-testing/lots_of_input_files/input-9", - "gs://cromwell-perf-testing/lots_of_input_files/input-10", - "gs://cromwell-perf-testing/lots_of_input_files/input-11", - "gs://cromwell-perf-testing/lots_of_input_files/input-12", - "gs://cromwell-perf-testing/lots_of_input_files/input-13", - "gs://cromwell-perf-testing/lots_of_input_files/input-14", - "gs://cromwell-perf-testing/lots_of_input_files/input-15", - "gs://cromwell-perf-testing/lots_of_input_files/input-16", - "gs://cromwell-perf-testing/lots_of_input_files/input-17", - "gs://cromwell-perf-testing/lots_of_input_files/input-18", - "gs://cromwell-perf-testing/lots_of_input_files/input-19", - "gs://cromwell-perf-testing/lots_of_input_files/input-20", - "gs://cromwell-perf-testing/lots_of_input_files/input-21", - "gs://cromwell-perf-testing/lots_of_input_files/input-22", - "gs://cromwell-perf-testing/lots_of_input_files/input-23", - "gs://cromwell-perf-testing/lots_of_input_files/input-24", - "gs://cromwell-perf-testing/lots_of_input_files/input-25", - "gs://cromwell-perf-testing/lots_of_input_files/input-26", - "gs://cromwell-perf-testing/lots_of_input_files/input-27", - "gs://cromwell-perf-testing/lots_of_input_files/input-28", - "gs://cromwell-perf-testing/lots_of_input_files/input-29", - "gs://cromwell-perf-testing/lots_of_input_files/input-30", - "gs://cromwell-perf-testing/lots_of_input_files/input-31", - "gs://cromwell-perf-testing/lots_of_input_files/input-32", - "gs://cromwell-perf-testing/lots_of_input_files/input-33", - "gs://cromwell-perf-testing/lots_of_input_files/input-34", - "gs://cromwell-perf-testing/lots_of_input_files/input-35", - "gs://cromwell-perf-testing/lots_of_input_files/input-36", - "gs://cromwell-perf-testing/lots_of_input_files/input-37", - "gs://cromwell-perf-testing/lots_of_input_files/input-38", - "gs://cromwell-perf-testing/lots_of_input_files/input-39", - "gs://cromwell-perf-testing/lots_of_input_files/input-40", - "gs://cromwell-perf-testing/lots_of_input_files/input-41", - "gs://cromwell-perf-testing/lots_of_input_files/input-42", - "gs://cromwell-perf-testing/lots_of_input_files/input-43", - "gs://cromwell-perf-testing/lots_of_input_files/input-44", - "gs://cromwell-perf-testing/lots_of_input_files/input-45", - "gs://cromwell-perf-testing/lots_of_input_files/input-46", - "gs://cromwell-perf-testing/lots_of_input_files/input-47", - "gs://cromwell-perf-testing/lots_of_input_files/input-48", - "gs://cromwell-perf-testing/lots_of_input_files/input-49", - "gs://cromwell-perf-testing/lots_of_input_files/input-50", - "gs://cromwell-perf-testing/lots_of_input_files/input-51", - "gs://cromwell-perf-testing/lots_of_input_files/input-52", - "gs://cromwell-perf-testing/lots_of_input_files/input-53", - "gs://cromwell-perf-testing/lots_of_input_files/input-54", - "gs://cromwell-perf-testing/lots_of_input_files/input-55", - "gs://cromwell-perf-testing/lots_of_input_files/input-56", - "gs://cromwell-perf-testing/lots_of_input_files/input-57", - "gs://cromwell-perf-testing/lots_of_input_files/input-58", - "gs://cromwell-perf-testing/lots_of_input_files/input-59", - "gs://cromwell-perf-testing/lots_of_input_files/input-60", - "gs://cromwell-perf-testing/lots_of_input_files/input-61", - "gs://cromwell-perf-testing/lots_of_input_files/input-62", - "gs://cromwell-perf-testing/lots_of_input_files/input-63", - "gs://cromwell-perf-testing/lots_of_input_files/input-64", - "gs://cromwell-perf-testing/lots_of_input_files/input-65", - "gs://cromwell-perf-testing/lots_of_input_files/input-66", - "gs://cromwell-perf-testing/lots_of_input_files/input-67", - "gs://cromwell-perf-testing/lots_of_input_files/input-68", - "gs://cromwell-perf-testing/lots_of_input_files/input-69", - "gs://cromwell-perf-testing/lots_of_input_files/input-70", - "gs://cromwell-perf-testing/lots_of_input_files/input-71", - "gs://cromwell-perf-testing/lots_of_input_files/input-72", - "gs://cromwell-perf-testing/lots_of_input_files/input-73", - "gs://cromwell-perf-testing/lots_of_input_files/input-74", - "gs://cromwell-perf-testing/lots_of_input_files/input-75", - "gs://cromwell-perf-testing/lots_of_input_files/input-76", - "gs://cromwell-perf-testing/lots_of_input_files/input-77", - "gs://cromwell-perf-testing/lots_of_input_files/input-78", - "gs://cromwell-perf-testing/lots_of_input_files/input-79", - "gs://cromwell-perf-testing/lots_of_input_files/input-80", - "gs://cromwell-perf-testing/lots_of_input_files/input-81", - "gs://cromwell-perf-testing/lots_of_input_files/input-82", - "gs://cromwell-perf-testing/lots_of_input_files/input-83", - "gs://cromwell-perf-testing/lots_of_input_files/input-84", - "gs://cromwell-perf-testing/lots_of_input_files/input-85", - "gs://cromwell-perf-testing/lots_of_input_files/input-86", - "gs://cromwell-perf-testing/lots_of_input_files/input-87", - "gs://cromwell-perf-testing/lots_of_input_files/input-88", - "gs://cromwell-perf-testing/lots_of_input_files/input-89", - "gs://cromwell-perf-testing/lots_of_input_files/input-90", - "gs://cromwell-perf-testing/lots_of_input_files/input-91", - "gs://cromwell-perf-testing/lots_of_input_files/input-92", - "gs://cromwell-perf-testing/lots_of_input_files/input-93", - "gs://cromwell-perf-testing/lots_of_input_files/input-94", - "gs://cromwell-perf-testing/lots_of_input_files/input-95", - "gs://cromwell-perf-testing/lots_of_input_files/input-96", - "gs://cromwell-perf-testing/lots_of_input_files/input-97", - "gs://cromwell-perf-testing/lots_of_input_files/input-98", - "gs://cromwell-perf-testing/lots_of_input_files/input-99", - "gs://cromwell-perf-testing/lots_of_input_files/input-100", - "gs://cromwell-perf-testing/lots_of_input_files/input-101", - "gs://cromwell-perf-testing/lots_of_input_files/input-102", - "gs://cromwell-perf-testing/lots_of_input_files/input-103", - "gs://cromwell-perf-testing/lots_of_input_files/input-104", - "gs://cromwell-perf-testing/lots_of_input_files/input-105", - "gs://cromwell-perf-testing/lots_of_input_files/input-106", - "gs://cromwell-perf-testing/lots_of_input_files/input-107", - "gs://cromwell-perf-testing/lots_of_input_files/input-108", - "gs://cromwell-perf-testing/lots_of_input_files/input-109", - "gs://cromwell-perf-testing/lots_of_input_files/input-110", - "gs://cromwell-perf-testing/lots_of_input_files/input-111", - "gs://cromwell-perf-testing/lots_of_input_files/input-112", - "gs://cromwell-perf-testing/lots_of_input_files/input-113", - "gs://cromwell-perf-testing/lots_of_input_files/input-114", - "gs://cromwell-perf-testing/lots_of_input_files/input-115", - "gs://cromwell-perf-testing/lots_of_input_files/input-116", - "gs://cromwell-perf-testing/lots_of_input_files/input-117", - "gs://cromwell-perf-testing/lots_of_input_files/input-118", - "gs://cromwell-perf-testing/lots_of_input_files/input-119", - "gs://cromwell-perf-testing/lots_of_input_files/input-120", - "gs://cromwell-perf-testing/lots_of_input_files/input-121", - "gs://cromwell-perf-testing/lots_of_input_files/input-122", - "gs://cromwell-perf-testing/lots_of_input_files/input-123", - "gs://cromwell-perf-testing/lots_of_input_files/input-124", - "gs://cromwell-perf-testing/lots_of_input_files/input-125", - "gs://cromwell-perf-testing/lots_of_input_files/input-126", - "gs://cromwell-perf-testing/lots_of_input_files/input-127", - "gs://cromwell-perf-testing/lots_of_input_files/input-128", - "gs://cromwell-perf-testing/lots_of_input_files/input-129", - "gs://cromwell-perf-testing/lots_of_input_files/input-130", - "gs://cromwell-perf-testing/lots_of_input_files/input-131", - "gs://cromwell-perf-testing/lots_of_input_files/input-132", - "gs://cromwell-perf-testing/lots_of_input_files/input-133", - "gs://cromwell-perf-testing/lots_of_input_files/input-134", - "gs://cromwell-perf-testing/lots_of_input_files/input-135", - "gs://cromwell-perf-testing/lots_of_input_files/input-136", - "gs://cromwell-perf-testing/lots_of_input_files/input-137", - "gs://cromwell-perf-testing/lots_of_input_files/input-138", - "gs://cromwell-perf-testing/lots_of_input_files/input-139", - "gs://cromwell-perf-testing/lots_of_input_files/input-140", - "gs://cromwell-perf-testing/lots_of_input_files/input-141", - "gs://cromwell-perf-testing/lots_of_input_files/input-142", - "gs://cromwell-perf-testing/lots_of_input_files/input-143", - "gs://cromwell-perf-testing/lots_of_input_files/input-144", - "gs://cromwell-perf-testing/lots_of_input_files/input-145", - "gs://cromwell-perf-testing/lots_of_input_files/input-146", - "gs://cromwell-perf-testing/lots_of_input_files/input-147", - "gs://cromwell-perf-testing/lots_of_input_files/input-148", - "gs://cromwell-perf-testing/lots_of_input_files/input-149", - "gs://cromwell-perf-testing/lots_of_input_files/input-150", - "gs://cromwell-perf-testing/lots_of_input_files/input-151", - "gs://cromwell-perf-testing/lots_of_input_files/input-152", - "gs://cromwell-perf-testing/lots_of_input_files/input-153", - "gs://cromwell-perf-testing/lots_of_input_files/input-154", - "gs://cromwell-perf-testing/lots_of_input_files/input-155", - "gs://cromwell-perf-testing/lots_of_input_files/input-156", - "gs://cromwell-perf-testing/lots_of_input_files/input-157", - "gs://cromwell-perf-testing/lots_of_input_files/input-158", - "gs://cromwell-perf-testing/lots_of_input_files/input-159", - "gs://cromwell-perf-testing/lots_of_input_files/input-160", - "gs://cromwell-perf-testing/lots_of_input_files/input-161", - "gs://cromwell-perf-testing/lots_of_input_files/input-162", - "gs://cromwell-perf-testing/lots_of_input_files/input-163", - "gs://cromwell-perf-testing/lots_of_input_files/input-164", - "gs://cromwell-perf-testing/lots_of_input_files/input-165", - "gs://cromwell-perf-testing/lots_of_input_files/input-166", - "gs://cromwell-perf-testing/lots_of_input_files/input-167", - "gs://cromwell-perf-testing/lots_of_input_files/input-168", - "gs://cromwell-perf-testing/lots_of_input_files/input-169", - "gs://cromwell-perf-testing/lots_of_input_files/input-170", - "gs://cromwell-perf-testing/lots_of_input_files/input-171", - "gs://cromwell-perf-testing/lots_of_input_files/input-172", - "gs://cromwell-perf-testing/lots_of_input_files/input-173", - "gs://cromwell-perf-testing/lots_of_input_files/input-174", - "gs://cromwell-perf-testing/lots_of_input_files/input-175", - "gs://cromwell-perf-testing/lots_of_input_files/input-176", - "gs://cromwell-perf-testing/lots_of_input_files/input-177", - "gs://cromwell-perf-testing/lots_of_input_files/input-178", - "gs://cromwell-perf-testing/lots_of_input_files/input-179", - "gs://cromwell-perf-testing/lots_of_input_files/input-180", - "gs://cromwell-perf-testing/lots_of_input_files/input-181", - "gs://cromwell-perf-testing/lots_of_input_files/input-182", - "gs://cromwell-perf-testing/lots_of_input_files/input-183", - "gs://cromwell-perf-testing/lots_of_input_files/input-184", - "gs://cromwell-perf-testing/lots_of_input_files/input-185", - "gs://cromwell-perf-testing/lots_of_input_files/input-186", - "gs://cromwell-perf-testing/lots_of_input_files/input-187", - "gs://cromwell-perf-testing/lots_of_input_files/input-188", - "gs://cromwell-perf-testing/lots_of_input_files/input-189", - "gs://cromwell-perf-testing/lots_of_input_files/input-190", - "gs://cromwell-perf-testing/lots_of_input_files/input-191", - "gs://cromwell-perf-testing/lots_of_input_files/input-192", - "gs://cromwell-perf-testing/lots_of_input_files/input-193", - "gs://cromwell-perf-testing/lots_of_input_files/input-194", - "gs://cromwell-perf-testing/lots_of_input_files/input-195", - "gs://cromwell-perf-testing/lots_of_input_files/input-196", - "gs://cromwell-perf-testing/lots_of_input_files/input-197", - "gs://cromwell-perf-testing/lots_of_input_files/input-198", - "gs://cromwell-perf-testing/lots_of_input_files/input-199", - "gs://cromwell-perf-testing/lots_of_input_files/input-200", - "gs://cromwell-perf-testing/lots_of_input_files/input-201", - "gs://cromwell-perf-testing/lots_of_input_files/input-202", - "gs://cromwell-perf-testing/lots_of_input_files/input-203", - "gs://cromwell-perf-testing/lots_of_input_files/input-204", - "gs://cromwell-perf-testing/lots_of_input_files/input-205", - "gs://cromwell-perf-testing/lots_of_input_files/input-206", - "gs://cromwell-perf-testing/lots_of_input_files/input-207", - "gs://cromwell-perf-testing/lots_of_input_files/input-208", - "gs://cromwell-perf-testing/lots_of_input_files/input-209", - "gs://cromwell-perf-testing/lots_of_input_files/input-210", - "gs://cromwell-perf-testing/lots_of_input_files/input-211", - "gs://cromwell-perf-testing/lots_of_input_files/input-212", - "gs://cromwell-perf-testing/lots_of_input_files/input-213", - "gs://cromwell-perf-testing/lots_of_input_files/input-214", - "gs://cromwell-perf-testing/lots_of_input_files/input-215", - "gs://cromwell-perf-testing/lots_of_input_files/input-216", - "gs://cromwell-perf-testing/lots_of_input_files/input-217", - "gs://cromwell-perf-testing/lots_of_input_files/input-218", - "gs://cromwell-perf-testing/lots_of_input_files/input-219", - "gs://cromwell-perf-testing/lots_of_input_files/input-220", - "gs://cromwell-perf-testing/lots_of_input_files/input-221", - "gs://cromwell-perf-testing/lots_of_input_files/input-222", - "gs://cromwell-perf-testing/lots_of_input_files/input-223", - "gs://cromwell-perf-testing/lots_of_input_files/input-224", - "gs://cromwell-perf-testing/lots_of_input_files/input-225", - "gs://cromwell-perf-testing/lots_of_input_files/input-226", - "gs://cromwell-perf-testing/lots_of_input_files/input-227", - "gs://cromwell-perf-testing/lots_of_input_files/input-228", - "gs://cromwell-perf-testing/lots_of_input_files/input-229", - "gs://cromwell-perf-testing/lots_of_input_files/input-230", - "gs://cromwell-perf-testing/lots_of_input_files/input-231", - "gs://cromwell-perf-testing/lots_of_input_files/input-232", - "gs://cromwell-perf-testing/lots_of_input_files/input-233", - "gs://cromwell-perf-testing/lots_of_input_files/input-234", - "gs://cromwell-perf-testing/lots_of_input_files/input-235", - "gs://cromwell-perf-testing/lots_of_input_files/input-236", - "gs://cromwell-perf-testing/lots_of_input_files/input-237", - "gs://cromwell-perf-testing/lots_of_input_files/input-238", - "gs://cromwell-perf-testing/lots_of_input_files/input-239", - "gs://cromwell-perf-testing/lots_of_input_files/input-240", - "gs://cromwell-perf-testing/lots_of_input_files/input-241", - "gs://cromwell-perf-testing/lots_of_input_files/input-242", - "gs://cromwell-perf-testing/lots_of_input_files/input-243", - "gs://cromwell-perf-testing/lots_of_input_files/input-244", - "gs://cromwell-perf-testing/lots_of_input_files/input-245", - "gs://cromwell-perf-testing/lots_of_input_files/input-246", - "gs://cromwell-perf-testing/lots_of_input_files/input-247", - "gs://cromwell-perf-testing/lots_of_input_files/input-248", - "gs://cromwell-perf-testing/lots_of_input_files/input-249", - "gs://cromwell-perf-testing/lots_of_input_files/input-250", - "gs://cromwell-perf-testing/lots_of_input_files/input-251", - "gs://cromwell-perf-testing/lots_of_input_files/input-252", - "gs://cromwell-perf-testing/lots_of_input_files/input-253", - "gs://cromwell-perf-testing/lots_of_input_files/input-254", - "gs://cromwell-perf-testing/lots_of_input_files/input-255", - "gs://cromwell-perf-testing/lots_of_input_files/input-256", - "gs://cromwell-perf-testing/lots_of_input_files/input-257", - "gs://cromwell-perf-testing/lots_of_input_files/input-258", - "gs://cromwell-perf-testing/lots_of_input_files/input-259", - "gs://cromwell-perf-testing/lots_of_input_files/input-260", - "gs://cromwell-perf-testing/lots_of_input_files/input-261", - "gs://cromwell-perf-testing/lots_of_input_files/input-262", - "gs://cromwell-perf-testing/lots_of_input_files/input-263", - "gs://cromwell-perf-testing/lots_of_input_files/input-264", - "gs://cromwell-perf-testing/lots_of_input_files/input-265", - "gs://cromwell-perf-testing/lots_of_input_files/input-266", - "gs://cromwell-perf-testing/lots_of_input_files/input-267", - "gs://cromwell-perf-testing/lots_of_input_files/input-268", - "gs://cromwell-perf-testing/lots_of_input_files/input-269", - "gs://cromwell-perf-testing/lots_of_input_files/input-270", - "gs://cromwell-perf-testing/lots_of_input_files/input-271", - "gs://cromwell-perf-testing/lots_of_input_files/input-272", - "gs://cromwell-perf-testing/lots_of_input_files/input-273", - "gs://cromwell-perf-testing/lots_of_input_files/input-274", - "gs://cromwell-perf-testing/lots_of_input_files/input-275", - "gs://cromwell-perf-testing/lots_of_input_files/input-276", - "gs://cromwell-perf-testing/lots_of_input_files/input-277", - "gs://cromwell-perf-testing/lots_of_input_files/input-278", - "gs://cromwell-perf-testing/lots_of_input_files/input-279", - "gs://cromwell-perf-testing/lots_of_input_files/input-280", - "gs://cromwell-perf-testing/lots_of_input_files/input-281", - "gs://cromwell-perf-testing/lots_of_input_files/input-282", - "gs://cromwell-perf-testing/lots_of_input_files/input-283", - "gs://cromwell-perf-testing/lots_of_input_files/input-284", - "gs://cromwell-perf-testing/lots_of_input_files/input-285", - "gs://cromwell-perf-testing/lots_of_input_files/input-286", - "gs://cromwell-perf-testing/lots_of_input_files/input-287", - "gs://cromwell-perf-testing/lots_of_input_files/input-288", - "gs://cromwell-perf-testing/lots_of_input_files/input-289", - "gs://cromwell-perf-testing/lots_of_input_files/input-290", - "gs://cromwell-perf-testing/lots_of_input_files/input-291", - "gs://cromwell-perf-testing/lots_of_input_files/input-292", - "gs://cromwell-perf-testing/lots_of_input_files/input-293", - "gs://cromwell-perf-testing/lots_of_input_files/input-294", - "gs://cromwell-perf-testing/lots_of_input_files/input-295", - "gs://cromwell-perf-testing/lots_of_input_files/input-296", - "gs://cromwell-perf-testing/lots_of_input_files/input-297", - "gs://cromwell-perf-testing/lots_of_input_files/input-298", - "gs://cromwell-perf-testing/lots_of_input_files/input-299", - "gs://cromwell-perf-testing/lots_of_input_files/input-300", - "gs://cromwell-perf-testing/lots_of_input_files/input-301", - "gs://cromwell-perf-testing/lots_of_input_files/input-302", - "gs://cromwell-perf-testing/lots_of_input_files/input-303", - "gs://cromwell-perf-testing/lots_of_input_files/input-304", - "gs://cromwell-perf-testing/lots_of_input_files/input-305", - "gs://cromwell-perf-testing/lots_of_input_files/input-306", - "gs://cromwell-perf-testing/lots_of_input_files/input-307", - "gs://cromwell-perf-testing/lots_of_input_files/input-308", - "gs://cromwell-perf-testing/lots_of_input_files/input-309", - "gs://cromwell-perf-testing/lots_of_input_files/input-310", - "gs://cromwell-perf-testing/lots_of_input_files/input-311", - "gs://cromwell-perf-testing/lots_of_input_files/input-312", - "gs://cromwell-perf-testing/lots_of_input_files/input-313", - "gs://cromwell-perf-testing/lots_of_input_files/input-314", - "gs://cromwell-perf-testing/lots_of_input_files/input-315", - "gs://cromwell-perf-testing/lots_of_input_files/input-316", - "gs://cromwell-perf-testing/lots_of_input_files/input-317", - "gs://cromwell-perf-testing/lots_of_input_files/input-318", - "gs://cromwell-perf-testing/lots_of_input_files/input-319", - "gs://cromwell-perf-testing/lots_of_input_files/input-320", - "gs://cromwell-perf-testing/lots_of_input_files/input-321", - "gs://cromwell-perf-testing/lots_of_input_files/input-322", - "gs://cromwell-perf-testing/lots_of_input_files/input-323", - "gs://cromwell-perf-testing/lots_of_input_files/input-324", - "gs://cromwell-perf-testing/lots_of_input_files/input-325", - "gs://cromwell-perf-testing/lots_of_input_files/input-326", - "gs://cromwell-perf-testing/lots_of_input_files/input-327", - "gs://cromwell-perf-testing/lots_of_input_files/input-328", - "gs://cromwell-perf-testing/lots_of_input_files/input-329", - "gs://cromwell-perf-testing/lots_of_input_files/input-330", - "gs://cromwell-perf-testing/lots_of_input_files/input-331", - "gs://cromwell-perf-testing/lots_of_input_files/input-332", - "gs://cromwell-perf-testing/lots_of_input_files/input-333", - "gs://cromwell-perf-testing/lots_of_input_files/input-334", - "gs://cromwell-perf-testing/lots_of_input_files/input-335", - "gs://cromwell-perf-testing/lots_of_input_files/input-336", - "gs://cromwell-perf-testing/lots_of_input_files/input-337", - "gs://cromwell-perf-testing/lots_of_input_files/input-338", - "gs://cromwell-perf-testing/lots_of_input_files/input-339", - "gs://cromwell-perf-testing/lots_of_input_files/input-340", - "gs://cromwell-perf-testing/lots_of_input_files/input-341", - "gs://cromwell-perf-testing/lots_of_input_files/input-342", - "gs://cromwell-perf-testing/lots_of_input_files/input-343", - "gs://cromwell-perf-testing/lots_of_input_files/input-344", - "gs://cromwell-perf-testing/lots_of_input_files/input-345", - "gs://cromwell-perf-testing/lots_of_input_files/input-346", - "gs://cromwell-perf-testing/lots_of_input_files/input-347", - "gs://cromwell-perf-testing/lots_of_input_files/input-348", - "gs://cromwell-perf-testing/lots_of_input_files/input-349", - "gs://cromwell-perf-testing/lots_of_input_files/input-350", - "gs://cromwell-perf-testing/lots_of_input_files/input-351", - "gs://cromwell-perf-testing/lots_of_input_files/input-352", - "gs://cromwell-perf-testing/lots_of_input_files/input-353", - "gs://cromwell-perf-testing/lots_of_input_files/input-354", - "gs://cromwell-perf-testing/lots_of_input_files/input-355", - "gs://cromwell-perf-testing/lots_of_input_files/input-356", - "gs://cromwell-perf-testing/lots_of_input_files/input-357", - "gs://cromwell-perf-testing/lots_of_input_files/input-358", - "gs://cromwell-perf-testing/lots_of_input_files/input-359", - "gs://cromwell-perf-testing/lots_of_input_files/input-360", - "gs://cromwell-perf-testing/lots_of_input_files/input-361", - "gs://cromwell-perf-testing/lots_of_input_files/input-362", - "gs://cromwell-perf-testing/lots_of_input_files/input-363", - "gs://cromwell-perf-testing/lots_of_input_files/input-364", - "gs://cromwell-perf-testing/lots_of_input_files/input-365", - "gs://cromwell-perf-testing/lots_of_input_files/input-366", - "gs://cromwell-perf-testing/lots_of_input_files/input-367", - "gs://cromwell-perf-testing/lots_of_input_files/input-368", - "gs://cromwell-perf-testing/lots_of_input_files/input-369", - "gs://cromwell-perf-testing/lots_of_input_files/input-370", - "gs://cromwell-perf-testing/lots_of_input_files/input-371", - "gs://cromwell-perf-testing/lots_of_input_files/input-372", - "gs://cromwell-perf-testing/lots_of_input_files/input-373", - "gs://cromwell-perf-testing/lots_of_input_files/input-374", - "gs://cromwell-perf-testing/lots_of_input_files/input-375", - "gs://cromwell-perf-testing/lots_of_input_files/input-376", - "gs://cromwell-perf-testing/lots_of_input_files/input-377", - "gs://cromwell-perf-testing/lots_of_input_files/input-378", - "gs://cromwell-perf-testing/lots_of_input_files/input-379", - "gs://cromwell-perf-testing/lots_of_input_files/input-380", - "gs://cromwell-perf-testing/lots_of_input_files/input-381", - "gs://cromwell-perf-testing/lots_of_input_files/input-382", - "gs://cromwell-perf-testing/lots_of_input_files/input-383", - "gs://cromwell-perf-testing/lots_of_input_files/input-384", - "gs://cromwell-perf-testing/lots_of_input_files/input-385", - "gs://cromwell-perf-testing/lots_of_input_files/input-386", - "gs://cromwell-perf-testing/lots_of_input_files/input-387", - "gs://cromwell-perf-testing/lots_of_input_files/input-388", - "gs://cromwell-perf-testing/lots_of_input_files/input-389", - "gs://cromwell-perf-testing/lots_of_input_files/input-390", - "gs://cromwell-perf-testing/lots_of_input_files/input-391", - "gs://cromwell-perf-testing/lots_of_input_files/input-392", - "gs://cromwell-perf-testing/lots_of_input_files/input-393", - "gs://cromwell-perf-testing/lots_of_input_files/input-394", - "gs://cromwell-perf-testing/lots_of_input_files/input-395", - "gs://cromwell-perf-testing/lots_of_input_files/input-396", - "gs://cromwell-perf-testing/lots_of_input_files/input-397", - "gs://cromwell-perf-testing/lots_of_input_files/input-398", - "gs://cromwell-perf-testing/lots_of_input_files/input-399", - "gs://cromwell-perf-testing/lots_of_input_files/input-400", - "gs://cromwell-perf-testing/lots_of_input_files/input-401", - "gs://cromwell-perf-testing/lots_of_input_files/input-402", - "gs://cromwell-perf-testing/lots_of_input_files/input-403", - "gs://cromwell-perf-testing/lots_of_input_files/input-404", - "gs://cromwell-perf-testing/lots_of_input_files/input-405", - "gs://cromwell-perf-testing/lots_of_input_files/input-406", - "gs://cromwell-perf-testing/lots_of_input_files/input-407", - "gs://cromwell-perf-testing/lots_of_input_files/input-408", - "gs://cromwell-perf-testing/lots_of_input_files/input-409", - "gs://cromwell-perf-testing/lots_of_input_files/input-410", - "gs://cromwell-perf-testing/lots_of_input_files/input-411", - "gs://cromwell-perf-testing/lots_of_input_files/input-412", - "gs://cromwell-perf-testing/lots_of_input_files/input-413", - "gs://cromwell-perf-testing/lots_of_input_files/input-414", - "gs://cromwell-perf-testing/lots_of_input_files/input-415", - "gs://cromwell-perf-testing/lots_of_input_files/input-416", - "gs://cromwell-perf-testing/lots_of_input_files/input-417", - "gs://cromwell-perf-testing/lots_of_input_files/input-418", - "gs://cromwell-perf-testing/lots_of_input_files/input-419", - "gs://cromwell-perf-testing/lots_of_input_files/input-420", - "gs://cromwell-perf-testing/lots_of_input_files/input-421", - "gs://cromwell-perf-testing/lots_of_input_files/input-422", - "gs://cromwell-perf-testing/lots_of_input_files/input-423", - "gs://cromwell-perf-testing/lots_of_input_files/input-424", - "gs://cromwell-perf-testing/lots_of_input_files/input-425", - "gs://cromwell-perf-testing/lots_of_input_files/input-426", - "gs://cromwell-perf-testing/lots_of_input_files/input-427", - "gs://cromwell-perf-testing/lots_of_input_files/input-428", - "gs://cromwell-perf-testing/lots_of_input_files/input-429", - "gs://cromwell-perf-testing/lots_of_input_files/input-430", - "gs://cromwell-perf-testing/lots_of_input_files/input-431", - "gs://cromwell-perf-testing/lots_of_input_files/input-432", - "gs://cromwell-perf-testing/lots_of_input_files/input-433", - "gs://cromwell-perf-testing/lots_of_input_files/input-434", - "gs://cromwell-perf-testing/lots_of_input_files/input-435", - "gs://cromwell-perf-testing/lots_of_input_files/input-436", - "gs://cromwell-perf-testing/lots_of_input_files/input-437", - "gs://cromwell-perf-testing/lots_of_input_files/input-438", - "gs://cromwell-perf-testing/lots_of_input_files/input-439", - "gs://cromwell-perf-testing/lots_of_input_files/input-440", - "gs://cromwell-perf-testing/lots_of_input_files/input-441", - "gs://cromwell-perf-testing/lots_of_input_files/input-442", - "gs://cromwell-perf-testing/lots_of_input_files/input-443", - "gs://cromwell-perf-testing/lots_of_input_files/input-444", - "gs://cromwell-perf-testing/lots_of_input_files/input-445", - "gs://cromwell-perf-testing/lots_of_input_files/input-446", - "gs://cromwell-perf-testing/lots_of_input_files/input-447", - "gs://cromwell-perf-testing/lots_of_input_files/input-448", - "gs://cromwell-perf-testing/lots_of_input_files/input-449", - "gs://cromwell-perf-testing/lots_of_input_files/input-450", - "gs://cromwell-perf-testing/lots_of_input_files/input-451", - "gs://cromwell-perf-testing/lots_of_input_files/input-452", - "gs://cromwell-perf-testing/lots_of_input_files/input-453", - "gs://cromwell-perf-testing/lots_of_input_files/input-454", - "gs://cromwell-perf-testing/lots_of_input_files/input-455", - "gs://cromwell-perf-testing/lots_of_input_files/input-456", - "gs://cromwell-perf-testing/lots_of_input_files/input-457", - "gs://cromwell-perf-testing/lots_of_input_files/input-458", - "gs://cromwell-perf-testing/lots_of_input_files/input-459", - "gs://cromwell-perf-testing/lots_of_input_files/input-460", - "gs://cromwell-perf-testing/lots_of_input_files/input-461", - "gs://cromwell-perf-testing/lots_of_input_files/input-462", - "gs://cromwell-perf-testing/lots_of_input_files/input-463", - "gs://cromwell-perf-testing/lots_of_input_files/input-464", - "gs://cromwell-perf-testing/lots_of_input_files/input-465", - "gs://cromwell-perf-testing/lots_of_input_files/input-466", - "gs://cromwell-perf-testing/lots_of_input_files/input-467", - "gs://cromwell-perf-testing/lots_of_input_files/input-468", - "gs://cromwell-perf-testing/lots_of_input_files/input-469", - "gs://cromwell-perf-testing/lots_of_input_files/input-470", - "gs://cromwell-perf-testing/lots_of_input_files/input-471", - "gs://cromwell-perf-testing/lots_of_input_files/input-472", - "gs://cromwell-perf-testing/lots_of_input_files/input-473", - "gs://cromwell-perf-testing/lots_of_input_files/input-474", - "gs://cromwell-perf-testing/lots_of_input_files/input-475", - "gs://cromwell-perf-testing/lots_of_input_files/input-476", - "gs://cromwell-perf-testing/lots_of_input_files/input-477", - "gs://cromwell-perf-testing/lots_of_input_files/input-478", - "gs://cromwell-perf-testing/lots_of_input_files/input-479", - "gs://cromwell-perf-testing/lots_of_input_files/input-480", - "gs://cromwell-perf-testing/lots_of_input_files/input-481", - "gs://cromwell-perf-testing/lots_of_input_files/input-482", - "gs://cromwell-perf-testing/lots_of_input_files/input-483", - "gs://cromwell-perf-testing/lots_of_input_files/input-484", - "gs://cromwell-perf-testing/lots_of_input_files/input-485", - "gs://cromwell-perf-testing/lots_of_input_files/input-486", - "gs://cromwell-perf-testing/lots_of_input_files/input-487", - "gs://cromwell-perf-testing/lots_of_input_files/input-488", - "gs://cromwell-perf-testing/lots_of_input_files/input-489", - "gs://cromwell-perf-testing/lots_of_input_files/input-490", - "gs://cromwell-perf-testing/lots_of_input_files/input-491", - "gs://cromwell-perf-testing/lots_of_input_files/input-492", - "gs://cromwell-perf-testing/lots_of_input_files/input-493", - "gs://cromwell-perf-testing/lots_of_input_files/input-494", - "gs://cromwell-perf-testing/lots_of_input_files/input-495", - "gs://cromwell-perf-testing/lots_of_input_files/input-496", - "gs://cromwell-perf-testing/lots_of_input_files/input-497", - "gs://cromwell-perf-testing/lots_of_input_files/input-498", - "gs://cromwell-perf-testing/lots_of_input_files/input-499", - "gs://cromwell-perf-testing/lots_of_input_files/input-500", - "gs://cromwell-perf-testing/lots_of_input_files/input-501", - "gs://cromwell-perf-testing/lots_of_input_files/input-502", - "gs://cromwell-perf-testing/lots_of_input_files/input-503", - "gs://cromwell-perf-testing/lots_of_input_files/input-504", - "gs://cromwell-perf-testing/lots_of_input_files/input-505", - "gs://cromwell-perf-testing/lots_of_input_files/input-506", - "gs://cromwell-perf-testing/lots_of_input_files/input-507", - "gs://cromwell-perf-testing/lots_of_input_files/input-508", - "gs://cromwell-perf-testing/lots_of_input_files/input-509", - "gs://cromwell-perf-testing/lots_of_input_files/input-510", - "gs://cromwell-perf-testing/lots_of_input_files/input-511", - "gs://cromwell-perf-testing/lots_of_input_files/input-512", - "gs://cromwell-perf-testing/lots_of_input_files/input-513", - "gs://cromwell-perf-testing/lots_of_input_files/input-514", - "gs://cromwell-perf-testing/lots_of_input_files/input-515", - "gs://cromwell-perf-testing/lots_of_input_files/input-516", - "gs://cromwell-perf-testing/lots_of_input_files/input-517", - "gs://cromwell-perf-testing/lots_of_input_files/input-518", - "gs://cromwell-perf-testing/lots_of_input_files/input-519", - "gs://cromwell-perf-testing/lots_of_input_files/input-520", - "gs://cromwell-perf-testing/lots_of_input_files/input-521", - "gs://cromwell-perf-testing/lots_of_input_files/input-522", - "gs://cromwell-perf-testing/lots_of_input_files/input-523", - "gs://cromwell-perf-testing/lots_of_input_files/input-524", - "gs://cromwell-perf-testing/lots_of_input_files/input-525", - "gs://cromwell-perf-testing/lots_of_input_files/input-526", - "gs://cromwell-perf-testing/lots_of_input_files/input-527", - "gs://cromwell-perf-testing/lots_of_input_files/input-528", - "gs://cromwell-perf-testing/lots_of_input_files/input-529", - "gs://cromwell-perf-testing/lots_of_input_files/input-530", - "gs://cromwell-perf-testing/lots_of_input_files/input-531", - "gs://cromwell-perf-testing/lots_of_input_files/input-532", - "gs://cromwell-perf-testing/lots_of_input_files/input-533", - "gs://cromwell-perf-testing/lots_of_input_files/input-534", - "gs://cromwell-perf-testing/lots_of_input_files/input-535", - "gs://cromwell-perf-testing/lots_of_input_files/input-536", - "gs://cromwell-perf-testing/lots_of_input_files/input-537", - "gs://cromwell-perf-testing/lots_of_input_files/input-538", - "gs://cromwell-perf-testing/lots_of_input_files/input-539", - "gs://cromwell-perf-testing/lots_of_input_files/input-540", - "gs://cromwell-perf-testing/lots_of_input_files/input-541", - "gs://cromwell-perf-testing/lots_of_input_files/input-542", - "gs://cromwell-perf-testing/lots_of_input_files/input-543", - "gs://cromwell-perf-testing/lots_of_input_files/input-544", - "gs://cromwell-perf-testing/lots_of_input_files/input-545", - "gs://cromwell-perf-testing/lots_of_input_files/input-546", - "gs://cromwell-perf-testing/lots_of_input_files/input-547", - "gs://cromwell-perf-testing/lots_of_input_files/input-548", - "gs://cromwell-perf-testing/lots_of_input_files/input-549", - "gs://cromwell-perf-testing/lots_of_input_files/input-550", - "gs://cromwell-perf-testing/lots_of_input_files/input-551", - "gs://cromwell-perf-testing/lots_of_input_files/input-552", - "gs://cromwell-perf-testing/lots_of_input_files/input-553", - "gs://cromwell-perf-testing/lots_of_input_files/input-554", - "gs://cromwell-perf-testing/lots_of_input_files/input-555", - "gs://cromwell-perf-testing/lots_of_input_files/input-556", - "gs://cromwell-perf-testing/lots_of_input_files/input-557", - "gs://cromwell-perf-testing/lots_of_input_files/input-558", - "gs://cromwell-perf-testing/lots_of_input_files/input-559", - "gs://cromwell-perf-testing/lots_of_input_files/input-560", - "gs://cromwell-perf-testing/lots_of_input_files/input-561", - "gs://cromwell-perf-testing/lots_of_input_files/input-562", - "gs://cromwell-perf-testing/lots_of_input_files/input-563", - "gs://cromwell-perf-testing/lots_of_input_files/input-564", - "gs://cromwell-perf-testing/lots_of_input_files/input-565", - "gs://cromwell-perf-testing/lots_of_input_files/input-566", - "gs://cromwell-perf-testing/lots_of_input_files/input-567", - "gs://cromwell-perf-testing/lots_of_input_files/input-568", - "gs://cromwell-perf-testing/lots_of_input_files/input-569", - "gs://cromwell-perf-testing/lots_of_input_files/input-570", - "gs://cromwell-perf-testing/lots_of_input_files/input-571", - "gs://cromwell-perf-testing/lots_of_input_files/input-572", - "gs://cromwell-perf-testing/lots_of_input_files/input-573", - "gs://cromwell-perf-testing/lots_of_input_files/input-574", - "gs://cromwell-perf-testing/lots_of_input_files/input-575", - "gs://cromwell-perf-testing/lots_of_input_files/input-576", - "gs://cromwell-perf-testing/lots_of_input_files/input-577", - "gs://cromwell-perf-testing/lots_of_input_files/input-578", - "gs://cromwell-perf-testing/lots_of_input_files/input-579", - "gs://cromwell-perf-testing/lots_of_input_files/input-580", - "gs://cromwell-perf-testing/lots_of_input_files/input-581", - "gs://cromwell-perf-testing/lots_of_input_files/input-582", - "gs://cromwell-perf-testing/lots_of_input_files/input-583", - "gs://cromwell-perf-testing/lots_of_input_files/input-584", - "gs://cromwell-perf-testing/lots_of_input_files/input-585", - "gs://cromwell-perf-testing/lots_of_input_files/input-586", - "gs://cromwell-perf-testing/lots_of_input_files/input-587", - "gs://cromwell-perf-testing/lots_of_input_files/input-588", - "gs://cromwell-perf-testing/lots_of_input_files/input-589", - "gs://cromwell-perf-testing/lots_of_input_files/input-590", - "gs://cromwell-perf-testing/lots_of_input_files/input-591", - "gs://cromwell-perf-testing/lots_of_input_files/input-592", - "gs://cromwell-perf-testing/lots_of_input_files/input-593", - "gs://cromwell-perf-testing/lots_of_input_files/input-594", - "gs://cromwell-perf-testing/lots_of_input_files/input-595", - "gs://cromwell-perf-testing/lots_of_input_files/input-596", - "gs://cromwell-perf-testing/lots_of_input_files/input-597", - "gs://cromwell-perf-testing/lots_of_input_files/input-598", - "gs://cromwell-perf-testing/lots_of_input_files/input-599", - "gs://cromwell-perf-testing/lots_of_input_files/input-600", - "gs://cromwell-perf-testing/lots_of_input_files/input-601", - "gs://cromwell-perf-testing/lots_of_input_files/input-602", - "gs://cromwell-perf-testing/lots_of_input_files/input-603", - "gs://cromwell-perf-testing/lots_of_input_files/input-604", - "gs://cromwell-perf-testing/lots_of_input_files/input-605", - "gs://cromwell-perf-testing/lots_of_input_files/input-606", - "gs://cromwell-perf-testing/lots_of_input_files/input-607", - "gs://cromwell-perf-testing/lots_of_input_files/input-608", - "gs://cromwell-perf-testing/lots_of_input_files/input-609", - "gs://cromwell-perf-testing/lots_of_input_files/input-610", - "gs://cromwell-perf-testing/lots_of_input_files/input-611", - "gs://cromwell-perf-testing/lots_of_input_files/input-612", - "gs://cromwell-perf-testing/lots_of_input_files/input-613", - "gs://cromwell-perf-testing/lots_of_input_files/input-614", - "gs://cromwell-perf-testing/lots_of_input_files/input-615", - "gs://cromwell-perf-testing/lots_of_input_files/input-616", - "gs://cromwell-perf-testing/lots_of_input_files/input-617", - "gs://cromwell-perf-testing/lots_of_input_files/input-618", - "gs://cromwell-perf-testing/lots_of_input_files/input-619", - "gs://cromwell-perf-testing/lots_of_input_files/input-620", - "gs://cromwell-perf-testing/lots_of_input_files/input-621", - "gs://cromwell-perf-testing/lots_of_input_files/input-622", - "gs://cromwell-perf-testing/lots_of_input_files/input-623", - "gs://cromwell-perf-testing/lots_of_input_files/input-624", - "gs://cromwell-perf-testing/lots_of_input_files/input-625", - "gs://cromwell-perf-testing/lots_of_input_files/input-626", - "gs://cromwell-perf-testing/lots_of_input_files/input-627", - "gs://cromwell-perf-testing/lots_of_input_files/input-628", - "gs://cromwell-perf-testing/lots_of_input_files/input-629", - "gs://cromwell-perf-testing/lots_of_input_files/input-630", - "gs://cromwell-perf-testing/lots_of_input_files/input-631", - "gs://cromwell-perf-testing/lots_of_input_files/input-632", - "gs://cromwell-perf-testing/lots_of_input_files/input-633", - "gs://cromwell-perf-testing/lots_of_input_files/input-634", - "gs://cromwell-perf-testing/lots_of_input_files/input-635", - "gs://cromwell-perf-testing/lots_of_input_files/input-636", - "gs://cromwell-perf-testing/lots_of_input_files/input-637", - "gs://cromwell-perf-testing/lots_of_input_files/input-638", - "gs://cromwell-perf-testing/lots_of_input_files/input-639", - "gs://cromwell-perf-testing/lots_of_input_files/input-640", - "gs://cromwell-perf-testing/lots_of_input_files/input-641", - "gs://cromwell-perf-testing/lots_of_input_files/input-642", - "gs://cromwell-perf-testing/lots_of_input_files/input-643", - "gs://cromwell-perf-testing/lots_of_input_files/input-644", - "gs://cromwell-perf-testing/lots_of_input_files/input-645", - "gs://cromwell-perf-testing/lots_of_input_files/input-646", - "gs://cromwell-perf-testing/lots_of_input_files/input-647", - "gs://cromwell-perf-testing/lots_of_input_files/input-648", - "gs://cromwell-perf-testing/lots_of_input_files/input-649", - "gs://cromwell-perf-testing/lots_of_input_files/input-650", - "gs://cromwell-perf-testing/lots_of_input_files/input-651", - "gs://cromwell-perf-testing/lots_of_input_files/input-652", - "gs://cromwell-perf-testing/lots_of_input_files/input-653", - "gs://cromwell-perf-testing/lots_of_input_files/input-654", - "gs://cromwell-perf-testing/lots_of_input_files/input-655", - "gs://cromwell-perf-testing/lots_of_input_files/input-656", - "gs://cromwell-perf-testing/lots_of_input_files/input-657", - "gs://cromwell-perf-testing/lots_of_input_files/input-658", - "gs://cromwell-perf-testing/lots_of_input_files/input-659", - "gs://cromwell-perf-testing/lots_of_input_files/input-660", - "gs://cromwell-perf-testing/lots_of_input_files/input-661", - "gs://cromwell-perf-testing/lots_of_input_files/input-662", - "gs://cromwell-perf-testing/lots_of_input_files/input-663", - "gs://cromwell-perf-testing/lots_of_input_files/input-664", - "gs://cromwell-perf-testing/lots_of_input_files/input-665", - "gs://cromwell-perf-testing/lots_of_input_files/input-666", - "gs://cromwell-perf-testing/lots_of_input_files/input-667", - "gs://cromwell-perf-testing/lots_of_input_files/input-668", - "gs://cromwell-perf-testing/lots_of_input_files/input-669", - "gs://cromwell-perf-testing/lots_of_input_files/input-670", - "gs://cromwell-perf-testing/lots_of_input_files/input-671", - "gs://cromwell-perf-testing/lots_of_input_files/input-672", - "gs://cromwell-perf-testing/lots_of_input_files/input-673", - "gs://cromwell-perf-testing/lots_of_input_files/input-674", - "gs://cromwell-perf-testing/lots_of_input_files/input-675", - "gs://cromwell-perf-testing/lots_of_input_files/input-676", - "gs://cromwell-perf-testing/lots_of_input_files/input-677", - "gs://cromwell-perf-testing/lots_of_input_files/input-678", - "gs://cromwell-perf-testing/lots_of_input_files/input-679", - "gs://cromwell-perf-testing/lots_of_input_files/input-680", - "gs://cromwell-perf-testing/lots_of_input_files/input-681", - "gs://cromwell-perf-testing/lots_of_input_files/input-682", - "gs://cromwell-perf-testing/lots_of_input_files/input-683", - "gs://cromwell-perf-testing/lots_of_input_files/input-684", - "gs://cromwell-perf-testing/lots_of_input_files/input-685", - "gs://cromwell-perf-testing/lots_of_input_files/input-686", - "gs://cromwell-perf-testing/lots_of_input_files/input-687", - "gs://cromwell-perf-testing/lots_of_input_files/input-688", - "gs://cromwell-perf-testing/lots_of_input_files/input-689", - "gs://cromwell-perf-testing/lots_of_input_files/input-690", - "gs://cromwell-perf-testing/lots_of_input_files/input-691", - "gs://cromwell-perf-testing/lots_of_input_files/input-692", - "gs://cromwell-perf-testing/lots_of_input_files/input-693", - "gs://cromwell-perf-testing/lots_of_input_files/input-694", - "gs://cromwell-perf-testing/lots_of_input_files/input-695", - "gs://cromwell-perf-testing/lots_of_input_files/input-696", - "gs://cromwell-perf-testing/lots_of_input_files/input-697", - "gs://cromwell-perf-testing/lots_of_input_files/input-698", - "gs://cromwell-perf-testing/lots_of_input_files/input-699", - "gs://cromwell-perf-testing/lots_of_input_files/input-700", - "gs://cromwell-perf-testing/lots_of_input_files/input-701", - "gs://cromwell-perf-testing/lots_of_input_files/input-702", - "gs://cromwell-perf-testing/lots_of_input_files/input-703", - "gs://cromwell-perf-testing/lots_of_input_files/input-704", - "gs://cromwell-perf-testing/lots_of_input_files/input-705", - "gs://cromwell-perf-testing/lots_of_input_files/input-706", - "gs://cromwell-perf-testing/lots_of_input_files/input-707", - "gs://cromwell-perf-testing/lots_of_input_files/input-708", - "gs://cromwell-perf-testing/lots_of_input_files/input-709", - "gs://cromwell-perf-testing/lots_of_input_files/input-710", - "gs://cromwell-perf-testing/lots_of_input_files/input-711", - "gs://cromwell-perf-testing/lots_of_input_files/input-712", - "gs://cromwell-perf-testing/lots_of_input_files/input-713", - "gs://cromwell-perf-testing/lots_of_input_files/input-714", - "gs://cromwell-perf-testing/lots_of_input_files/input-715", - "gs://cromwell-perf-testing/lots_of_input_files/input-716", - "gs://cromwell-perf-testing/lots_of_input_files/input-717", - "gs://cromwell-perf-testing/lots_of_input_files/input-718", - "gs://cromwell-perf-testing/lots_of_input_files/input-719", - "gs://cromwell-perf-testing/lots_of_input_files/input-720", - "gs://cromwell-perf-testing/lots_of_input_files/input-721", - "gs://cromwell-perf-testing/lots_of_input_files/input-722", - "gs://cromwell-perf-testing/lots_of_input_files/input-723", - "gs://cromwell-perf-testing/lots_of_input_files/input-724", - "gs://cromwell-perf-testing/lots_of_input_files/input-725", - "gs://cromwell-perf-testing/lots_of_input_files/input-726", - "gs://cromwell-perf-testing/lots_of_input_files/input-727", - "gs://cromwell-perf-testing/lots_of_input_files/input-728", - "gs://cromwell-perf-testing/lots_of_input_files/input-729", - "gs://cromwell-perf-testing/lots_of_input_files/input-730", - "gs://cromwell-perf-testing/lots_of_input_files/input-731", - "gs://cromwell-perf-testing/lots_of_input_files/input-732", - "gs://cromwell-perf-testing/lots_of_input_files/input-733", - "gs://cromwell-perf-testing/lots_of_input_files/input-734", - "gs://cromwell-perf-testing/lots_of_input_files/input-735", - "gs://cromwell-perf-testing/lots_of_input_files/input-736", - "gs://cromwell-perf-testing/lots_of_input_files/input-737", - "gs://cromwell-perf-testing/lots_of_input_files/input-738", - "gs://cromwell-perf-testing/lots_of_input_files/input-739", - "gs://cromwell-perf-testing/lots_of_input_files/input-740", - "gs://cromwell-perf-testing/lots_of_input_files/input-741", - "gs://cromwell-perf-testing/lots_of_input_files/input-742", - "gs://cromwell-perf-testing/lots_of_input_files/input-743", - "gs://cromwell-perf-testing/lots_of_input_files/input-744", - "gs://cromwell-perf-testing/lots_of_input_files/input-745", - "gs://cromwell-perf-testing/lots_of_input_files/input-746", - "gs://cromwell-perf-testing/lots_of_input_files/input-747", - "gs://cromwell-perf-testing/lots_of_input_files/input-748", - "gs://cromwell-perf-testing/lots_of_input_files/input-749", - "gs://cromwell-perf-testing/lots_of_input_files/input-750", - "gs://cromwell-perf-testing/lots_of_input_files/input-751", - "gs://cromwell-perf-testing/lots_of_input_files/input-752", - "gs://cromwell-perf-testing/lots_of_input_files/input-753", - "gs://cromwell-perf-testing/lots_of_input_files/input-754", - "gs://cromwell-perf-testing/lots_of_input_files/input-755", - "gs://cromwell-perf-testing/lots_of_input_files/input-756", - "gs://cromwell-perf-testing/lots_of_input_files/input-757", - "gs://cromwell-perf-testing/lots_of_input_files/input-758", - "gs://cromwell-perf-testing/lots_of_input_files/input-759", - "gs://cromwell-perf-testing/lots_of_input_files/input-760", - "gs://cromwell-perf-testing/lots_of_input_files/input-761", - "gs://cromwell-perf-testing/lots_of_input_files/input-762", - "gs://cromwell-perf-testing/lots_of_input_files/input-763", - "gs://cromwell-perf-testing/lots_of_input_files/input-764", - "gs://cromwell-perf-testing/lots_of_input_files/input-765", - "gs://cromwell-perf-testing/lots_of_input_files/input-766", - "gs://cromwell-perf-testing/lots_of_input_files/input-767", - "gs://cromwell-perf-testing/lots_of_input_files/input-768", - "gs://cromwell-perf-testing/lots_of_input_files/input-769", - "gs://cromwell-perf-testing/lots_of_input_files/input-770", - "gs://cromwell-perf-testing/lots_of_input_files/input-771", - "gs://cromwell-perf-testing/lots_of_input_files/input-772", - "gs://cromwell-perf-testing/lots_of_input_files/input-773", - "gs://cromwell-perf-testing/lots_of_input_files/input-774", - "gs://cromwell-perf-testing/lots_of_input_files/input-775", - "gs://cromwell-perf-testing/lots_of_input_files/input-776", - "gs://cromwell-perf-testing/lots_of_input_files/input-777", - "gs://cromwell-perf-testing/lots_of_input_files/input-778", - "gs://cromwell-perf-testing/lots_of_input_files/input-779", - "gs://cromwell-perf-testing/lots_of_input_files/input-780", - "gs://cromwell-perf-testing/lots_of_input_files/input-781", - "gs://cromwell-perf-testing/lots_of_input_files/input-782", - "gs://cromwell-perf-testing/lots_of_input_files/input-783", - "gs://cromwell-perf-testing/lots_of_input_files/input-784", - "gs://cromwell-perf-testing/lots_of_input_files/input-785", - "gs://cromwell-perf-testing/lots_of_input_files/input-786", - "gs://cromwell-perf-testing/lots_of_input_files/input-787", - "gs://cromwell-perf-testing/lots_of_input_files/input-788", - "gs://cromwell-perf-testing/lots_of_input_files/input-789", - "gs://cromwell-perf-testing/lots_of_input_files/input-790", - "gs://cromwell-perf-testing/lots_of_input_files/input-791", - "gs://cromwell-perf-testing/lots_of_input_files/input-792", - "gs://cromwell-perf-testing/lots_of_input_files/input-793", - "gs://cromwell-perf-testing/lots_of_input_files/input-794", - "gs://cromwell-perf-testing/lots_of_input_files/input-795", - "gs://cromwell-perf-testing/lots_of_input_files/input-796", - "gs://cromwell-perf-testing/lots_of_input_files/input-797", - "gs://cromwell-perf-testing/lots_of_input_files/input-798", - "gs://cromwell-perf-testing/lots_of_input_files/input-799", - "gs://cromwell-perf-testing/lots_of_input_files/input-800" - ] -} \ No newline at end of file diff --git a/scripts/perf/test_cases/call_caching_lots_of_inputs/lots_of_inputs.wdl b/scripts/perf/test_cases/call_caching_lots_of_inputs/lots_of_inputs.wdl deleted file mode 100644 index 961a3b36891..00000000000 --- a/scripts/perf/test_cases/call_caching_lots_of_inputs/lots_of_inputs.wdl +++ /dev/null @@ -1,28 +0,0 @@ -version 1.0 - -task lots_of_inputs { - input { - Array[File] files - } - - command { - echo ~{sep=' ' files} - } - - runtime { - docker: "ubuntu@sha256:de774a3145f7ca4f0bd144c7d4ffb2931e06634f11529653b23eba85aef8e378" - } - - output { - File out = stdout() - } -} - -workflow call_cache_read_input_files_perf_test { - input { - Array[File] files - } - scatter(i in range(100)) { - call lots_of_inputs { input: files = files } - } -} diff --git a/scripts/perf/test_cases/call_caching_stress_no_hint/call_caching_stress_no_hint.test b/scripts/perf/test_cases/call_caching_stress_no_hint/call_caching_stress_no_hint.test deleted file mode 100644 index 41fdcc6858f..00000000000 --- a/scripts/perf/test_cases/call_caching_stress_no_hint/call_caching_stress_no_hint.test +++ /dev/null @@ -1,12 +0,0 @@ -name: call_caching_stress_no_hint -testFormat: workflowsuccess - -files { - workflow: "../call_caching_stress_workflow/callCachingStress.wdl" - inputs: "../call_caching_stress_workflow/callCachingStress.inputs" - options: "gs://cromwell-perf-test/workflow_related_files/call_caching_sa2_workflow_options.json" -} - -metadata { - status: Succeeded -} diff --git a/scripts/perf/test_cases/call_caching_stress_no_hint/centaur.conf b/scripts/perf/test_cases/call_caching_stress_no_hint/centaur.conf deleted file mode 100644 index 2b7ce0eed62..00000000000 --- a/scripts/perf/test_cases/call_caching_stress_no_hint/centaur.conf +++ /dev/null @@ -1,18 +0,0 @@ -include required("centaur_default.conf") - -centaur { - metadata-args { - includeKey = [ - "id", - "%shardIndex%", - "%hitFailures%", - "%executionEvents%", - "status", - "workflowName", - "submission", - "start", - "end", - "failures" - ] - } -} diff --git a/scripts/perf/test_cases/call_caching_stress_no_hint/cromwell.conf b/scripts/perf/test_cases/call_caching_stress_no_hint/cromwell.conf deleted file mode 100644 index 0ce40f48486..00000000000 --- a/scripts/perf/test_cases/call_caching_stress_no_hint/cromwell.conf +++ /dev/null @@ -1,5 +0,0 @@ -include required("cromwell_default.conf") - -engine.filesystems.gcs.auth = "user-service-account" -backend.providers.Papi.config.genomics.auth = "user-service-account" -backend.providers.Papi.config.filesystems.gcs.auth = "user-service-account" diff --git a/scripts/perf/test_cases/call_caching_stress_with_hint/call_caching_stress_with_hint.test b/scripts/perf/test_cases/call_caching_stress_with_hint/call_caching_stress_with_hint.test deleted file mode 100644 index f8a02e989d5..00000000000 --- a/scripts/perf/test_cases/call_caching_stress_with_hint/call_caching_stress_with_hint.test +++ /dev/null @@ -1,12 +0,0 @@ -name: call_caching_stress_with_hint -testFormat: workflowsuccess - -files { - workflow: "https://raw.githubusercontent.com/broadinstitute/cromwell/$BRANCH/scripts/perf/test_cases/call_caching_stress_workflow/callCachingStress.wdl" - inputs: "https://raw.githubusercontent.com/broadinstitute/cromwell/$BRANCH/scripts/perf/test_cases/call_caching_stress_workflow/callCachingStress.inputs" - options: "gs://cromwell-perf-test/workflow_related_files/call_caching_sa2_options_with_cache_hints.json" -} - -metadata { - status: Succeeded -} diff --git a/scripts/perf/test_cases/call_caching_stress_with_hint/centaur.conf b/scripts/perf/test_cases/call_caching_stress_with_hint/centaur.conf deleted file mode 100644 index 2b7ce0eed62..00000000000 --- a/scripts/perf/test_cases/call_caching_stress_with_hint/centaur.conf +++ /dev/null @@ -1,18 +0,0 @@ -include required("centaur_default.conf") - -centaur { - metadata-args { - includeKey = [ - "id", - "%shardIndex%", - "%hitFailures%", - "%executionEvents%", - "status", - "workflowName", - "submission", - "start", - "end", - "failures" - ] - } -} diff --git a/scripts/perf/test_cases/call_caching_stress_with_hint/cromwell.conf b/scripts/perf/test_cases/call_caching_stress_with_hint/cromwell.conf deleted file mode 100644 index 0ce40f48486..00000000000 --- a/scripts/perf/test_cases/call_caching_stress_with_hint/cromwell.conf +++ /dev/null @@ -1,5 +0,0 @@ -include required("cromwell_default.conf") - -engine.filesystems.gcs.auth = "user-service-account" -backend.providers.Papi.config.genomics.auth = "user-service-account" -backend.providers.Papi.config.filesystems.gcs.auth = "user-service-account" diff --git a/scripts/perf/test_cases/call_caching_stress_workflow/callCachingStress.inputs b/scripts/perf/test_cases/call_caching_stress_workflow/callCachingStress.inputs deleted file mode 100644 index 281858ee08d..00000000000 --- a/scripts/perf/test_cases/call_caching_stress_workflow/callCachingStress.inputs +++ /dev/null @@ -1,14 +0,0 @@ -{ - "callCachingStress.input_files": [ - "gs://call-caching-testing-sa1/inputs/input1.txt", - "gs://call-caching-testing-sa1/inputs/input2.txt", - "gs://call-caching-testing-sa1/inputs/input3.txt", - "gs://call-caching-testing-sa1/inputs/input4.txt", - "gs://call-caching-testing-sa1/inputs/input5.txt", - "gs://call-caching-testing-sa1/inputs/input6.txt", - "gs://call-caching-testing-sa1/inputs/input7.txt", - "gs://call-caching-testing-sa1/inputs/input8.txt", - "gs://call-caching-testing-sa1/inputs/input9.txt", - "gs://call-caching-testing-sa1/inputs/input10.txt" - ] -} diff --git a/scripts/perf/test_cases/call_caching_stress_workflow/callCachingStress.wdl b/scripts/perf/test_cases/call_caching_stress_workflow/callCachingStress.wdl deleted file mode 100644 index c61f55ecd35..00000000000 --- a/scripts/perf/test_cases/call_caching_stress_workflow/callCachingStress.wdl +++ /dev/null @@ -1,32 +0,0 @@ -version 1.0 - -#NOTE: Please do not change the spelling for 'nb_ouptputs'. This workflow is used for Call Cache test in Perf. -# Since there are cache entries inside test db with wrong spelling, correcting the spelling -# will fail the test as it won't be call cached anymore! -task simpleTask { - input { - Int seed - Int nb_ouptputs = 10 - Array[File] input_files - } - command { - for i in `seq ~{nb_ouptputs}`; do echo $i > $i.txt; done - } - runtime { - docker: "ubuntu@sha256:3f119dc0737f57f704ebecac8a6d8477b0f6ca1ca0332c7ee1395ed2c6a82be7" - } - output { - Array[File] outputs = glob("*.txt") - } -} - -workflow callCachingStress { - input { - Int scatterSize = 1000 - Array[File] input_files - } - - scatter(i in range(scatterSize)) { - call simpleTask { input: input_files = input_files, seed = i } - } -} diff --git a/scripts/perf/test_cases/five_dollar_genome/five_dollar_genome.inputs b/scripts/perf/test_cases/five_dollar_genome/five_dollar_genome.inputs deleted file mode 100644 index 31365490649..00000000000 --- a/scripts/perf/test_cases/five_dollar_genome/five_dollar_genome.inputs +++ /dev/null @@ -1,39 +0,0 @@ -{ - "germline_single_sample_workflow.ref_fasta_index": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.fasta.fai", - "germline_single_sample_workflow.final_vcf_base_name": "NA12878_small", - "germline_single_sample_workflow.skip_QC": "False", - "germline_single_sample_workflow.wgs_evaluation_interval_list": "gs://gcp-public-data--broad-references/hg38/v0/wgs_evaluation_regions.hg38.interval_list", - "germline_single_sample_workflow.ref_dict": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.dict", - "germline_single_sample_workflow.contamination_sites_ud": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.contam.UD", - "germline_single_sample_workflow.ref_amb": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.fasta.64.amb", - "germline_single_sample_workflow.sample_name": "NA12878_small", - "germline_single_sample_workflow.wgs_calling_interval_list": "gs://gcp-public-data--broad-references/hg38/v0/wgs_calling_regions.hg38.interval_list", - "germline_single_sample_workflow.agg_preemptible_tries": 3, - "germline_single_sample_workflow.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.dbsnp138.vcf.idx", - "germline_single_sample_workflow.preemptible_tries": 3, - "germline_single_sample_workflow.contamination_sites_mu": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.contam.mu", - "germline_single_sample_workflow.base_file_name": "NA12878_small", - "germline_single_sample_workflow.contamination_sites_bed": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.contam.bed", - "germline_single_sample_workflow.use_gatk4_haplotype_caller": "True", - "germline_single_sample_workflow.make_gatk4_single_sample_vcf": "True", - "germline_single_sample_workflow.ref_pac": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.fasta.64.pac", - "germline_single_sample_workflow.break_bands_at_multiples_of": 1000000, - "germline_single_sample_workflow.ref_fasta": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.fasta", - "germline_single_sample_workflow.ref_ann": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.fasta.64.ann", - "germline_single_sample_workflow.known_indels_sites_VCFs": [ - "gs://gcp-public-data--broad-references/hg38/v0/Mills_and_1000G_gold_standard.indels.hg38.vcf.gz", - "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.known_indels.vcf.gz" - ], - "germline_single_sample_workflow.ref_alt": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.fasta.64.alt", - "germline_single_sample_workflow.flowcell_unmapped_bams_fofn": "gs://gatk-test-data/wgs_ubam/NA12878_24RG/NA12878_24RG_small.txt", - "germline_single_sample_workflow.ref_sa": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.fasta.64.sa", - "germline_single_sample_workflow.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.dbsnp138.vcf", - "germline_single_sample_workflow.known_indels_sites_indices": [ - "gs://gcp-public-data--broad-references/hg38/v0/Mills_and_1000G_gold_standard.indels.hg38.vcf.gz.tbi", - "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.known_indels.vcf.gz.tbi" - ], - "germline_single_sample_workflow.wgs_coverage_interval_list": "gs://gcp-public-data--broad-references/hg38/v0/wgs_coverage_regions.hg38.interval_list", - "germline_single_sample_workflow.ref_bwt": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.fasta.64.bwt", - "germline_single_sample_workflow.unmapped_bam_suffix": ".unmapped.bam", - "germline_single_sample_workflow.haplotype_scatter_count": 50 -} diff --git a/scripts/perf/test_cases/five_dollar_genome/five_dollar_genome.test b/scripts/perf/test_cases/five_dollar_genome/five_dollar_genome.test deleted file mode 100644 index 876f247a045..00000000000 --- a/scripts/perf/test_cases/five_dollar_genome/five_dollar_genome.test +++ /dev/null @@ -1,11 +0,0 @@ -name: five_dollar_genome -testFormat: workflowsuccess - -files { - workflow: "https://raw.githubusercontent.com/broadinstitute/cromwell/$BRANCH/scripts/perf/test_cases/five_dollar_genome/five_dollar_genome.wdl" - inputs: "https://raw.githubusercontent.com/broadinstitute/cromwell/$BRANCH/scripts/perf/test_cases/five_dollar_genome/five_dollar_genome.inputs" -} - -metadata { - status: Succeeded -} diff --git a/scripts/perf/test_cases/five_dollar_genome/five_dollar_genome.wdl b/scripts/perf/test_cases/five_dollar_genome/five_dollar_genome.wdl deleted file mode 100644 index 9d46689650a..00000000000 --- a/scripts/perf/test_cases/five_dollar_genome/five_dollar_genome.wdl +++ /dev/null @@ -1,521 +0,0 @@ -## Copyright Broad Institute, 2018 -## -## This WDL pipeline implements data pre-processing and initial variant calling (GVCF -## generation) according to the GATK Best Practices (June 2016) for germline SNP and -## Indel discovery in human whole-genome sequencing data. -## -## Requirements/expectations : -## - Human whole-genome pair-end sequencing data in unmapped BAM (uBAM) format -## - One or more read groups, one per uBAM file, all belonging to a single sample (SM) -## - Input uBAM files must additionally comply with the following requirements: -## - - filenames all have the same suffix (we use ".unmapped.bam") -## - - files must pass validation by ValidateSamFile -## - - reads are provided in query-sorted order -## - - all reads must have an RG tag -## - GVCF output names must end in ".g.vcf.gz" -## - Reference genome must be Hg38 with ALT contigs -## -## Runtime parameters are optimized for Broad's Google Cloud Platform implementation. -## For program versions, see docker containers. -## -## LICENSING : -## This script is released under the WDL source code license (BSD-3) (see LICENSE in -## https://github.com/broadinstitute/wdl). Note however that the programs it calls may -## be subject to different licenses. Users are responsible for checking that they are -## authorized to run all programs before running this script. Please see the docker -## page at https://hub.docker.com/r/broadinstitute/genomes-in-the-cloud/ for detailed -## licensing information pertaining to the included programs. - -import "https://api.firecloud.org/ga4gh/v1/tools/gatk:alignment/versions/4/plain-WDL/descriptor" as Alignment -import "https://api.firecloud.org/ga4gh/v1/tools/gatk:split-large-readgroup/versions/4/plain-WDL/descriptor" as SplitRG -import "https://api.firecloud.org/ga4gh/v1/tools/gatk:quality-control/versions/2/plain-WDL/descriptor" as QC -import "https://api.firecloud.org/ga4gh/v1/tools/gatk:bam-processing/versions/4/plain-WDL/descriptor" as Processing -import "https://api.firecloud.org/ga4gh/v1/tools/gatk:germline-variant-discovery/versions/3/plain-WDL/descriptor" as Calling -import "https://api.firecloud.org/ga4gh/v1/tools/gatk:utilities/versions/2/plain-WDL/descriptor" as Utils - -# WORKFLOW DEFINITION -workflow germline_single_sample_workflow { - - File flowcell_unmapped_bams_fofn - Array[File] flowcell_unmapped_bams = read_lines(flowcell_unmapped_bams_fofn) - - File contamination_sites_ud - File contamination_sites_bed - File contamination_sites_mu - File wgs_evaluation_interval_list - File wgs_coverage_interval_list - - String sample_name - String base_file_name - String final_vcf_base_name - String unmapped_bam_suffix - - File wgs_calling_interval_list - Int haplotype_scatter_count - Int break_bands_at_multiples_of - Int read_length = 250 - - File ref_fasta - File ref_fasta_index - File ref_dict - File ref_alt - File ref_bwt - File ref_sa - File ref_amb - File ref_ann - File ref_pac - - File dbSNP_vcf - File dbSNP_vcf_index - Array[File] known_indels_sites_VCFs - Array[File] known_indels_sites_indices - - Int preemptible_tries - Int agg_preemptible_tries - - Boolean skip_QC - Boolean make_gatk4_single_sample_vcf - Boolean use_gatk4_haplotype_caller - - Float cutoff_for_large_rg_in_gb = 20.0 - - String bwa_commandline="bwa mem -K 100000000 -p -v 3 -t 16 -Y $bash_ref_fasta" - - String recalibrated_bam_basename = base_file_name + ".aligned.duplicates_marked.recalibrated" - - Int compression_level = 2 - - # Get the version of BWA to include in the PG record in the header of the BAM produced - # by MergeBamAlignment. - call Alignment.GetBwaVersion - - # Align flowcell-level unmapped input bams in parallel - scatter (unmapped_bam in flowcell_unmapped_bams) { - - Float unmapped_bam_size = size(unmapped_bam, "GB") - - String unmapped_bam_basename = basename(unmapped_bam, unmapped_bam_suffix) - - if (!skip_QC) { - # QC the unmapped BAM - call QC.CollectQualityYieldMetrics as CollectQualityYieldMetrics { - input: - input_bam = unmapped_bam, - metrics_filename = unmapped_bam_basename + ".unmapped.quality_yield_metrics", - preemptible_tries = preemptible_tries - } - } - - if (unmapped_bam_size > cutoff_for_large_rg_in_gb) { - # Split bam into multiple smaller bams, - # map reads to reference and recombine into one bam - call SplitRG.split_large_readgroup as SplitRG { - input: - input_bam = unmapped_bam, - bwa_commandline = bwa_commandline, - bwa_version = GetBwaVersion.version, - output_bam_basename = unmapped_bam_basename + ".aligned.unsorted", - ref_fasta = ref_fasta, - ref_fasta_index = ref_fasta_index, - ref_dict = ref_dict, - ref_alt = ref_alt, - ref_amb = ref_amb, - ref_ann = ref_ann, - ref_bwt = ref_bwt, - ref_pac = ref_pac, - ref_sa = ref_sa, - compression_level = compression_level, - preemptible_tries = preemptible_tries - } - } - - if (unmapped_bam_size <= cutoff_for_large_rg_in_gb) { - # Map reads to reference - call Alignment.SamToFastqAndBwaMemAndMba as SamToFastqAndBwaMemAndMba { - input: - input_bam = unmapped_bam, - bwa_commandline = bwa_commandline, - output_bam_basename = unmapped_bam_basename + ".aligned.unsorted", - ref_fasta = ref_fasta, - ref_fasta_index = ref_fasta_index, - ref_dict = ref_dict, - ref_alt = ref_alt, - ref_bwt = ref_bwt, - ref_amb = ref_amb, - ref_ann = ref_ann, - ref_pac = ref_pac, - ref_sa = ref_sa, - bwa_version = GetBwaVersion.version, - compression_level = compression_level, - preemptible_tries = preemptible_tries - } - } - - File output_aligned_bam = select_first([SamToFastqAndBwaMemAndMba.output_bam, SplitRG.aligned_bam]) - - Float mapped_bam_size = size(output_aligned_bam, "GB") - - if (!skip_QC) { - # QC the aligned but unsorted readgroup BAM - # no reference as the input here is unsorted, providing a reference would cause an error - call QC.CollectUnsortedReadgroupBamQualityMetrics as CollectUnsortedReadgroupBamQualityMetrics { - input: - input_bam = output_aligned_bam, - output_bam_prefix = unmapped_bam_basename + ".readgroup", - preemptible_tries = preemptible_tries - } - } - } - - # Sum the read group bam sizes to approximate the aggregated bam size - call Utils.SumFloats as SumFloats { - input: - sizes = mapped_bam_size, - preemptible_tries = preemptible_tries - } - - # Aggregate aligned+merged flowcell BAM files and mark duplicates - # We take advantage of the tool's ability to take multiple BAM inputs and write out a single output - # to avoid having to spend time just merging BAM files. - call Processing.MarkDuplicates as MarkDuplicates { - input: - input_bams = output_aligned_bam, - output_bam_basename = base_file_name + ".aligned.unsorted.duplicates_marked", - metrics_filename = base_file_name + ".duplicate_metrics", - total_input_size = SumFloats.total_size, - compression_level = compression_level, - preemptible_tries = agg_preemptible_tries - } - - # Sort aggregated+deduped BAM file - call Processing.SortSam as SortSampleBam { - input: - input_bam = MarkDuplicates.output_bam, - output_bam_basename = base_file_name + ".aligned.duplicate_marked.sorted", - compression_level = compression_level, - preemptible_tries = agg_preemptible_tries - } - - # Create list of sequences for scatter-gather parallelization - call Utils.CreateSequenceGroupingTSV as CreateSequenceGroupingTSV { - input: - ref_dict = ref_dict, - preemptible_tries = preemptible_tries - } - - # Estimate level of cross-sample contamination - call Processing.CheckContamination as CheckContamination { - input: - input_bam = SortSampleBam.output_bam, - input_bam_index = SortSampleBam.output_bam_index, - contamination_sites_ud = contamination_sites_ud, - contamination_sites_bed = contamination_sites_bed, - contamination_sites_mu = contamination_sites_mu, - ref_fasta = ref_fasta, - ref_fasta_index = ref_fasta_index, - output_prefix = base_file_name + ".preBqsr", - preemptible_tries = agg_preemptible_tries, - contamination_underestimation_factor = 0.75 - } - - # We need disk to localize the sharded input and output due to the scatter for BQSR. - # If we take the number we are scattering by and reduce by 3 we will have enough disk space - # to account for the fact that the data is not split evenly. - Int num_of_bqsr_scatters = length(CreateSequenceGroupingTSV.sequence_grouping) - Int potential_bqsr_divisor = num_of_bqsr_scatters - 10 - Int bqsr_divisor = if potential_bqsr_divisor > 1 then potential_bqsr_divisor else 1 - - # Perform Base Quality Score Recalibration (BQSR) on the sorted BAM in parallel - scatter (subgroup in CreateSequenceGroupingTSV.sequence_grouping) { - # Generate the recalibration model by interval - call Processing.BaseRecalibrator as BaseRecalibrator { - input: - input_bam = SortSampleBam.output_bam, - recalibration_report_filename = base_file_name + ".recal_data.csv", - sequence_group_interval = subgroup, - dbSNP_vcf = dbSNP_vcf, - dbSNP_vcf_index = dbSNP_vcf_index, - known_indels_sites_VCFs = known_indels_sites_VCFs, - known_indels_sites_indices = known_indels_sites_indices, - ref_dict = ref_dict, - ref_fasta = ref_fasta, - ref_fasta_index = ref_fasta_index, - bqsr_scatter = bqsr_divisor, - preemptible_tries = agg_preemptible_tries - } - } - - # Merge the recalibration reports resulting from by-interval recalibration - # The reports are always the same size - call Processing.GatherBqsrReports as GatherBqsrReports { - input: - input_bqsr_reports = BaseRecalibrator.recalibration_report, - output_report_filename = base_file_name + ".recal_data.csv", - preemptible_tries = preemptible_tries - } - - scatter (subgroup in CreateSequenceGroupingTSV.sequence_grouping_with_unmapped) { - # Apply the recalibration model by interval - call Processing.ApplyBQSR as ApplyBQSR { - input: - input_bam = SortSampleBam.output_bam, - output_bam_basename = recalibrated_bam_basename, - recalibration_report = GatherBqsrReports.output_bqsr_report, - sequence_group_interval = subgroup, - ref_dict = ref_dict, - ref_fasta = ref_fasta, - ref_fasta_index = ref_fasta_index, - bqsr_scatter = bqsr_divisor, - compression_level = compression_level, - preemptible_tries = agg_preemptible_tries - } - } - - Float agg_bam_size = size(SortSampleBam.output_bam, "GB") - - # Merge the recalibrated BAM files resulting from by-interval recalibration - call Processing.GatherSortedBamFiles as GatherBamFiles { - input: - input_bams = ApplyBQSR.recalibrated_bam, - output_bam_basename = base_file_name, - total_input_size = agg_bam_size, - compression_level = compression_level, - preemptible_tries = agg_preemptible_tries - } - - #BQSR bins the qualities which makes a significantly smaller bam - Float binned_qual_bam_size = size(GatherBamFiles.output_bam, "GB") - - File? chimerism_metrics - - if (!skip_QC) { - # QC the final BAM (consolidated after scattered BQSR) - call QC.CollectReadgroupBamQualityMetrics as CollectReadgroupBamQualityMetrics { - input: - input_bam = GatherBamFiles.output_bam, - input_bam_index = GatherBamFiles.output_bam_index, - output_bam_prefix = base_file_name + ".readgroup", - ref_dict = ref_dict, - ref_fasta = ref_fasta, - ref_fasta_index = ref_fasta_index, - preemptible_tries = agg_preemptible_tries - } - - # QC the final BAM some more (no such thing as too much QC) - call QC.CollectAggregationMetrics as CollectAggregationMetrics { - input: - input_bam = GatherBamFiles.output_bam, - input_bam_index = GatherBamFiles.output_bam_index, - output_bam_prefix = base_file_name, - ref_dict = ref_dict, - ref_fasta = ref_fasta, - ref_fasta_index = ref_fasta_index, - preemptible_tries = agg_preemptible_tries - } - - # QC the sample WGS metrics (stringent thresholds) - call QC.CollectWgsMetrics as CollectWgsMetrics { - input: - input_bam = GatherBamFiles.output_bam, - input_bam_index = GatherBamFiles.output_bam_index, - metrics_filename = base_file_name + ".wgs_metrics", - ref_fasta = ref_fasta, - ref_fasta_index = ref_fasta_index, - wgs_coverage_interval_list = wgs_coverage_interval_list, - read_length = read_length, - preemptible_tries = agg_preemptible_tries - } - - # QC the sample raw WGS metrics (common thresholds) - call QC.CollectRawWgsMetrics as CollectRawWgsMetrics { - input: - input_bam = GatherBamFiles.output_bam, - input_bam_index = GatherBamFiles.output_bam_index, - metrics_filename = base_file_name + ".raw_wgs_metrics", - ref_fasta = ref_fasta, - ref_fasta_index = ref_fasta_index, - wgs_coverage_interval_list = wgs_coverage_interval_list, - read_length = read_length, - preemptible_tries = agg_preemptible_tries - } - - # Generate a checksum per readgroup in the final BAM - call QC.CalculateReadGroupChecksum as CalculateReadGroupChecksum { - input: - input_bam = GatherBamFiles.output_bam, - input_bam_index = GatherBamFiles.output_bam_index, - read_group_md5_filename = recalibrated_bam_basename + ".bam.read_group_md5", - preemptible_tries = agg_preemptible_tries - } - } - - # ValidateSamFile runs out of memory in mate validation on crazy edge case data, so we want to skip the mate validation - # in those cases. These values set the thresholds for what is considered outside the normal realm of "reasonable" data. - Float max_duplication_in_reasonable_sample = 0.30 - Float max_chimerism_in_reasonable_sample = 0.15 - - # Convert the final merged recalibrated BAM file to CRAM format - call Utils.ConvertToCram as ConvertToCram { - input: - input_bam = GatherBamFiles.output_bam, - ref_fasta = ref_fasta, - ref_fasta_index = ref_fasta_index, - output_basename = base_file_name, - preemptible_tries = agg_preemptible_tries - } - - if (!skip_QC) { - # Check whether the data has massively high duplication or chimerism rates - call QC.CheckPreValidation as CheckPreValidation { - input: - duplication_metrics = MarkDuplicates.duplicate_metrics, - chimerism_metrics = select_first([CollectAggregationMetrics.alignment_summary_metrics, ""]), - max_duplication_in_reasonable_sample = max_duplication_in_reasonable_sample, - max_chimerism_in_reasonable_sample = max_chimerism_in_reasonable_sample, - preemptible_tries = agg_preemptible_tries - } - } - - Boolean is_outlier_data = select_first([CheckPreValidation.is_outlier_data, false]) - - # Validate the CRAM file - call QC.ValidateSamFile as ValidateCram { - input: - input_bam = ConvertToCram.output_cram, - input_bam_index = ConvertToCram.output_cram_index, - report_filename = base_file_name + ".cram.validation_report", - ref_dict = ref_dict, - ref_fasta = ref_fasta, - ref_fasta_index = ref_fasta_index, - ignore = ["MISSING_TAG_NM"], - max_output = 1000000000, - is_outlier_data = is_outlier_data, - preemptible_tries = agg_preemptible_tries - } - - # Break the calling interval_list into sub-intervals - # Perform variant calling on the sub-intervals, and then gather the results - call Utils.ScatterIntervalList as ScatterIntervalList { - input: - interval_list = wgs_calling_interval_list, - scatter_count = haplotype_scatter_count, - break_bands_at_multiples_of = break_bands_at_multiples_of - } - - # We need disk to localize the sharded input and output due to the scatter for HaplotypeCaller. - # If we take the number we are scattering by and reduce by 20 we will have enough disk space - # to account for the fact that the data is quite uneven across the shards. - Int potential_hc_divisor = ScatterIntervalList.interval_count - 20 - Int hc_divisor = if potential_hc_divisor > 1 then potential_hc_divisor else 1 - - # Call variants in parallel over WGS calling intervals - scatter (index in range(ScatterIntervalList.interval_count)) { - - if (make_gatk4_single_sample_vcf || use_gatk4_haplotype_caller) { - call Calling.HaplotypeCaller_GATK4_VCF as HaplotypeCaller4 { - input: - contamination = CheckContamination.contamination, - input_bam = GatherBamFiles.output_bam, - interval_list = ScatterIntervalList.out[index], - make_gvcf = !make_gatk4_single_sample_vcf, - vcf_basename = base_file_name, - ref_dict = ref_dict, - ref_fasta = ref_fasta, - ref_fasta_index = ref_fasta_index, - hc_scatter = hc_divisor, - preemptible_tries = agg_preemptible_tries - } - - if (make_gatk4_single_sample_vcf) { - call Calling.HardFilterVcf as FilterVcf { - input: - input_vcf = HaplotypeCaller4.output_vcf, - input_vcf_index = HaplotypeCaller4.output_vcf_index, - vcf_basename = base_file_name, - interval_list = ScatterIntervalList.out[index], - preemptible_tries = preemptible_tries - } - } - } - if (!make_gatk4_single_sample_vcf && !use_gatk4_haplotype_caller) { - call Calling.HaplotypeCaller_GATK35_GVCF as HaplotypeCaller3 { - input: - contamination = CheckContamination.contamination, - input_bam = GatherBamFiles.output_bam, - interval_list = ScatterIntervalList.out[index], - gvcf_basename = base_file_name, - ref_dict = ref_dict, - ref_fasta = ref_fasta, - ref_fasta_index = ref_fasta_index, - hc_scatter = hc_divisor, - preemptible_tries = agg_preemptible_tries - } - } - - File merge_input = select_first([FilterVcf.output_vcf, HaplotypeCaller4.output_vcf, HaplotypeCaller3.output_gvcf]) - File merge_input_index = select_first([FilterVcf.output_vcf_index, HaplotypeCaller4.output_vcf_index, HaplotypeCaller3.output_gvcf_index]) - } - - String name_token = if make_gatk4_single_sample_vcf then ".filtered" else ".g" - - # Combine by-interval VCFs into a single sample VCF file - call Calling.MergeVCFs as MergeVCFs { - input: - input_vcfs = merge_input, - input_vcfs_indexes = merge_input_index, - output_vcf_name = final_vcf_base_name + name_token + ".vcf.gz", - preemptible_tries = agg_preemptible_tries - } - - # Outputs that will be retained when execution is complete - output { - Array[File?] quality_yield_metrics = CollectQualityYieldMetrics.quality_yield_metrics - - Array[File?] unsorted_read_group_base_distribution_by_cycle_pdf = CollectUnsortedReadgroupBamQualityMetrics.base_distribution_by_cycle_pdf - Array[File?] unsorted_read_group_base_distribution_by_cycle_metrics = CollectUnsortedReadgroupBamQualityMetrics.base_distribution_by_cycle_metrics - Array[File?] unsorted_read_group_insert_size_histogram_pdf = CollectUnsortedReadgroupBamQualityMetrics.insert_size_histogram_pdf - Array[File?] unsorted_read_group_insert_size_metrics = CollectUnsortedReadgroupBamQualityMetrics.insert_size_metrics - Array[File?] unsorted_read_group_quality_by_cycle_pdf = CollectUnsortedReadgroupBamQualityMetrics.quality_by_cycle_pdf - Array[File?] unsorted_read_group_quality_by_cycle_metrics = CollectUnsortedReadgroupBamQualityMetrics.quality_by_cycle_metrics - Array[File?] unsorted_read_group_quality_distribution_pdf = CollectUnsortedReadgroupBamQualityMetrics.quality_distribution_pdf - Array[File?] unsorted_read_group_quality_distribution_metrics = CollectUnsortedReadgroupBamQualityMetrics.quality_distribution_metrics - - File? read_group_alignment_summary_metrics = CollectReadgroupBamQualityMetrics.alignment_summary_metrics - File? read_group_gc_bias_detail_metrics = CollectReadgroupBamQualityMetrics.gc_bias_detail_metrics - File? read_group_gc_bias_pdf = CollectReadgroupBamQualityMetrics.gc_bias_pdf - File? read_group_gc_bias_summary_metrics = CollectReadgroupBamQualityMetrics.gc_bias_summary_metrics - - File selfSM = CheckContamination.selfSM - Float contamination = CheckContamination.contamination - - File? calculate_read_group_checksum_md5 = CalculateReadGroupChecksum.md5_file - - File? agg_alignment_summary_metrics = CollectAggregationMetrics.alignment_summary_metrics - File? agg_bait_bias_detail_metrics = CollectAggregationMetrics.bait_bias_detail_metrics - File? agg_bait_bias_summary_metrics = CollectAggregationMetrics.bait_bias_summary_metrics - File? agg_gc_bias_detail_metrics = CollectAggregationMetrics.gc_bias_detail_metrics - File? agg_gc_bias_pdf = CollectAggregationMetrics.gc_bias_pdf - File? agg_gc_bias_summary_metrics = CollectAggregationMetrics.gc_bias_summary_metrics - File? agg_insert_size_histogram_pdf = CollectAggregationMetrics.insert_size_histogram_pdf - File? agg_insert_size_metrics = CollectAggregationMetrics.insert_size_metrics - File? agg_pre_adapter_detail_metrics = CollectAggregationMetrics.pre_adapter_detail_metrics - File? agg_pre_adapter_summary_metrics = CollectAggregationMetrics.pre_adapter_summary_metrics - File? agg_quality_distribution_pdf = CollectAggregationMetrics.quality_distribution_pdf - File? agg_quality_distribution_metrics = CollectAggregationMetrics.quality_distribution_metrics - - File? wgs_metrics = CollectWgsMetrics.metrics - File? raw_wgs_metrics = CollectRawWgsMetrics.metrics - - File duplicate_metrics = MarkDuplicates.duplicate_metrics - File output_bqsr_reports = GatherBqsrReports.output_bqsr_report - - File output_cram = ConvertToCram.output_cram - File output_cram_index = ConvertToCram.output_cram_index - File output_cram_md5 = ConvertToCram.output_cram_md5 - - File validate_cram_file_report = ValidateCram.report - - File output_vcf = MergeVCFs.output_vcf - File output_vcf_index = MergeVCFs.output_vcf_index - } -} diff --git a/scripts/perf/test_cases/gary/gary.inputs b/scripts/perf/test_cases/gary/gary.inputs deleted file mode 100644 index 6c630f09e27..00000000000 --- a/scripts/perf/test_cases/gary/gary.inputs +++ /dev/null @@ -1,12 +0,0 @@ -{ - "echo_strings.echo_files.input1": "gs://debtest3/h.txt", - "echo_strings.echo_files.input5": "gs://debtest3/h.txt", - "echo_strings.echo_files.input2": "gs://debtest3/h.txt", - "echo_strings.echo_files.input10": "gs://debtest3/h.txt", - "echo_strings.echo_files.input6": "gs://debtest3/h.txt", - "echo_strings.echo_files.input9": "gs://debtest3/h.txt", - "echo_strings.echo_files.input3": "gs://debtest3/h.txt", - "echo_strings.echo_files.input7": "gs://debtest3/h.txt", - "echo_strings.echo_files.input8": "gs://debtest3/h.txt", - "echo_strings.echo_files.input4": "gs://debtest3/h.txt" -} diff --git a/scripts/perf/test_cases/gary/gary.test b/scripts/perf/test_cases/gary/gary.test deleted file mode 100644 index 5b1f5783afd..00000000000 --- a/scripts/perf/test_cases/gary/gary.test +++ /dev/null @@ -1,11 +0,0 @@ -name: gary -testFormat: workflowsuccess - -files { - workflow: "https://raw.githubusercontent.com/broadinstitute/cromwell/$BRANCH/scripts/perf/test_cases/gary/gary.wdl" - inputs: "https://raw.githubusercontent.com/broadinstitute/cromwell/$BRANCH/scripts/perf/test_cases/gary/gary.inputs" -} - -metadata { - status: Succeeded -} diff --git a/scripts/perf/test_cases/gary/gary.wdl b/scripts/perf/test_cases/gary/gary.wdl deleted file mode 100644 index 57652bb28e4..00000000000 --- a/scripts/perf/test_cases/gary/gary.wdl +++ /dev/null @@ -1,40 +0,0 @@ -task echo_files { - File input1 - File input2 - File input3 - File input4 - File input5 - File input6 - File input7 - File input8 - File input9 - File input10 - - Float ref_size = size(input1, "GB") + size(input2, "GB") + size(input3, "GB") + size(input4, "GB") + size(input5, "GB") + size(input6, "GB") + size(input7, "GB") + size(input8, "GB") + size(input9, "GB") + size(input10, "GB") - - output { - String out = read_string(stdout()) - } - - command { - # sleep for 1 hour - # sleep 3600 - - echo "result: ${ref_size}" - } - - runtime { - docker: "ubuntu:latest" - cpu: "1" - memory: "0.1 GB" - preemptible: 3 - disks: "local-disk 1 HDD" - bootDiskSizeGb: 10 - } -} - -workflow echo_strings { - scatter (i in range(30)) { - call echo_files - } -} diff --git a/scripts/perf/test_cases/hello/hello.test b/scripts/perf/test_cases/hello/hello.test deleted file mode 100644 index accd48f9a23..00000000000 --- a/scripts/perf/test_cases/hello/hello.test +++ /dev/null @@ -1,16 +0,0 @@ -name: hello -testFormat: workflowsuccess - -files { - workflow: "https://raw.githubusercontent.com/broadinstitute/cromwell/develop/centaur/src/main/resources/standardTestCases/hello/hello.wdl" - inputs: "https://raw.githubusercontent.com/broadinstitute/cromwell/develop/centaur/src/main/resources/standardTestCases/hello/hello.inputs" -} - -metadata { - workflowName: wf_hello - status: Succeeded - "calls.wf_hello.hello.executionStatus": Done - "calls.wf_hello.hello.runtimeAttributes.docker": "ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950" - "outputs.wf_hello.hello.salutation": "Hello m'Lord!" - "inputs.wf_hello.hello.addressee": "m'Lord" -} diff --git a/scripts/perf/test_cases/reader/cromwell.conf b/scripts/perf/test_cases/reader/cromwell.conf deleted file mode 100644 index 7a282f4e171..00000000000 --- a/scripts/perf/test_cases/reader/cromwell.conf +++ /dev/null @@ -1,8 +0,0 @@ -include required("cromwell_default.conf") - -system { - cromwell_id = "perf-reader" - max-concurrent-workflows = 0 - max-workflow-launch-count = 0 - new-workflow-poll-rate = 999999 -} diff --git a/scripts/perf/vm_scripts/centaur/centaur.conf b/scripts/perf/vm_scripts/centaur/centaur.conf deleted file mode 100644 index 6f6cbfab9d1..00000000000 --- a/scripts/perf/vm_scripts/centaur/centaur.conf +++ /dev/null @@ -1,17 +0,0 @@ -akka.http.client.parsing.max-content-length = 524288000 - -centaur { - error-reporter { - retry-attempts: 0 - providers { - gcs { - # Reports metadata to gcs for both successes and failures - class: "centaur.reporting.GcsReporter" - config { - report-bucket: ${?GCS_REPORT_BUCKET} - report-path: ${?GCS_REPORT_PATH}/metadata.json - } - } - } - } -} diff --git a/scripts/perf/vm_scripts/cromwell-dashboard.json b/scripts/perf/vm_scripts/cromwell-dashboard.json deleted file mode 100644 index f11c19a2981..00000000000 --- a/scripts/perf/vm_scripts/cromwell-dashboard.json +++ /dev/null @@ -1,1703 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_LOCAL_GRAPHITE", - "label": "Local Graphite", - "description": "", - "type": "datasource", - "pluginId": "graphite", - "pluginName": "Graphite" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "4.4.3" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "datasource", - "id": "graphite", - "name": "Graphite", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "" - } - ], - "annotations": { - "list": [] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "id": null, - "links": [], - "refresh": "5s", - "rows": [ - { - "collapse": false, - "height": 222, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_GRAPHITE}", - "fill": 1, - "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "B", - "target": "stats.gauges.cromwell.load.IO" - }, - { - "refId": "C", - "target": "stats.gauges.cromwell.load.JobStoreReaderActor" - }, - { - "refId": "D", - "target": "stats.gauges.cromwell.load.JobStoreWriterActor" - }, - { - "refId": "E", - "target": "stats.gauges.cromwell.load.KvReadActor" - }, - { - "refId": "F", - "target": "stats.gauges.cromwell.load.KvWriteActor" - }, - { - "refId": "G", - "target": "stats.gauges.cromwell.load.Memory" - }, - { - "refId": "H", - "target": "stats.gauges.cromwell.load.PAPIQueryManager" - }, - { - "refId": "I", - "target": "stats.gauges.cromwell.load.WriteMetadataActor" - }, - { - "refId": "J", - "target": "maxSeries(stats.gauges.cromwell.load.CallCacheReadActor.*)" - }, - { - "refId": "A", - "target": "stats.gauges.cromwell.load.CallCacheWriteActor" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Load Levels", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "${DS_LOCAL_GRAPHITE}", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 9, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "refId": "A", - "target": "keepLastValue(stats.gauges.cromwell.load.global, 1)" - } - ], - "thresholds": "1", - "title": "Load", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "Normal", - "value": "0" - }, - { - "op": "=", - "text": "High", - "value": "1" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "${DS_LOCAL_GRAPHITE}", - "editable": true, - "error": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "hideTimeOverride": true, - "id": 20, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "refId": "A", - "target": "stats.gauges.cromwell.workflow.Queued" - } - ], - "thresholds": "", - "timeFrom": "5m", - "timeShift": null, - "title": "Workflows Queued", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "${DS_LOCAL_GRAPHITE}", - "editable": true, - "error": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "hideTimeOverride": true, - "id": 19, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "refId": "A", - "target": "stats.gauges.cromwell.workflow.Running" - } - ], - "thresholds": "", - "timeFrom": "5m", - "timeShift": null, - "title": "Workflows Running", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "${DS_LOCAL_GRAPHITE}", - "format": "none", - "gauge": { - "maxValue": 75000, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": false - }, - "hideTimeOverride": true, - "id": 12, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": "100", - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "refId": "A", - "target": "keepLastValue(stats.gauges.cromwell.job.Running, 1)" - } - ], - "thresholds": "50000,70000", - "timeFrom": "10m", - "timeShift": null, - "title": "Jobs Running", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "max" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "${DS_LOCAL_GRAPHITE}", - "editable": true, - "error": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "hideTimeOverride": true, - "id": 16, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "refId": "A", - "target": "stats.gauges.cromwell.job.QueuedInCromwell" - } - ], - "thresholds": "", - "timeFrom": "5m", - "timeShift": null, - "title": "Job Queued", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6" - }, - { - "collapse": false, - "height": 255, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "alias(scale(stats.gauges.cromwell.workflow.Succeeded.m1_rate, 60), 'Succeeded')" - }, - { - "refId": "B", - "target": "alias(scale(stats.gauges.cromwell.workflow.Aborted.m1_rate, 60), 'Aborted')" - }, - { - "refId": "C", - "target": "alias(scale(stats.gauges.cromwell.workflow.Failed.m1_rate, 60), 'Failed')" - }, - { - "refId": "D", - "target": "alias(scale(stats.gauges.cromwell.workflow.Submitted.m1_rate, 60), 'Submitted')" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Workflow Throughput", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "per minute", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "stats.gauges.cromwell.workflow.Queued" - }, - { - "refId": "B", - "target": "stats.gauges.cromwell.workflow.Running" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Workflows", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6" - }, - { - "collapse": false, - "height": 292, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_GRAPHITE}", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByMetric(stats.gauges.cromwell.job.Running)" - }, - { - "refId": "B", - "target": "aliasByMetric(stats.gauges.cromwell.job.QueuedInCromwell)" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Jobs", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_GRAPHITE}", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "stats.gauges.cromwell.job.Started.m1_rate" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Jobs Start Rate", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "id": 21, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "scaleToSeconds(absolute(removeAboveValue(derivative(keepLastValue(removeBelowValue(stats.gauges.*.job.QueuedInCromwell, 1), 100)), -1)), 60)" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Jobs Dequeue Rate", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6" - }, - { - "collapse": false, - "height": 250, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_GRAPHITE}", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "B", - "target": "stats.gauges.cromwell.io.backpressure.m1_rate" - }, - { - "refId": "C", - "target": "stats.gauges.cromwell.io.gcs.*.failure.*.m1_rate" - }, - { - "refId": "D", - "target": "stats.gauges.cromwell.io.gcs.*.success.m1_rate" - }, - { - "refId": "A", - "target": "stats.gauges.cromwell.io.gcs.*.retry.*.m1_rate" - }, - { - "refId": "E", - "target": "stats.gauges.cromwell.io.gcs.*.failure.m1_rate" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "I/O", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6" - }, - { - "collapse": false, - "height": 305, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_GRAPHITE}", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "B", - "target": "stats.gauges.cromwell.backend.papi.poll.*.m1_rate" - }, - { - "refId": "A", - "target": "stats.gauges.cromwell.backend.papi.run.*.m1_rate" - }, - { - "refId": "C", - "target": "stats.gauges.cromwell.backend.papi.abort.*.m1_rate" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "PAPI Throughput", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_GRAPHITE}", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "stats.gauges.cromwell.backend.papi.queue_size" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "PAPI Query Queue", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6" - }, - { - "collapse": false, - "height": 342, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_GRAPHITE}", - "fill": 1, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "stats.gauges.cromwell.services.keyvalue.*.processed.m1_rate" - }, - { - "refId": "C", - "target": "stats.gauges.cromwell.job.store.*.processed.m1_rate" - }, - { - "refId": "D", - "target": "sumSeries(stats.gauges.cromwell.job.callcaching.read.*.processed.m1_rate, *)" - }, - { - "refId": "E", - "target": "stats.gauges.cromwell.job.callcaching.write.processed.m1_rate" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Engine Throughput", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_GRAPHITE}", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "C", - "target": "stats.gauges.cromwell.job.store.*.queue" - }, - { - "refId": "D", - "target": "stats.gauges.cromwell.services.keyvalue.*.queue" - }, - { - "refId": "A", - "target": "sumSeries(stats.gauges.cromwell.job.callcaching.read.*.queue, *)" - }, - { - "refId": "B", - "target": "stats.gauges.cromwell.job.callcaching.write.queue" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Engine Queues", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6" - }, - { - "collapse": false, - "height": 315, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_GRAPHITE}", - "fill": 1, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "stats.gauges.cromwell.services.metadata.processed.m1_rate" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Metadata Throughput", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_GRAPHITE}", - "fill": 1, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "stats.gauges.cromwell.services.metadata.queue" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Metadata Queue", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6" - }, - { - "collapse": false, - "height": 250, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_GRAPHITE}", - "fill": 1, - "id": 22, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "stats.gauges.cromwell.io.gcs.batch.timing.mean" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Batch request timing", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Cromwell", - "version": 6 -} \ No newline at end of file diff --git a/scripts/perf/vm_scripts/cromwell/cromwell.conf b/scripts/perf/vm_scripts/cromwell/cromwell.conf deleted file mode 100644 index 9fb29a28b16..00000000000 --- a/scripts/perf/vm_scripts/cromwell/cromwell.conf +++ /dev/null @@ -1,95 +0,0 @@ -include required(classpath("application.conf")) - -akka.http.server.request-timeout = 600 seconds - -google { - - application-name = "cromwell" - - auths = [ - { - name = "application-default" - scheme = "application_default" - } - { - name = "user-service-account" - scheme = "user_service_account" - } - ] -} - -engine { - filesystems { - gcs { - auth = "application-default" - project = ${?CROMWELL_PROJECT} - } - } -} - -call-caching { - # Allows re-use of existing results for jobs you've already run - # (default: false) - enabled = true - - # Whether to invalidate a cache result forever if we cannot reuse them. Disable this if you expect some cache copies - # to fail for external reasons which should not invalidate the cache (e.g. auth differences between users): - # (default: true) - invalidate-bad-cache-results = false -} - -system { - cromwell_id = "perf-runner" - io.number-of-requests = 10240611 -} - -backend { - default = "Papi" - enabled = ["Papi"] - providers { - Papi { - actor-factory = "cromwell.backend.google.pipelines.v1alpha2.PipelinesApiLifecycleActorFactory" - config { - project = ${?CROMWELL_PROJECT} - root = ${?CROMWELL_EXECUTION_ROOT} - maximum-polling-interval = 600 - concurrent-job-limit = 1000 - - genomics { - auth = "application-default" - endpoint-url = "https://genomics.googleapis.com/" - } - - filesystems { - gcs { - auth = "application-default" - project = ${?CROMWELL_PROJECT} - } - } - } - } - } -} - -services { - Instrumentation { - # Stackdriver - Send metrics to Google's monitoring API - class = "cromwell.services.instrumentation.impl.stackdriver.StackdriverInstrumentationServiceActor" - config { - auth = "application-default" - google-project = ${?CROMWELL_PROJECT} - # rate at which aggregated metrics will be sent to Stackdriver API, must be 1 minute or more. - flush-rate = 1 minute - cromwell-perf-test-case = ${?TEST_CASE_DIRECTORY} - } - } -} - -database { - db.url = "jdbc:mysql://cloudsql-db/cromwell?useSSL=true&rewriteBatchedStatements=true" - db.user = ${?CLOUD_SQL_DB_USER} - db.password = ${?CLOUD_SQL_DB_PASSWORD} - db.driver = "com.mysql.cj.jdbc.Driver" - db.connectionTimeout=30000 - profile = "slick.jdbc.MySQLProfile$" -} diff --git a/scripts/perf/vm_scripts/docker-compose.yml b/scripts/perf/vm_scripts/docker-compose.yml deleted file mode 100644 index a6e05ced58c..00000000000 --- a/scripts/perf/vm_scripts/docker-compose.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: '2' -networks: - mysql_net: - driver: bridge -services: - cromwell: - image: "${CROMWELL_DOCKER_IMAGE}" - volumes: - - ${CROMWELL_CONF_DIR}:/conf - environment: - - CROMWELL_PROJECT - - CROMWELL_EXECUTION_ROOT - - CLOUD_SQL_DB_USER - - CLOUD_SQL_DB_PASSWORD - - TEST_CASE_DIRECTORY - - JAVA_OPTS=-Dconfig.file=/conf/cromwell.conf - command: ["server"] - depends_on: - - cloudsql-db - ports: - - "8000:8000" - networks: - mysql_net: - container_name: cromwell - cloudsql-db: - image: "gcr.io/cloudsql-docker/gce-proxy:1.11" - ports: - - "127.0.0.1:3306:3306" - container_name: cloudsql-db - command: ["/cloud_sql_proxy", "-instances=broad-dsde-cromwell-perf:us-central1:${CLOUD_SQL_INSTANCE}=tcp:0.0.0.0:3306"] - networks: - mysql_net: diff --git a/scripts/perf/vm_scripts/statsd-proxy/proxy.conf b/scripts/perf/vm_scripts/statsd-proxy/proxy.conf deleted file mode 100644 index 900347fa203..00000000000 --- a/scripts/perf/vm_scripts/statsd-proxy/proxy.conf +++ /dev/null @@ -1,11 +0,0 @@ -proxy { - host = ${?PROXY_HOST} - port = ${?PROXY_PORT} -} - -statsd { - host = ${?CROMWELL_STATSD_HOST} - port = ${?CROMWELL_STATSD_PORT} -} - -output-file = "file:///conf/statsd.log"