diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index d18ab65108..eacf26ed9c 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -62,6 +62,14 @@ jobs: - name: Check Maven publication run: ./gradlew publishToMavenLocal sourceTarball + - name: Polaris Apprunner Gradle plugin + working-directory: tools/apprunner + run: ./gradlew --continue check + + - name: Check Apprunner Gradle Maven publication + working-directory: tools/apprunner + run: ./gradlew publishToMavenLocal + - name: Archive test results uses: actions/upload-artifact@v4 if: always() diff --git a/apprunner-demo-in-tree/build.gradle.kts b/apprunner-demo-in-tree/build.gradle.kts new file mode 100644 index 0000000000..ad6507a68a --- /dev/null +++ b/apprunner-demo-in-tree/build.gradle.kts @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + id("polaris-server") + id("org.apache.polaris.apprunner") +} + +dependencies { polarisQuarkusServer(project(":polaris-quarkus-server", "quarkusRunner")) } + +testing { + suites { + val demoTest by registering(JvmTestSuite::class) + } +} + +polarisQuarkusApp { includeTask(tasks.named("demoTest")) } diff --git a/apprunner-demo-in-tree/src/demoTest/java/org/apache/polaris/apprunner/demo/DemoTest.java b/apprunner-demo-in-tree/src/demoTest/java/org/apache/polaris/apprunner/demo/DemoTest.java new file mode 100644 index 0000000000..74c55b042c --- /dev/null +++ b/apprunner-demo-in-tree/src/demoTest/java/org/apache/polaris/apprunner/demo/DemoTest.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.apprunner.demo; + +import org.junit.jupiter.api.Test; + +public class DemoTest { + @Test + public void test() { + System.err.println(System.getProperty("quarkus.http.test-port")); + System.err.println(System.getProperty("quarkus.http.test-url")); + System.err.println(System.getProperty("quarkus.management.test-port")); + System.err.println(System.getProperty("quarkus.management.test-url")); + } +} diff --git a/apprunner-demo-tarball/build.gradle.kts b/apprunner-demo-tarball/build.gradle.kts new file mode 100644 index 0000000000..4f3fa49f91 --- /dev/null +++ b/apprunner-demo-tarball/build.gradle.kts @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + id("polaris-server") + id("org.apache.polaris.apprunner") +} + +// Gradle configuration to reference the tarball +val polarisTarball by + configurations.creating { description = "Used to reference the distribution tarball" } + +dependencies { polarisTarball(project(":polaris-quarkus-server", "distributionTar")) } + +testing { + suites { + val demoTest by registering(JvmTestSuite::class) + } +} + +// Directory where the Polaris tarball is extracted to +val unpackedTarball = project.layout.buildDirectory.dir("polaris-tarball") + +// Extracts the Polaris tarball, truncating the path +val polarisUnpackedTarball by + tasks.registering(Sync::class) { + inputs.files(polarisTarball) + destinationDir = unpackedTarball.get().asFile + from(provider { tarTree(polarisTarball.singleFile) }) + eachFile { + // truncates the path (removes the first path element) + relativePath = RelativePath(true, *relativePath.segments.drop(1).toTypedArray()) + } + includeEmptyDirs = false + } + +val demoTest = + tasks.named("demoTest") { + // Dependency to have the extracted tarball + dependsOn(polarisUnpackedTarball) + } + +polarisQuarkusApp { + includeTask(demoTest) + // Reference the quarkus-run.jar in the tarball, apprunner plugin will then run that jar + executableJar = provider { unpackedTarball.get().file("quarkus-run.jar") } +} diff --git a/apprunner-demo-tarball/src/demoTest/java/org/apache/polaris/apprunner/demo/DemoTest.java b/apprunner-demo-tarball/src/demoTest/java/org/apache/polaris/apprunner/demo/DemoTest.java new file mode 100644 index 0000000000..74c55b042c --- /dev/null +++ b/apprunner-demo-tarball/src/demoTest/java/org/apache/polaris/apprunner/demo/DemoTest.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.apprunner.demo; + +import org.junit.jupiter.api.Test; + +public class DemoTest { + @Test + public void test() { + System.err.println(System.getProperty("quarkus.http.test-port")); + System.err.println(System.getProperty("quarkus.http.test-url")); + System.err.println(System.getProperty("quarkus.management.test-port")); + System.err.println(System.getProperty("quarkus.management.test-url")); + } +} diff --git a/build-logic/src/main/kotlin/polaris-root.gradle.kts b/build-logic/src/main/kotlin/polaris-root.gradle.kts index 96faa07b82..82d3c32caf 100644 --- a/build-logic/src/main/kotlin/polaris-root.gradle.kts +++ b/build-logic/src/main/kotlin/polaris-root.gradle.kts @@ -52,6 +52,7 @@ if (System.getProperty("idea.sync.active").toBoolean()) { excludeDirs + setOf( projectDir.resolve("build-logic/.kotlin"), + projectDir.resolve("tools/apprunner/apprunner-build-logic/.kotlin"), projectDir.resolve("integration-tests/build"), projectDir.resolve("site/resources/_gen"), projectDir.resolve("site/build"), diff --git a/build.gradle.kts b/build.gradle.kts index e39abe385d..a0c1766691 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -125,6 +125,11 @@ tasks.named("rat").configure { excludes.add("**/kotlin-compiler*") excludes.add("**/build-logic/.kotlin/**") + + excludes.add( + "tools/apprunner/gradle-plugin/src/main/resources/META-INF/gradle-plugins/org.apache.polaris.apprunner" + ) + excludes.add("tools/apprunner/maven-plugin/target/**") } // Pass environment variables: diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index 3904dd06ee..e55c8986dd 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -43,3 +43,6 @@ polaris-misc-types=tools/misc-types polaris-config-docs-annotations=tools/config-docs/annotations polaris-config-docs-generator=tools/config-docs/generator polaris-config-docs-site=tools/config-docs/site + +apprunner-demo-in-tree=apprunner-demo-in-tree +apprunner-demo-tarball=apprunner-demo-tarball diff --git a/settings.gradle.kts b/settings.gradle.kts index 3884bea5bc..c78515f62a 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -21,6 +21,8 @@ import java.util.Properties includeBuild("build-logic") { name = "polaris-build-logic" } +includeBuild("tools/apprunner") { name = "polaris-apprunner" } + if (!JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_21)) { throw GradleException( """ diff --git a/tools/apprunner/.gitignore b/tools/apprunner/.gitignore new file mode 100644 index 0000000000..d28c77550d --- /dev/null +++ b/tools/apprunner/.gitignore @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Ignore Gradle wrapper jar file +gradle/wrapper/gradle-wrapper.jar +gradle/wrapper/gradle-wrapper-*.sha256 + +# Avoid ignore Gradle wrappper properties +!gradle-wrapper.properties + +# IntelliJ +/.idea +*.iml +*.ipr +*.iws + +# Gradle +/.gradle +/apprunner-build-logic/.gradle +/apprunner-build-logic/.kotlin +**/build/ +!src/**/build/ + +# Maven plugin / special +maven-plugin/target + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + +# macOS +*.DS_Store diff --git a/tools/apprunner/README.md b/tools/apprunner/README.md new file mode 100644 index 0000000000..b5329025b6 --- /dev/null +++ b/tools/apprunner/README.md @@ -0,0 +1,348 @@ + + +# Polaris Apprunner Gradle and Maven Plugins + +Gradle and Maven plugins to run a Polaris process and "properly" terminate it for integration testing. + +## Java integration tests + +Tests that run via a Gradle `Test` type task, "decorated" with the Polaris Apprunner plugin, have access to +four system properties. Integration tests using the Maven plugin have access to the same system properties. +The names of the system properties can be changed, if needed. See the [Gradle Kotlin DSL](#kotlin-dsl--all-in-one) +and [Maven](#maven) sections below for a summary of the available options. + +* `quarkus.http.test-port` the port on which the Quarkus server listens for application HTTP requests +* `quarkus.management.test-port` the URL on which the Quarkus server listens for management HTTP requests, this + URL is the one emitted by Quarkus during startup and will contain `0.0.0.0` as the host. +* `quarkus.http.test-url` the port on which the Quarkus server listens for application HTTP requests +* `quarkus.management.test-url` the URL on which the Quarkus server listens for management HTTP requests, this + URL is the one emitted by Quarkus during startup and will contain `0.0.0.0` as the host. + +The preferred way to get the URI/URL for application HTTP requests is to get the `quarkus.http.test-port` system +property and construct the URI against `127.0.0.1` (or `::1` if you prefer). + +```java +public class ITWorksWithPolaris { + static final URI POLARIS_SERVER_URI = + URI.create( + String.format( + "http://127.0.0.1:%s/", + requireNonNull( + System.getProperty("quarkus.http.test-port"), + "Required system property quarkus.http.test-port is not set"))); + + @Test + public void pingNessie() { + // Use the POLARIS_SERVER_URI in your tests ... + } +} +``` + +## Gradle + +The Polaris Apprunner Gradle ensures that the Polaris Quarkus Server is up and running if and when the configured +test tasks run. It also ensures, as long as you do not forcibly kill Gradle processes, that the Polaris Quarkus Server +is shutdown after the configured test task has finished. Each configured test task gets its "own" Polaris Quarkus +Server started up. + +It is possible to configure multiple tasks/test-suites within a Gradle project to run with a Polaris Quarkus Server. +Since tasks of the same Gradle project do not run concurrently (as of today), there are should be no conflicts, except +potentially the working directory. + +### Kotlin DSL / step by step + +`build.gradle.kts` + +1. add the plugin + ```kotlin + plugins { + // Replace the version with a binary release of Polaris. + // ATTENTION! Within the Polaris repository do _NOT_ add the plugin version, it's implicit ! + id("org.apache.polaris.apprunner") version "0.0.0" + } + ``` +2. Add the Polaris Quarkus Server as a dependency + ```kotlin + dependencies { + // Tell the plugin which project and configuration it shall 'pull' the Quarkus server from + polarisQuarkusServer(project(":polaris-quarkus-server", "quarkusRunner")) + } + ``` +3. If necessary, add a separate test suite + ```kotlin + testing { + suites { + val polarisServerTest by registering(JvmTestSuite::class) { + // more test-suite related configurations + } + } + } + ``` +4. Tell the Apprunner plugin which test tasks need the Polaris server + ```kotlin + polarisQuarkusApp { + // Add the name of the test task - usually the same as the name of your test source + includeTask(tasks.named("polarisServerTest")) + } + ``` + +Note: the above also works within the `:polaris-quarkus-server` project, but the test suite must be neither +`test` nor `intTest` nor `integrationTest`. + +### Kotlin DSL / all in one + +`build.gradle.kts` - note: the version number needs to be replaced with a (not yet existing) binary release of +Apache Polaris. + +```kotlin +plugins { + `java-library` + // Replace the version with a binary release of Polaris. + // ATTENTION! Within the Polaris repository do _NOT_ add the plugin version, it's implicit ! + id("org.apache.polaris.apprunner") version "0.0.0" +} + +dependencies { + // specify the GAV of the Polaris Quarkus server runnable (uber-jar) + polarisQuarkusServer("org.apache.polaris:polaris-quarkus-server:0.0.0:runner") +} + +polarisQuarkusApp { + // Ensure that the `test` task has a Polaris Server available. + includeTask(tasks.named("test")) + // Note: prefer setting up separate `polarisServerIntegrationTest` test suite (the name is up to you!), + // see https://docs.gradle.org/current/userguide/jvm_test_suite_plugin.html + + // Override the default Java version (21) to run the Polaris server / Quarkus. + // Must be at least 21! + javaVersion.set(21) + // Additional environment variables for the Polaris server / Quarkus + // (Added to the Gradle inputs used to determine Gradle's UP-TO-DATE status) + environment.put("MY_ENV_VAR", "value") + // Additional environment variables for the Polaris server / Quarkus + // (NOT a Gradle inputs used to determine Gradle's UP-TO-DATE status) + // Put system specific variables here + environmentNonInput.put("MY_ENV_VAR", "value") + // System properties for the Polaris server / Quarkus + // (Added to the Gradle inputs used to determine Gradle's UP-TO-DATE status) + systemProperties.put("my.sys.prop", "value") + // System properties for the Polaris server / Quarkus + // (NOT a Gradle inputs used to determine Gradle's UP-TO-DATE status) + // Put system specific variables here + systemPropertiesNonInput.put("my.sys.prop", "value") + // JVM arguments for the Polaris server JVM (list of strings) + // (Added to the Gradle inputs used to determine Gradle's UP-TO-DATE status) + jvmArguments.add("some-arg") + // JVM arguments for the Polaris server JVM (list of strings) + // (NOT a Gradle inputs used to determine Gradle's UP-TO-DATE status) + // Put system specific variables here + jvmArgumentsNonInput.add("some-arg") + // Use this (full) path to the executable jar of the Polaris server. + // Note: This option should generally be avoided in build scripts, prefer the 'polarisQuarkusServer' + // configuration mentioned above. + executableJar = file("/my/custom/polars-quarkus-server.jar") + // Override the working directory for the Polaris Quarkus server, defaults to `polaris-quarkus-server/` + // in the Gradle project's `build/` directory. + workingDirectory = file("/i/want/it/to/run/here") + // override the default timeout of 30 seconds to wait for the Polaris Quarkus Server to emit the + // listen URLs. + timeToListenUrlMillis = 30000 + // Override the default timeout of 15 seconds to wait for the Polaris Quarkus Server to stop before + // it is forcefully killed + timeToStopMillis = 15000 + // Arguments for the Polaris server (list of strings) + // (Added to the Gradle inputs used to determine Gradle's UP-TO-DATE status) + arguments.add("some-arg") + // Arguments for the Polaris server (list of strings) + // (NOT a Gradle inputs used to determine Gradle's UP-TO-DATE status) + // Put system specific variables here + argumentsNonInput.add("some-arg") + // The following options can be used to use different property names than described above + // in this README + httpListenPortProperty = "quarkus.http.test-port" + httpListenUrlProperty = "quarkus.http.test-url" + managementListenPortProperty = "quarkus.management.test-port" + managementListenUrlProperty = "quarkus.management.test-url" +} +``` + +### Groovy DSL + +`build.gradle` - note: the version number needs to be replaced with a (not yet existing) binary release of +Apache Polaris. + +```groovy +plugins { + id 'java-library' + id 'org.apache.polaris.apprunner' version "0.0.0" +} + +dependencies { + // specify the GAV of the Polaris Quarkus server runnable (uber-jar) + polarisQuarkusServer "org.apache.polaris:polaris-quarkus-server:0.0.0:runner" +} + +polarisQuarkusApp { + // Ensure that the `test` task has a Polaris Server available when the tests run. + includeTask(tasks.named("test")) + // Note: prefer setting up separate `polarisServerIntegrationTest` test suite (the name is up to you!), + // see https://docs.gradle.org/current/userguide/jvm_test_suite_plugin.html + + // See the Kotlin DSL description above for information about the options +} +``` + +## Maven + +The `org.apache.polaris.apprunner:polaris-apprunner-maven-plugin` Maven plugin should be used together with the +standard `maven-failsafe-plugin` + +`pom.xml` - note: the version number needs to be replaced with a (not yet existing) binary release of +Apache Polaris. + +```xml + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + + org.apache.polaris.apprunner + polaris-apprunner-maven-plugin + > + 0.0.0 + + + > + org.apache.polaris:polaris-quarkus-server:jar:runner:0.0.0 + + + bar + + + + world + + + + + + + start + pre-integration-test + + start + + + + + stop + post-integration-test + + stop + + + + + + + +``` + +## Implicit Quarkus options + +The plugins always pass the following configuration options as system properties to Quarkus: + +``` +quarkus.http.port=0 +quarkus.management.port=0 +quarkus.log.level=INFO +quarkus.log.console.level=INFO +``` + +Those are meant to let Quarkus bind to a random-ish port, so that the started instances do not conflict with anything +else running on the system and that the necessary log line containing the listen-URLs gets emitted. + +You can explicitly override those via the `systemProperties` options of the Gradle and Maven plugins. + +## Developing the plugins + +The Polaris Apprunner plugins are built via a Gradle "included build" (composite build). As generally with composite +builds, task selection via `gradlew` does _not_ get propagated to included builds. This is especially true for +tasks like `spotlessApply`. + +In other words, running `./gradlew spotlessApply` against the "main" Polaris build will run `spotlessApply` +_only_ in the projects in the "main" Polaris build, but _not_ in the apprunner build. This is also true for other +tasks like `check`. + +This means, the easiest way is to just change the current working directory to `tools/apprunner` and work from there. +Publishing the plugins also has to be done from the `tools/apprunner` directory. This is why `gradlew` & co are +present in `tools/apprunner`. + +## FAQ + +### Does it have to be a Polaris Quarkus server? + +The plugins work with any Quarkus application that listens for HTTP requests. + +The only requirement of the Polaris Apprunner plugins for the runnable jar is that it is a Quarkus (web) server, +that emits at least the HTTP listen URL (and optionally the management listen URL). + +This means, that this plugin can be used with basically any Quarkus based application, whether it's Apache Polaris +or Nessie or something else. + +### The plugin always times out starting the server, even if the server starts up + +Make sure that Quarkus emits a line like the following: + +``` +2025-01-16 13:29:25,959 INFO [io.quarkus] (main) Apache Polaris Server (incubating) 1.0.0-incubating-SNAPSHOT on JVM (powered by Quarkus 3.17.7) started in 0.998s. Listening on: http://0.0.0.0:8181. Management interface listening on http://0.0.0.0:8182. +``` + +The important part is `Listening on: http://0.0.0.0:8181. Management interface listening on http://0.0.0.0:8182.`, +especially the `Listening on: http://0.0.0.0:8181.` is mandatory, the port number used by Quarkus does not matter. + +If that line does not get logged to stdout, the Polaris Apprunner plugin does not detect the Quarkus application to +be running. Make sure that your Quarkus logging configuration allows logging this line to stdout. + +The plugins do their best to enforce that, + +## Origin of the Polaris Apprunner plugins + +The Polaris Apprunner Gradle and Maven plugins are based +on [projectnessie's apprunner](https://github.com/projectnessie/nessie-apprunner). diff --git a/tools/apprunner/apprunner-build-logic/build.gradle.kts b/tools/apprunner/apprunner-build-logic/build.gradle.kts new file mode 100644 index 0000000000..9b9df89543 --- /dev/null +++ b/tools/apprunner/apprunner-build-logic/build.gradle.kts @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { `kotlin-dsl` } + +dependencies { + implementation(gradleKotlinDsl()) + implementation(baselibs.errorprone) + implementation(baselibs.idea.ext) + implementation(baselibs.nexus.publish) + implementation(baselibs.shadow) + implementation(baselibs.spotless) +} diff --git a/tools/apprunner/apprunner-build-logic/settings.gradle.kts b/tools/apprunner/apprunner-build-logic/settings.gradle.kts new file mode 100644 index 0000000000..e4031cd145 --- /dev/null +++ b/tools/apprunner/apprunner-build-logic/settings.gradle.kts @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +dependencyResolutionManagement { + versionCatalogs { create("baselibs") { from(files("../gradle/baselibs.versions.toml")) } } +} + +dependencyResolutionManagement { + repositories { + mavenCentral() + gradlePluginPortal() + } +} diff --git a/tools/apprunner/apprunner-build-logic/src/main/kotlin/polaris-apprunner-java.gradle.kts b/tools/apprunner/apprunner-build-logic/src/main/kotlin/polaris-apprunner-java.gradle.kts new file mode 100644 index 0000000000..d1620df870 --- /dev/null +++ b/tools/apprunner/apprunner-build-logic/src/main/kotlin/polaris-apprunner-java.gradle.kts @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import com.diffplug.spotless.FormatterFunc +import java.io.Serializable +import net.ltgt.gradle.errorprone.errorprone +import publishing.PublishingHelperPlugin + +plugins { + `java-library` + `java-test-fixtures` + `jvm-test-suite` + id("com.diffplug.spotless") + id("net.ltgt.errorprone") +} + +apply() + +tasks.withType(JavaCompile::class.java).configureEach { + options.compilerArgs.addAll(listOf("-Xlint:unchecked", "-Xlint:deprecation")) + options.errorprone.disableAllWarnings = true + options.errorprone.disableWarningsInGeneratedCode = true + options.errorprone.error( + "DefaultCharset", + "FallThrough", + "MissingCasesInEnumSwitch", + "MissingOverride", + "ModifiedButNotUsed", + "OrphanedFormatString", + "PatternMatchingInstanceof", + "StringCaseLocaleUsage", + ) + options.release = 21 +} + +tasks.register("compileAll").configure { + group = "build" + description = "Runs all compilation and jar tasks" + dependsOn(tasks.withType(), tasks.withType()) +} + +tasks.register("format").configure { + group = "verification" + description = "Runs all code formatting tasks" + dependsOn("spotlessApply") +} + +tasks.named("test").configure { jvmArgs("-Duser.language=en") } + +testing { + suites { + withType { + val libs = versionCatalogs.named("libs") + + useJUnitJupiter( + libs + .findLibrary("junit-bom") + .orElseThrow { GradleException("junit-bom not declared in libs.versions.toml") } + .map { it.version!! } + ) + + dependencies { + implementation(project()) + implementation(testFixtures(project())) + implementation( + libs.findLibrary("assertj-core").orElseThrow { + GradleException("assertj-core not declared in libs.versions.toml") + } + ) + } + + targets.all { + if (testTask.name != "test") { + testTask.configure { shouldRunAfter("test") } + } + } + } + } +} + +dependencies { + val libs = versionCatalogs.named("libs") + testFixturesImplementation( + platform( + libs.findLibrary("junit-bom").orElseThrow { + GradleException("junit-bom not declared in libs.versions.toml") + } + ) + ) + testFixturesImplementation("org.junit.jupiter:junit-jupiter") + testFixturesImplementation( + libs.findLibrary("assertj-core").orElseThrow { + GradleException("assertj-core not declared in libs.versions.toml") + } + ) +} + +spotless { + java { + target("src/main/java/**/*.java", "src/testFixtures/java/**/*.java", "src/test/java/**/*.java") + googleJavaFormat() + licenseHeaderFile(rootProject.file("codestyle/copyright-header-java.txt")) + endWithNewline() + custom( + "disallowWildcardImports", + object : Serializable, FormatterFunc { + override fun apply(text: String): String { + val regex = "~/import .*\\.\\*;/".toRegex() + if (regex.matches(text)) { + throw GradleException("Wildcard imports disallowed - ${regex.findAll(text)}") + } + return text + } + }, + ) + toggleOffOn() + } + kotlinGradle { + ktfmt().googleStyle() + licenseHeaderFile(rootProject.file("codestyle/copyright-header-java.txt"), "$") + target("*.gradle.kts") + } + format("xml") { + target("src/**/*.xml", "src/**/*.xsd") + targetExclude("codestyle/copyright-header.xml") + eclipseWtp(com.diffplug.spotless.extra.wtp.EclipseWtpFormatterStep.XML) + .configFile(rootProject.file("codestyle/org.eclipse.wst.xml.core.prefs")) + // getting the license-header delimiter right is a bit tricky. + // licenseHeaderFile(rootProject.file("codestyle/copyright-header.xml"), '<^[!?].*$') + } +} + +dependencies { errorprone(versionCatalogs.named("libs").findLibrary("errorprone").get()) } + +java { + withJavadocJar() + withSourcesJar() +} + +tasks.withType().configureEach { + val opt = options as CoreJavadocOptions + // don't spam log w/ "warning: no @param/@return" + opt.addStringOption("Xdoclint:-reference", "-quiet") +} diff --git a/tools/apprunner/apprunner-build-logic/src/main/kotlin/polaris-apprunner-root.gradle.kts b/tools/apprunner/apprunner-build-logic/src/main/kotlin/polaris-apprunner-root.gradle.kts new file mode 100644 index 0000000000..c6988f2217 --- /dev/null +++ b/tools/apprunner/apprunner-build-logic/src/main/kotlin/polaris-apprunner-root.gradle.kts @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.gradle.kotlin.dsl.apply +import publishing.PublishingHelperPlugin + +plugins { id("com.diffplug.spotless") } + +apply() + +spotless { + kotlinGradle { + ktfmt().googleStyle() + licenseHeaderFile(rootProject.file("codestyle/copyright-header-java.txt"), "$") + target("*.gradle.kts", "build-logic/*.gradle.kts", "build-logic/src/**/*.kt*") + } +} + +tasks.register("compileAll").configure { + group = "build" + description = "Runs all compilation and jar tasks" + dependsOn(tasks.withType(), tasks.withType()) +} diff --git a/tools/apprunner/apprunner-build-logic/src/main/kotlin/publishing b/tools/apprunner/apprunner-build-logic/src/main/kotlin/publishing new file mode 120000 index 0000000000..12f2de4853 --- /dev/null +++ b/tools/apprunner/apprunner-build-logic/src/main/kotlin/publishing @@ -0,0 +1 @@ +../../../../../../build-logic/src/main/kotlin/publishing \ No newline at end of file diff --git a/tools/apprunner/build.gradle.kts b/tools/apprunner/build.gradle.kts new file mode 100644 index 0000000000..1eb4e355a9 --- /dev/null +++ b/tools/apprunner/build.gradle.kts @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.net.URI + +buildscript { repositories { maven { url = java.net.URI("https://plugins.gradle.org/m2/") } } } + +plugins { id("polaris-apprunner-root") } + +version = rootProject.rootDir.resolve("../../version.txt").readText().trim() + +publishingHelper { + asfProjectName = "polaris" + overrideName = "polaris-apprunner" + overrideDescription = "Polaris Apprunner Gradle + Maven plugins" +} + +// Pass environment variables: +// ORG_GRADLE_PROJECT_apacheUsername +// ORG_GRADLE_PROJECT_apachePassword +// OR in ~/.gradle/gradle.properties set +// apacheUsername +// apachePassword +// Call targets: +// publishToApache +// closeApacheStagingRepository +// releaseApacheStagingRepository +// or closeAndReleaseApacheStagingRepository +// +// Username is your ASF ID +// Password: your ASF LDAP password - or better: a token generated via +// https://repository.apache.org/ +nexusPublishing { + transitionCheckOptions { + // default==60 (10 minutes), wait up to 120 minutes + maxRetries = 720 + // default 10s + // delayBetween = java.time.Duration.ofSeconds(10) + } + + repositories { + register("apache") { + nexusUrl = URI.create("https://repository.apache.org/service/local/") + snapshotRepositoryUrl = + URI.create("https://repository.apache.org/content/repositories/snapshots/") + } + } +} diff --git a/tools/apprunner/codestyle b/tools/apprunner/codestyle new file mode 120000 index 0000000000..2c6dd3761f --- /dev/null +++ b/tools/apprunner/codestyle @@ -0,0 +1 @@ +../../codestyle \ No newline at end of file diff --git a/tools/apprunner/common/build.gradle.kts b/tools/apprunner/common/build.gradle.kts new file mode 100644 index 0000000000..89ca14e78d --- /dev/null +++ b/tools/apprunner/common/build.gradle.kts @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { id("polaris-apprunner-java") } + +dependencies { compileOnly(libs.jakarta.annotation.api) } diff --git a/tools/apprunner/common/src/main/java/org/apache/polaris/apprunner/common/InputBuffer.java b/tools/apprunner/common/src/main/java/org/apache/polaris/apprunner/common/InputBuffer.java new file mode 100644 index 0000000000..8e21fa8230 --- /dev/null +++ b/tools/apprunner/common/src/main/java/org/apache/polaris/apprunner/common/InputBuffer.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.apprunner.common; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.charset.Charset; +import java.util.function.Consumer; + +/** Captures input from an {@link InputStream} and emits full lines terminated with a {@code LF}. */ +final class InputBuffer { + + private final Reader input; + private final Consumer output; + private final StringBuilder lineBuffer = new StringBuilder(); + private boolean failed; + + InputBuffer(InputStream input, Consumer output) { + this(new BufferedReader(new InputStreamReader(input, Charset.defaultCharset())), output); + } + + InputBuffer(Reader input, Consumer output) { + this.input = input; + this.output = output; + } + + /** + * Drains the input passed to the constructor until there's no more data to read, captures full + * lines terminated with a {@code LF}) and pushes these lines to the consumer passed into the + * constructor. + * + * @return {@code true} if any data has been read from the input stream + */ + boolean io() { + // Note: cannot use BufferedReader.readLine() here, because that would block. + try { + if (failed || !input.ready()) { + return false; + } + + var any = false; + while (input.ready()) { + var c = input.read(); + + if (c == -1) { + return any; + } + + any = true; + switch (c) { + case 13 -> { // CR + } + case 10 -> { // LF + output.accept(lineBuffer.toString()); + lineBuffer.setLength(0); + } + default -> { + lineBuffer.append((char) c); + } + } + } + return true; + } catch (IOException e) { + e.printStackTrace(); + failed = true; + return false; + } + } + + void flush() { + if (!lineBuffer.isEmpty()) { + output.accept(lineBuffer.toString()); + lineBuffer.setLength(0); + } + } +} diff --git a/tools/apprunner/common/src/main/java/org/apache/polaris/apprunner/common/JavaVM.java b/tools/apprunner/common/src/main/java/org/apache/polaris/apprunner/common/JavaVM.java new file mode 100644 index 0000000000..bfbd3dcfd5 --- /dev/null +++ b/tools/apprunner/common/src/main/java/org/apache/polaris/apprunner/common/JavaVM.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.apprunner.common; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.function.IntFunction; +import java.util.regex.Pattern; + +/** + * Helper class to locate a JDK by Java major version and return the path to the {@code java} + * executable. + */ +public final class JavaVM { + + public static final int MAX_JAVA_VERSION_TO_CHECK = 19; + + private static final Pattern MAJOR_VERSION_PATTERN = Pattern.compile("^(1[.])?([0-9]+)([.].+)?$"); + + private static final AtomicReference CURRENT_JVM = new AtomicReference<>(); + private final Path javaHome; + + static String locateJavaHome( + int majorVersion, + Function getenv, + Function getProperty, + IntFunction macosHelper) { + + var home = getenv.apply(String.format("JDK%d_HOME", majorVersion)); + if (home != null) { + return home; + } + home = getenv.apply(String.format("JAVA%d_HOME", majorVersion)); + if (home != null) { + return home; + } + + home = getProperty.apply(String.format("jdk%d.home", majorVersion)); + if (home != null) { + return home; + } + home = getProperty.apply(String.format("java%d.home", majorVersion)); + if (home != null) { + return home; + } + + if (getProperty.apply("os.name").toLowerCase(Locale.ROOT).contains("darwin")) { + return macosHelper.apply(majorVersion); + } + + return null; + } + + /** + * Loops from {@code majorVersion} up to {@value #MAX_JAVA_VERSION_TO_CHECK} until a call to + * {@link #findJavaVMTryExact(int)} returns a Java-Home. + * + *

Returns the current JVM from {@link #getCurrentJavaVM()}, if its major version is greater + * than or equal to the requested {@code majorVersion}. + * + * @param majorVersion the Java major-version to start with. + * @return a Java-VM on the local system or {@code null}, if no matching Java-Home could be found. + */ + public static JavaVM findJavaVM(int majorVersion) { + if (currentJavaVMMajorVersion() >= majorVersion) { + return getCurrentJavaVM(); + } + for (var i = majorVersion; i < MAX_JAVA_VERSION_TO_CHECK; i++) { + var jvm = findJavaVMTryExact(i); + if (jvm != null) { + return jvm; + } + } + return null; + } + + /** + * Find a Java-Home that exactly matches the given Java major version. + * + *

Searches for the Java-Home in these places in this exact order: + * + *

    + *
  1. Environment variable {@code JDKxx_HOME}, where {@code xx} is the {@code majorVersion}. + *
  2. Environment variable {@code JAVAxx_HOME}, where {@code xx} is the {@code majorVersion}. + *
  3. System property {@code jdkXX.home}, where {@code XX} is the {@code majorVersion}. + *
  4. System property {@code javaXX.home}, where {@code XX} is the {@code majorVersion}. + *
  5. Using the {@code /usr/libexec/java_home} on MacOS, which may return a newer Java version. + *
+ * + * @param majorVersion the Java major-version to search for. + * @return a Java-VM on the local system or {@code null}, if no matching Java-Home could be found. + */ + public static JavaVM findJavaVMTryExact(int majorVersion) { + if (majorVersion == currentJavaVMMajorVersion()) { + return getCurrentJavaVM(); + } + + var home = + locateJavaHome( + majorVersion, + System::getenv, + System::getProperty, + ver -> { + try { + String versionArg = ver < 9 ? ("1." + ver) : Integer.toString(ver); + Process proc = + new ProcessBuilder() + .command("/usr/libexec/java_home", "-v", versionArg) + .start(); + return new BufferedReader( + new InputStreamReader(proc.getInputStream(), Charset.defaultCharset())) + .readLine(); + } catch (IOException e) { + return null; + } + }); + if (home != null) { + return forJavaHome(home); + } + return null; + } + + /** + * Get the {@link JavaVM} instance for the current JVM. + * + * @return {@link JavaVM} instance for the current JVM, never {@code null}. + */ + public static JavaVM getCurrentJavaVM() { + var current = CURRENT_JVM.get(); + if (current == null) { + CURRENT_JVM.set(current = forJavaHome(Paths.get(System.getProperty("java.home")))); + } + return current; + } + + public static int currentJavaVMMajorVersion() { + return majorVersionFromString(System.getProperty("java.version")); + } + + /** + * Extracts the major version from a Java-version-string as returned from {@code + * System.getProperty("java.version)}. + * + * @param versionString the Java-version-string + * @return extracted Java major version + */ + public static int majorVersionFromString(String versionString) { + var m = MAJOR_VERSION_PATTERN.matcher(versionString); + if (!m.matches()) { + throw new IllegalArgumentException( + String.format("%s is not a valid Java version string", versionString)); + } + return Integer.parseInt(m.group(2)); + } + + static Path fixJavaHome(Path javaHome) { + if ("jre".equals(javaHome.getFileName().toString()) + && Files.isExecutable(javaHome.resolve("bin").resolve(executableName("java")))) { + var check = javaHome.getParent(); + if (Files.isExecutable(check.resolve("bin").resolve(executableName("java")))) { + javaHome = check; + } + } + return javaHome; + } + + public static JavaVM forJavaHome(String javaHome) { + return forJavaHome(Paths.get(javaHome)); + } + + public static JavaVM forJavaHome(Path javaHome) { + return new JavaVM(fixJavaHome(javaHome)); + } + + private JavaVM(Path javaHome) { + this.javaHome = javaHome; + } + + public Path getJavaHome() { + return javaHome; + } + + public Path getJavaExecutable() { + return getExecutable("java"); + } + + private Path getExecutable(String executable) { + return javaHome.resolve("bin").resolve(executableName(executable)); + } + + static String executableName(String executable) { + if (System.getProperty("os.name").toLowerCase(Locale.ROOT).contains("windows")) { + return executable + ".exe"; + } + return executable; + } +} diff --git a/tools/apprunner/common/src/main/java/org/apache/polaris/apprunner/common/ListenUrlWaiter.java b/tools/apprunner/common/src/main/java/org/apache/polaris/apprunner/common/ListenUrlWaiter.java new file mode 100644 index 0000000000..9fd38a67cb --- /dev/null +++ b/tools/apprunner/common/src/main/java/org/apache/polaris/apprunner/common/ListenUrlWaiter.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.apprunner.common; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; +import java.util.function.LongSupplier; +import java.util.regex.Pattern; + +/** + * Accepts {@link String}s via it's {@link #accept(String)} method and checks for the {@code + * Listening on: http...} pattern. + */ +final class ListenUrlWaiter implements Consumer { + + private static final Pattern HTTP_PORT_LOG_PATTERN = + Pattern.compile( + "^.*Listening on: (http[s]?://[^ ]*)([.] Management interface listening on (http[s]?://[^ ]*)[.])?$"); + static final String TIMEOUT_MESSAGE = + "Did not get the http(s) listen URL from the console output."; + private static final long MAX_ITER_WAIT_NANOS = TimeUnit.MILLISECONDS.toNanos(50); + public static final String NOTHING_RECEIVED = " No output received from process."; + public static final String CAPTURED_LOG_FOLLOWS = " Captured output follows:\n"; + + private final LongSupplier clock; + private final Consumer stdoutTarget; + private final long deadlineListenUrl; + + private final CompletableFuture> listenUrl = new CompletableFuture<>(); + private final List capturedLog = new ArrayList<>(); + + /** + * Construct a new instance to wait for Quarkus' {@code Listening on: ...} message. + * + * @param clock monotonic clock, nanoseconds + * @param timeToListenUrlMillis timeout in millis, the "Listen on: ..." must be received within + * this time (otherwise it will fail) + * @param stdoutTarget "real" target for "stdout" + */ + ListenUrlWaiter(LongSupplier clock, long timeToListenUrlMillis, Consumer stdoutTarget) { + this.clock = clock; + this.stdoutTarget = stdoutTarget; + this.deadlineListenUrl = + clock.getAsLong() + TimeUnit.MILLISECONDS.toNanos(timeToListenUrlMillis); + } + + @Override + public void accept(String line) { + if (!listenUrl.isDone()) { + synchronized (capturedLog) { + capturedLog.add(line); + var m = HTTP_PORT_LOG_PATTERN.matcher(line); + if (m.matches()) { + listenUrl.complete(Arrays.asList(m.group(1), m.group(3))); + capturedLog.clear(); + } + } + } + stdoutTarget.accept(line); + } + + List peekListenUrls() { + try { + return listenUrl.isDone() ? listenUrl.get() : null; + } catch (Exception e) { + throw new RuntimeException(); + } + } + + /** + * Get the first captured {@code Listening on: http...} pattern. + * + * @return the captured listen URL or {@code null} if none has been found (so far). + */ + List getListenUrls() throws InterruptedException, TimeoutException { + while (true) { + var remainingNanos = remainingNanos(); + // must succeed if the listen-url has been captured, even if it's called after the timeout has + // elapsed + if (remainingNanos < 0 && !listenUrl.isDone()) { + throw getTimeoutException(null); + } + + try { + return listenUrl.get(Math.min(MAX_ITER_WAIT_NANOS, remainingNanos), TimeUnit.NANOSECONDS); + } catch (TimeoutException e) { + // Continue, check above. + // This "short get()" is implemented to make the unit test TestListenUrlWaiter.noTimeout() + // run faster. + } catch (ExecutionException e) { + if (e.getCause() instanceof TimeoutException) { + throw getTimeoutException(e.getCause()); + } + if (e.getCause() instanceof RuntimeException) { + throw (RuntimeException) e.getCause(); + } else { + throw new RuntimeException(e.getCause()); + } + } + } + } + + private TimeoutException getTimeoutException(Throwable cause) { + String log; + synchronized (capturedLog) { + log = String.join("\n", capturedLog); + } + var ex = + new TimeoutException( + TIMEOUT_MESSAGE + (log.isEmpty() ? NOTHING_RECEIVED : (CAPTURED_LOG_FOLLOWS + log))); + if (cause != null) { + ex.addSuppressed(cause); + } + return ex; + } + + void stopped(String reason) { + listenUrl.completeExceptionally(new RuntimeException(reason)); + } + + void timedOut() { + listenUrl.completeExceptionally(new TimeoutException()); + } + + public void exited(int exitCode) { + // No-op, if the listen-URL has already been received, so using the TIMEOUT_MESSAGE here is + // fine. + String log; + synchronized (capturedLog) { + log = String.join("\n", capturedLog); + } + listenUrl.completeExceptionally( + new RuntimeException( + ListenUrlWaiter.TIMEOUT_MESSAGE + + " Process exited early, exit code is " + + exitCode + + "." + + (log.isEmpty() ? NOTHING_RECEIVED : (CAPTURED_LOG_FOLLOWS + log)))); + } + + long remainingNanos() { + return deadlineListenUrl - clock.getAsLong(); + } + + boolean isTimeout() { + if (listenUrl.isDone() && !listenUrl.isCompletedExceptionally()) { + return false; + } + return remainingNanos() < 0; + } +} diff --git a/tools/apprunner/common/src/main/java/org/apache/polaris/apprunner/common/ProcessHandler.java b/tools/apprunner/common/src/main/java/org/apache/polaris/apprunner/common/ProcessHandler.java new file mode 100644 index 0000000000..0dd9bcf3f6 --- /dev/null +++ b/tools/apprunner/common/src/main/java/org/apache/polaris/apprunner/common/ProcessHandler.java @@ -0,0 +1,294 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.apprunner.common; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.function.LongSupplier; + +/** + * Handles the execution of an external process, focused on running a Quarkus application jar. + * + *

Starts the process configured in a {@link ProcessBuilder}, provides a method to get the {@link + * #getListenUrls() Quarkus HTTP listen URL} as Quarkus prints to stdout, and manages process + * lifetime and line-by-line I/O pass-through for stdout + stderr. + * + *

Any instance of this class can only be used to start (and stop) one process and cannot be + * reused for another process. + * + *

This implementation is not thread-safe. + */ +public class ProcessHandler { + + // intentionally long timeouts - think: slow CI systems + public static final long MILLIS_TO_HTTP_PORT = 30_000L; + public static final long MILLIS_TO_STOP = 15_000L; + + private LongSupplier ticker = System::nanoTime; + + private static final int NOT_STARTED = -1; + private static final int RUNNING = -2; + private static final int ERROR = -3; + private final AtomicInteger exitCode = new AtomicInteger(NOT_STARTED); + + private final AtomicBoolean stopped = new AtomicBoolean(); + + private Process process; + + private long timeToListenUrlMillis = MILLIS_TO_HTTP_PORT; + private long timeStopMillis = MILLIS_TO_STOP; + + private Consumer stdoutTarget = System.out::println; + private ListenUrlWaiter listenUrlWaiter; + + private volatile ExecutorService watchdogExecutor; + private volatile Future watchdogFuture; + private volatile Thread shutdownHook; + + public ProcessHandler() { + // empty + } + + public ProcessHandler setTimeToListenUrlMillis(long timeToListenUrlMillis) { + this.timeToListenUrlMillis = timeToListenUrlMillis; + return this; + } + + public ProcessHandler setTimeStopMillis(long timeStopMillis) { + this.timeStopMillis = timeStopMillis; + return this; + } + + public ProcessHandler setStdoutTarget(Consumer stdoutTarget) { + this.stdoutTarget = stdoutTarget; + return this; + } + + public ProcessHandler setTicker(LongSupplier ticker) { + this.ticker = ticker; + return this; + } + + /** + * Starts the process from the given {@link ProcessBuilder}. + * + * @param processBuilder process to start + * @return instance handling the process' runtime + * @throws IOException usually, if the process fails to start + */ + public ProcessHandler start(ProcessBuilder processBuilder) throws IOException { + if (process != null) { + throw new IllegalStateException("Process already started"); + } + + return started(processBuilder.redirectErrorStream(true).start()); + } + + /** + * Alternative to {@link #start(ProcessBuilder)}, directly configures a running process. + * + * @param process running process + * @return {@code this} + */ + ProcessHandler started(Process process) { + if (this.process != null) { + throw new IllegalStateException("Process already started"); + } + + listenUrlWaiter = new ListenUrlWaiter(ticker, timeToListenUrlMillis, stdoutTarget); + + this.process = process; + exitCode.set(RUNNING); + + shutdownHook = new Thread(this::shutdownHandler); + Runtime.getRuntime().addShutdownHook(shutdownHook); + + watchdogExecutor = Executors.newSingleThreadExecutor(); + watchdogFuture = watchdogExecutor.submit(this::watchdog); + + return this; + } + + /** + * Returns the http(s) listen URL as a string as emitted to stdout by Quarkus. + * + *

If the Quarkus process does not emit that URL within the time configured via {@link + * #setTimeToListenUrlMillis(long)}, which defaults to {@value #MILLIS_TO_HTTP_PORT} ms, this + * method will throw an {@link IllegalStateException}. + * + * @return the listen URL, never {@code null}. + * @throws InterruptedException if the current thread was interrupted while waiting for the listen + * URL. + * @throws TimeoutException if the Quarkus process did not write the listen URL to stdout. + */ + public List getListenUrls() throws InterruptedException, TimeoutException { + return listenUrlWaiter.getListenUrls(); + } + + /** + * Stops the process. + * + *

Tries to gracefully stop the process via a {@code SIGTERM}. If the process is still alive + * after {@link #setTimeStopMillis(long)}, which defaults to {@value #MILLIS_TO_STOP} ms, the + * process will be killed with a {@code SIGKILL}. + */ + public void stop() { + if (process == null) { + throw new IllegalStateException("No process started"); + } + + doStop("Stopped by plugin"); + + watchdogExitGrace(); + } + + private void shutdownHandler() { + doStop("Stop by shutdown handler"); + } + + private void doStop(String reason) { + if (stopped.compareAndSet(false, true)) { + try { + if (reason != null) { + listenUrlWaiter.stopped(reason); + } else { + listenUrlWaiter.timedOut(); + } + process.destroy(); + try { + if (!process.waitFor(timeStopMillis, TimeUnit.MILLISECONDS)) { + process.destroyForcibly(); + } + } catch (InterruptedException e) { + process.destroyForcibly(); + Thread.currentThread().interrupt(); + } + watchdogExecutor.shutdown(); + } finally { + try { + // Don't remove the shutdown-hook if we're running in the shutdown-hook + Runtime.getRuntime().removeShutdownHook(shutdownHook); + } catch (IllegalStateException e) { + // ignore (might happen, when a JVM shutdown is already in progress) + } + } + } + } + + void watchdogExitGrace() { + try { + // Give the watchdog task/thread some time to finish its work + watchdogFuture.get(timeStopMillis, TimeUnit.MILLISECONDS); + } catch (ExecutionException e) { + throw new RuntimeException("ProcessHandler's watchdog thread failed.", e); + } catch (TimeoutException e) { + throw new IllegalStateException("ProcessHandler's watchdog thread failed to finish in time."); + } catch (InterruptedException e) { + process.destroyForcibly(); + Thread.currentThread().interrupt(); + } + } + + public boolean isAlive() { + return exitCode.get() == RUNNING; + } + + /** + * Retrieves the exit-code of the process, if it terminated or throws a {@link + * IllegalThreadStateException} if it is still alive. + * + * @return the exit code of the process + * @throws IllegalThreadStateException if the process is still alive + */ + public int getExitCode() throws IllegalThreadStateException { + if (isAlive()) { + throw new IllegalThreadStateException(); + } + return exitCode.get(); + } + + long remainingWaitTimeNanos() { + return listenUrlWaiter.remainingNanos(); + } + + private Object watchdog() throws IOException { + try (var out = process.getInputStream()) { + var stdout = new InputBuffer(out, listenUrlWaiter); + try { + + /* + * I/O loop. + * + * Fetches data from stdout + stderr and pushes the read data to the associated `InputBuffer` + * instances. The one for `stdout` listens for the HTTP listen address from Quarkus. + * + * As long as there is data from stdout or stderr, the loop does not wait/sleep to get data + * out as fast as possible. If there's no data available, the loop will "yield" via a + * Thread.sleep(1L), which is good enough. + * + * Note: we cannot do blocking-I/O here, because we have to read from both stdout+stderr. + * + * If the process exits, the loop will exit as soon as there is no more data left from + * stdout/stderr. + */ + while (true) { + var anyIo = stdout.io(); + + try { + var ec = process.exitValue(); + exitCode.set(ec); + if (!anyIo) { + listenUrlWaiter.exited(exitCode.get()); + break; + } + } catch (IllegalThreadStateException e) { + // server still alive + } + + if (listenUrlWaiter.isTimeout() && !stopped.get()) { + doStop(null); + } + + if (!anyIo) { + try { + // Yield CPU for a little while, so this background thread does not consume 100% CPU. + Thread.sleep(1L); + } catch (InterruptedException interruptedException) { + doStop("ProcessHandler's watchdog thread interrupted."); + exitCode.set(ERROR); + break; + } + } + } + } finally { + stdout.flush(); + } + } + return null; + } +} diff --git a/tools/apprunner/common/src/test/java/org/apache/polaris/apprunner/common/TestInputBuffer.java b/tools/apprunner/common/src/test/java/org/apache/polaris/apprunner/common/TestInputBuffer.java new file mode 100644 index 0000000000..9ca258a3bc --- /dev/null +++ b/tools/apprunner/common/src/test/java/org/apache/polaris/apprunner/common/TestInputBuffer.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.apprunner.common; + +import java.io.StringReader; +import java.util.ArrayList; +import java.util.concurrent.ArrayBlockingQueue; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(SoftAssertionsExtension.class) +class TestInputBuffer { + @InjectSoftAssertions SoftAssertions soft; + + @Test + void emptyInput() { + var lines = new ArrayList(); + var buf = new InputBuffer(new StringReader(""), lines::add); + soft.assertThat(buf.io()).isFalse(); + soft.assertThat(lines).isEmpty(); + buf.flush(); + soft.assertThat(lines).isEmpty(); + } + + @Test + void scattered() { + var characters = new ArrayBlockingQueue(250); + + // Just need some Reader implementation that implements ready() + read() + var reader = + new StringReader("") { + @Override + public int read() { + return characters.poll(); + } + + @Override + public boolean ready() { + return characters.size() > 0; + } + }; + + var lines = new ArrayList(); + var buf = new InputBuffer(reader, lines::add); + soft.assertThat(buf.io()).isFalse(); + soft.assertThat(lines).isEmpty(); + + for (var c : "Hello World".toCharArray()) { + characters.add((int) c); + } + + // It should have done some I/O ... + soft.assertThat(buf.io()).isTrue(); + // ... but there was no trailing newline, so nothing to print (yet) + soft.assertThat(lines).isEmpty(); + + for (var c : "\nFoo Bar Baz\nMeep".toCharArray()) { + characters.add((int) c); + } + + // It should have done some I/O ... + soft.assertThat(buf.io()).isTrue(); + // ... and give us the first two lines ("Meep" is on an unterminated line) + soft.assertThat(lines).containsExactly("Hello World", "Foo Bar Baz"); + + // Just a CR does not trigger a "line complete" + characters.add(13); + soft.assertThat(buf.io()).isTrue(); + soft.assertThat(lines).containsExactly("Hello World", "Foo Bar Baz"); + + // ... but a LF does + characters.add(10); + soft.assertThat(buf.io()).isTrue(); + soft.assertThat(lines).containsExactly("Hello World", "Foo Bar Baz", "Meep"); + + // Add some more data, with an unterminated line... + for (char c : "\nMore text\nNo EOL".toCharArray()) { + characters.add((int) c); + } + soft.assertThat(buf.io()).isTrue(); + soft.assertThat(lines).containsExactly("Hello World", "Foo Bar Baz", "Meep", "", "More text"); + + // "Final" flush() should yield the remaining data + buf.flush(); + soft.assertThat(lines) + .containsExactly("Hello World", "Foo Bar Baz", "Meep", "", "More text", "No EOL"); + } +} diff --git a/tools/apprunner/common/src/test/java/org/apache/polaris/apprunner/common/TestJavaVM.java b/tools/apprunner/common/src/test/java/org/apache/polaris/apprunner/common/TestJavaVM.java new file mode 100644 index 0000000000..2f8e95bea7 --- /dev/null +++ b/tools/apprunner/common/src/test/java/org/apache/polaris/apprunner/common/TestJavaVM.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.apprunner.common; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.HashMap; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; + +@ExtendWith(SoftAssertionsExtension.class) +class TestJavaVM { + @InjectSoftAssertions SoftAssertions soft; + + @Test + void checkJavaVersionStrings() { + soft.assertThat(JavaVM.majorVersionFromString("11")).isEqualTo(11); + soft.assertThat(JavaVM.majorVersionFromString("17.0.1")).isEqualTo(17); + soft.assertThat(JavaVM.majorVersionFromString("1.8.0-foo+bar")).isEqualTo(8); + } + + @Test + void checkResolveEnvJdkHomeLinux() { + var env = new HashMap(); + env.put("JDK11_HOME", "/mycomputer/java11"); + env.put("JAVA17_HOME", "/mycomputer/java17"); + + var sysProps = new HashMap(); + sysProps.put("jdk9.home", "/mycomputer/java9"); + sysProps.put("java10.home", "/mycomputer/java10"); + sysProps.put("os.name", "Linux"); + + soft.assertThat(JavaVM.locateJavaHome(8, env::get, sysProps::get, i -> "/hello/there")) + .isNull(); + soft.assertThat(JavaVM.locateJavaHome(9, env::get, sysProps::get, i -> "/hello/there")) + .isEqualTo("/mycomputer/java9"); + soft.assertThat(JavaVM.locateJavaHome(10, env::get, sysProps::get, i -> "/hello/there")) + .isEqualTo("/mycomputer/java10"); + soft.assertThat(JavaVM.locateJavaHome(11, env::get, sysProps::get, i -> "/hello/there")) + .isEqualTo("/mycomputer/java11"); + soft.assertThat(JavaVM.locateJavaHome(14, env::get, sysProps::get, i -> "/hello/there")) + .isNull(); + soft.assertThat(JavaVM.locateJavaHome(17, env::get, sysProps::get, i -> "/hello/there")) + .isEqualTo("/mycomputer/java17"); + } + + @Test + void checkResolveEnvJdkHomeMacOS() { + var env = new HashMap(); + env.put("JDK11_HOME", "/mycomputer/java11"); + env.put("JAVA17_HOME", "/mycomputer/java17"); + + var sysProps = new HashMap(); + sysProps.put("jdk9.home", "/mycomputer/java9"); + sysProps.put("java10.home", "/mycomputer/java10"); + sysProps.put("os.name", "Darwin"); + + soft.assertThat( + JavaVM.locateJavaHome( + 8, env::get, sysProps::get, i -> i == 8 ? "/from_java_home/v8" : null)) + .isEqualTo("/from_java_home/v8"); + soft.assertThat(JavaVM.locateJavaHome(9, env::get, sysProps::get, i -> "/hello/there")) + .isEqualTo("/mycomputer/java9"); + soft.assertThat(JavaVM.locateJavaHome(10, env::get, sysProps::get, i -> "/hello/there")) + .isEqualTo("/mycomputer/java10"); + soft.assertThat(JavaVM.locateJavaHome(11, env::get, sysProps::get, i -> "/hello/there")) + .isEqualTo("/mycomputer/java11"); + soft.assertThat( + JavaVM.locateJavaHome( + 14, env::get, sysProps::get, i -> i >= 12 ? "/from_java_home/v16" : null)) + .isEqualTo("/from_java_home/v16"); + soft.assertThat( + JavaVM.locateJavaHome( + 17, env::get, sysProps::get, i -> i >= 12 ? "/from_java_home/v8" : null)) + .isEqualTo("/mycomputer/java17"); + } + + @Test + void checkJreResolve(@TempDir Path jdkDir) throws Exception { + var jdkBinDir = jdkDir.resolve("bin"); + var jdkJavaFile = jdkBinDir.resolve(JavaVM.executableName("java")); + var jreDir = jdkDir.resolve("jre"); + var jreBinDir = jreDir.resolve("bin"); + var jreJavaFile = jreBinDir.resolve(JavaVM.executableName("java")); + + Files.createDirectories(jdkBinDir); + Files.createDirectories(jreBinDir); + Files.createFile( + jreJavaFile, + PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwxr-x---"))); + + var env = new HashMap(); + env.put("JDK8_HOME", jreDir.toString()); + + var sysProps = new HashMap(); + sysProps.put("os.name", "Linux"); + + soft.assertThat(JavaVM.locateJavaHome(8, env::get, sysProps::get, x -> null)) + .isEqualTo(jreDir.toString()); + soft.assertThat(JavaVM.fixJavaHome(jreDir)).isEqualTo(jreDir); + soft.assertThat(JavaVM.forJavaHome(jreDir).getJavaExecutable()).isEqualTo(jreJavaFile); + soft.assertThat(JavaVM.forJavaHome(jreDir).getJavaHome()).isEqualTo(jreDir); + soft.assertThat(JavaVM.forJavaHome(jreDir.toString()).getJavaHome()).isEqualTo(jreDir); + + Files.createFile( + jdkJavaFile, + PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwxr-x---"))); + + soft.assertThat(JavaVM.fixJavaHome(jreDir)).isEqualTo(jdkDir); + soft.assertThat(JavaVM.forJavaHome(jreDir).getJavaExecutable()).isEqualTo(jdkJavaFile); + soft.assertThat(JavaVM.forJavaHome(jreDir).getJavaHome()).isEqualTo(jdkDir); + soft.assertThat(JavaVM.forJavaHome(jreDir.toString()).getJavaHome()).isEqualTo(jdkDir); + } +} diff --git a/tools/apprunner/common/src/test/java/org/apache/polaris/apprunner/common/TestListenUrlWaiter.java b/tools/apprunner/common/src/test/java/org/apache/polaris/apprunner/common/TestListenUrlWaiter.java new file mode 100644 index 0000000000..15ce06f510 --- /dev/null +++ b/tools/apprunner/common/src/test/java/org/apache/polaris/apprunner/common/TestListenUrlWaiter.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.apprunner.common; + +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.RepeatedTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(SoftAssertionsExtension.class) +class TestListenUrlWaiter { + @InjectSoftAssertions SoftAssertions soft; + + private static ExecutorService executor; + + @BeforeAll + static void createExecutor() { + executor = Executors.newCachedThreadPool(); + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + @AfterAll + static void stopExecutor() throws Exception { + executor.shutdown(); + executor.awaitTermination(10, TimeUnit.SECONDS); + } + + @Test + void ioHandling() { + var clock = new AtomicLong(); + var timeout = 10L; + + var line = new AtomicReference<>(); + var waiter = new ListenUrlWaiter(clock::get, timeout, line::set); + + waiter.accept("Hello World"); + soft.assertThat(line.getAndSet(null)).isEqualTo("Hello World"); + soft.assertThat(waiter.peekListenUrls()).isNull(); + + waiter.accept(""); + soft.assertThat(line.getAndSet(null)).isEqualTo(""); + soft.assertThat(waiter.peekListenUrls()).isNull(); + + var listenLine = + "2021-05-28 12:12:25,753 INFO [io.quarkus] (main) nessie-quarkus 0.6.2-SNAPSHOT on JVM (powered by Quarkus 1.13.4.Final) started in 1.444s. Listening on: http://0.0.0.0:39423. Management interface listening on http://0.0.0.0:9000."; + waiter.accept(listenLine); + soft.assertThat(line.getAndSet(null)).isEqualTo(listenLine); + soft.assertThat(waiter.peekListenUrls()) + .containsExactly("http://0.0.0.0:39423", "http://0.0.0.0:9000"); + + // Must *not* change the already extracted listen-url + listenLine = + "2021-05-28 12:12:25,753 INFO [io.quarkus] (main) nessie-quarkus 0.6.2-SNAPSHOT on JVM (powered by Quarkus 1.13.4.Final) started in 1.444s. Listening on: http://4.2.4.2:4242"; + waiter.accept(listenLine); + soft.assertThat(line.getAndSet(null)).isEqualTo(listenLine); + soft.assertThat(waiter.peekListenUrls()) + .containsExactly("http://0.0.0.0:39423", "http://0.0.0.0:9000"); + + waiter = new ListenUrlWaiter(clock::get, timeout, line::set); + listenLine = + "2021-05-28 12:12:25,753 INFO [io.quarkus] (main) nessie-quarkus 0.6.2-SNAPSHOT on JVM (powered by Quarkus 1.13.4.Final) started in 1.444s. Listening on: https://localhost.in.some.space:12345"; + waiter.accept(listenLine); + soft.assertThat(line.getAndSet(null)).isEqualTo(listenLine); + soft.assertThat(waiter.peekListenUrls()) + .containsExactly("https://localhost.in.some.space:12345", null); + + waiter = new ListenUrlWaiter(clock::get, timeout, line::set); + listenLine = "Listening on: https://localhost.in.some.space:4242"; + waiter.accept(listenLine); + soft.assertThat(line.getAndSet(null)).isEqualTo(listenLine); + soft.assertThat(waiter.peekListenUrls()) + .containsExactly("https://localhost.in.some.space:4242", null); + } + + @RepeatedTest(20) // repeat, risk of flakiness + void timeout() { + var clock = new AtomicLong(); + var timeout = 10_000L; // long timeout, for slow CI + + var line = new AtomicReference<>(); + var waiter = new ListenUrlWaiter(clock::get, timeout, line::set); + + soft.assertThat(waiter.isTimeout()).isFalse(); + + clock.set(TimeUnit.MILLISECONDS.toNanos(timeout + 1)); + + soft.assertThat(waiter.isTimeout()).isTrue(); + + soft.assertThat(executor.submit(waiter::getListenUrls)) + .failsWithin(5, TimeUnit.SECONDS) + .withThrowableOfType(ExecutionException.class) + .withRootCauseExactlyInstanceOf(TimeoutException.class) + .withMessageEndingWith(ListenUrlWaiter.TIMEOUT_MESSAGE + ListenUrlWaiter.NOTHING_RECEIVED); + } + + @RepeatedTest(20) // repeat, risk of flakiness + void noTimeout() throws Exception { + var clock = new AtomicLong(); + var timeout = 10_000L; // long timeout, for slow CI + + // Note: the implementation uses "our clock" to check the timeout, but uses a "standard + // Future.get(time)" for the actual get. + + var line = new AtomicReference<>(); + var waiter = new ListenUrlWaiter(clock::get, timeout, line::set); + + // Clock exactly at the timeout-boundary is not a timeout + clock.set(TimeUnit.MILLISECONDS.toNanos(timeout)); + + var listenLine = + "2021-05-28 12:12:25,753 INFO [io.quarkus] (main) nessie-quarkus 0.6.2-SNAPSHOT on JVM (powered by Quarkus 1.13.4.Final) started in 1.444s. Listening on: http://4.2.4.2:4242. Management interface listening on http://4.2.4.2:2424."; + waiter.accept(listenLine); + soft.assertThat(line.getAndSet(null)).isEqualTo(listenLine); + soft.assertThat(waiter.getListenUrls()) + .containsExactly("http://4.2.4.2:4242", "http://4.2.4.2:2424"); + soft.assertThat(waiter.isTimeout()).isFalse(); + + // Clock post the timeout-boundary (so a timeout-check would trigger) + clock.set(TimeUnit.MILLISECONDS.toNanos(timeout + 1)); + soft.assertThat(waiter.getListenUrls()) + .containsExactly("http://4.2.4.2:4242", "http://4.2.4.2:2424"); + soft.assertThat(waiter.isTimeout()).isFalse(); + } +} diff --git a/tools/apprunner/common/src/test/java/org/apache/polaris/apprunner/common/TestProcessHandler.java b/tools/apprunner/common/src/test/java/org/apache/polaris/apprunner/common/TestProcessHandler.java new file mode 100644 index 0000000000..bc3ccc65c0 --- /dev/null +++ b/tools/apprunner/common/src/test/java/org/apache/polaris/apprunner/common/TestProcessHandler.java @@ -0,0 +1,325 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.apprunner.common; + +import static java.util.concurrent.TimeUnit.SECONDS; + +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.RepeatedTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(SoftAssertionsExtension.class) +class TestProcessHandler { + @InjectSoftAssertions SoftAssertions soft; + + private static ExecutorService executor; + + @BeforeAll + static void createExecutor() { + executor = Executors.newCachedThreadPool(); + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + @AfterAll + static void stopExecutor() throws Exception { + executor.shutdown(); + executor.awaitTermination(10, SECONDS); + } + + @Test + void notStarted() { + var phMock = new ProcessHandlerMock(); + + soft.assertThatThrownBy(phMock.ph::stop) + .isInstanceOf(IllegalStateException.class) + .hasMessage("No process started"); + } + + @Test + void doubleStart() { + var phMock = new ProcessHandlerMock(); + + phMock.ph.started(phMock.proc); + + soft.assertThatThrownBy(() -> phMock.ph.started(phMock.proc)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Process already started"); + } + + @RepeatedTest(20) + // repeat, risk of flakiness + void processWithNoOutput() { + var phMock = new ProcessHandlerMock(); + + phMock.ph.started(phMock.proc); + + var futureListenUrl = executor.submit(phMock.ph::getListenUrls); + + while (phMock.clock.get() < TimeUnit.MILLISECONDS.toNanos(phMock.timeToUrl)) { + soft.assertThat(futureListenUrl).isNotDone(); + soft.assertAll(); + phMock.clock.addAndGet(TimeUnit.MILLISECONDS.toNanos(10)); + } + // should be exactly at (but not "past") the time to wait for the listen-url now + + // bump the clock "past" the listen-url-timeout + phMock.clock.addAndGet(TimeUnit.MILLISECONDS.toNanos(10)); + + soft.assertThat(futureListenUrl) + .failsWithin(5, SECONDS) + .withThrowableOfType(ExecutionException.class) // EE from ForkJoinPool/executor (test code) + .withRootCauseInstanceOf( + TimeoutException.class) // TE from ProcessHandler/ListenUrlWaiter.getListenUrl + .withMessageEndingWith(ListenUrlWaiter.TIMEOUT_MESSAGE + ListenUrlWaiter.NOTHING_RECEIVED); + + // Need to wait for the watchdog to finish, before we can do any further assertion + phMock.ph.watchdogExitGrace(); + + soft.assertThat(phMock.ph.isAlive()).isFalse(); + } + + @RepeatedTest(20) + // repeat, risk of flakiness + void processExitsEarly() { + var phMock = new ProcessHandlerMock(); + + phMock.ph.started(phMock.proc); + + var futureListenUrl = executor.submit(phMock.ph::getListenUrls); + + soft.assertThat(phMock.ph.isAlive()).isTrue(); + soft.assertThatThrownBy(() -> phMock.ph.getExitCode()) + .isInstanceOf(IllegalThreadStateException.class); + + phMock.exitCode.set(88); + + soft.assertThat(futureListenUrl) + .failsWithin(5, SECONDS) + .withThrowableOfType(ExecutionException.class) // EE from ForkJoinPool/executor (test code) + .withMessageEndingWith( + ListenUrlWaiter.TIMEOUT_MESSAGE + + " Process exited early, exit code is 88." + + ListenUrlWaiter.NOTHING_RECEIVED); + + // Need to wait for the watchdog to finish, before we can do any further assertion + phMock.ph.watchdogExitGrace(); + + soft.assertThat(phMock.ph.isAlive()).isFalse(); + soft.assertThat(phMock.ph.getExitCode()).isEqualTo(88); + } + + @RepeatedTest(20) + // repeat, risk of flakiness + void processLotsOfIoNoListen() throws Exception { + var phMock = new ProcessHandlerMock(); + + phMock.ph.started(phMock.proc); + + var futureListenUrl = executor.submit(phMock.ph::getListenUrls); + + var stdoutMessage = "Hello world\n"; + var message = stdoutMessage.toCharArray(); + while (phMock.clock.get() < TimeUnit.MILLISECONDS.toNanos(phMock.timeToUrl)) { + for (var c : message) { + phMock.stdout.add((byte) c); + } + soft.assertThat(futureListenUrl).isNotDone(); + soft.assertAll(); + phMock.clock.addAndGet(TimeUnit.MILLISECONDS.toNanos(10)); + } + // should be exactly at (but not "past") the time to wait for the listen-url now + + soft.assertThat(phMock.ph.remainingWaitTimeNanos()).isEqualTo(0); + soft.assertThat(phMock.ph.isAlive()).isTrue(); + + var timeoutFail = System.currentTimeMillis() + SECONDS.toMillis(10); + while (!phMock.stdout.isEmpty()) { + soft.assertThat(System.currentTimeMillis() < timeoutFail).isTrue(); + soft.assertAll(); + Thread.sleep(1L); + } + + // bump the clock "past" the listen-url-timeout + phMock.clock.addAndGet(TimeUnit.MILLISECONDS.toNanos(10)); + + soft.assertThat(futureListenUrl) + .failsWithin(5, SECONDS) + .withThrowableOfType(ExecutionException.class) // EE from ForkJoinPool/executor (test code) + .withRootCauseInstanceOf( + TimeoutException.class) // TE from ProcessHandler/ListenUrlWaiter.getListenUrl + .withMessageContaining( + ListenUrlWaiter.TIMEOUT_MESSAGE + ListenUrlWaiter.CAPTURED_LOG_FOLLOWS) + .withMessageContaining(stdoutMessage); + + // Need to wait for the watchdog to finish, before we can do any further assertion + phMock.ph.watchdogExitGrace(); + + soft.assertThat(phMock.ph.isAlive()).isFalse(); + soft.assertThat(phMock.ph.getExitCode()).isGreaterThanOrEqualTo(0); + + soft.assertThat(phMock.stdoutLines).hasSize((int) (phMock.timeToUrl / 10)); + } + + @RepeatedTest(20) + // repeat, risk of flakiness + void processLotsOfIoProperListenUrl() { + var phMock = new ProcessHandlerMock(); + + phMock.ph.started(phMock.proc); + + var futureListenUrl = executor.submit(phMock.ph::getListenUrls); + + while (phMock.clock.get() < TimeUnit.MILLISECONDS.toNanos(phMock.timeToUrl / 2)) { + for (var c : "Hello world\n".toCharArray()) { + phMock.stdout.add((byte) c); + } + soft.assertThat(futureListenUrl).isNotDone(); + soft.assertAll(); + phMock.clock.addAndGet(TimeUnit.MILLISECONDS.toNanos(10)); + } + // should be exactly at (but not "past") the time to wait for the listen-url now + + for (var c : "Quarkus startup message... Listening on: http://0.0.0.0:4242\n".toCharArray()) { + phMock.stdout.add((byte) c); + } + + // bump the clock "past" the listen-url-timeout + phMock.clock.addAndGet(TimeUnit.MILLISECONDS.toNanos(10)); + + soft.assertThat(futureListenUrl) + .succeedsWithin(5, SECONDS) + .isEqualTo(Arrays.asList("http://0.0.0.0:4242", null)); + + soft.assertThat(phMock.ph.isAlive()).isTrue(); + soft.assertThatThrownBy(() -> phMock.ph.getExitCode()) + .isInstanceOf(IllegalThreadStateException.class); + + // The .stop() waits until the watchdog has finished its work + phMock.ph.stop(); + + soft.assertThat(phMock.ph.isAlive()).isFalse(); + soft.assertThat(phMock.ph.getExitCode()).isGreaterThanOrEqualTo(0); + + soft.assertThat(phMock.stdoutLines).hasSize((int) (phMock.timeToUrl / 10 / 2) + 1); + } + + static final class ProcessHandlerMock { + + AtomicLong clock = new AtomicLong(); + + AtomicInteger exitCode = new AtomicInteger(-1); + + // Full lines received "form the process" via stdout/stderr is collected in these lists + List stdoutLines = Collections.synchronizedList(new ArrayList<>()); + + // Data that's "written by the process" to stdout/stderr is "piped" through these queues + ArrayBlockingQueue stdout = new ArrayBlockingQueue<>(1024); + + @SuppressWarnings("InputStreamSlowMultibyteRead") + InputStream stdoutStream = + new InputStream() { + @Override + public int available() { + return stdout.size(); + } + + @Override + public int read() { + var b = stdout.poll(); + return b == null ? -1 : b.intValue(); + } + }; + + Process proc = + new Process() { + @Override + public OutputStream getOutputStream() { + throw new UnsupportedOperationException(); + } + + @Override + public InputStream getInputStream() { + return stdoutStream; + } + + @Override + public InputStream getErrorStream() { + return stdoutStream; + } + + @Override + public int waitFor() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean waitFor(long timeout, TimeUnit unit) throws InterruptedException { + return super.waitFor(timeout, unit); + } + + @Override + public int exitValue() { + var ec = exitCode.get(); + if (ec < 0) { + throw new IllegalThreadStateException(); + } + return ec; + } + + @Override + public void destroy() { + exitCode.set(42); + } + + @Override + public Process destroyForcibly() { + exitCode.set(42); + return this; + } + }; + + long timeToUrl = 500; + + ProcessHandler ph = + new ProcessHandler() + .setStdoutTarget(stdoutLines::add) + .setTicker(clock::get) + .setTimeToListenUrlMillis(timeToUrl) + .setTimeStopMillis(42); + } +} diff --git a/tools/apprunner/gradle-plugin/build.gradle.kts b/tools/apprunner/gradle-plugin/build.gradle.kts new file mode 100644 index 0000000000..b50626c194 --- /dev/null +++ b/tools/apprunner/gradle-plugin/build.gradle.kts @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + id("polaris-apprunner-java") + `java-gradle-plugin` +} + +dependencies { + compileOnly(libs.jakarta.annotation.api) + implementation(project(":polaris-apprunner-common")) +} + +gradlePlugin { + plugins { + register("polaris-apprunner") { + id = "org.apache.polaris.apprunner" + implementationClass = "org.apache.polaris.apprunner.plugin.PolarisRunnerPlugin" + displayName = "Polaris Runner" + description = "Start and stop a Polaris server for integration testing" + tags.addAll("test", "integration", "quarkus", "polaris") + } + } + website.set("https://polaris.apache.org") + vcsUrl.set("https://github.com/apache/polaris") +} + +tasks.named("test") { + systemProperties("polaris-version" to version, "junit-version" to libs.junit.bom.get().version) +} diff --git a/tools/apprunner/gradle-plugin/src/main/java/org/apache/polaris/apprunner/plugin/PolarisRunnerExtension.java b/tools/apprunner/gradle-plugin/src/main/java/org/apache/polaris/apprunner/plugin/PolarisRunnerExtension.java new file mode 100644 index 0000000000..82b59874b3 --- /dev/null +++ b/tools/apprunner/gradle-plugin/src/main/java/org/apache/polaris/apprunner/plugin/PolarisRunnerExtension.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.apprunner.plugin; + +import org.gradle.api.Action; +import org.gradle.api.Project; +import org.gradle.api.Task; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.provider.ListProperty; +import org.gradle.api.provider.MapProperty; +import org.gradle.api.provider.Property; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.TaskCollection; +import org.gradle.api.tasks.TaskProvider; + +public class PolarisRunnerExtension { + private final MapProperty environment; + private final MapProperty environmentNonInput; + private final MapProperty systemProperties; + private final MapProperty systemPropertiesNonInput; + private final ListProperty arguments; + private final ListProperty argumentsNonInput; + private final ListProperty jvmArguments; + private final ListProperty jvmArgumentsNonInput; + private final Property javaVersion; + private final Property httpListenPortProperty; + private final Property httpListenUrlProperty; + private final Property managementListenPortProperty; + private final Property managementListenUrlProperty; + private final RegularFileProperty executableJar; + private final RegularFileProperty workingDirectory; + private final Property timeToListenUrlMillis; + private final Property timeToStopMillis; + + private final Provider polarisRunnerServiceProvider; + + public PolarisRunnerExtension( + Project project, Provider polarisRunnerServiceProvider) { + this.polarisRunnerServiceProvider = polarisRunnerServiceProvider; + + environment = project.getObjects().mapProperty(String.class, String.class); + environmentNonInput = project.getObjects().mapProperty(String.class, String.class); + systemProperties = project.getObjects().mapProperty(String.class, String.class); + systemPropertiesNonInput = project.getObjects().mapProperty(String.class, String.class); + arguments = project.getObjects().listProperty(String.class); + argumentsNonInput = project.getObjects().listProperty(String.class); + jvmArguments = project.getObjects().listProperty(String.class); + jvmArgumentsNonInput = project.getObjects().listProperty(String.class); + javaVersion = project.getObjects().property(Integer.class).convention(21); + httpListenUrlProperty = + project.getObjects().property(String.class).convention("quarkus.http.test-url"); + httpListenPortProperty = + project.getObjects().property(String.class).convention("quarkus.http.test-port"); + managementListenUrlProperty = + project.getObjects().property(String.class).convention("quarkus.management.test-url"); + managementListenPortProperty = + project.getObjects().property(String.class).convention("quarkus.management.test-port"); + workingDirectory = + project + .getObjects() + .fileProperty() + .convention(project.getLayout().getBuildDirectory().file("polaris-quarkus-server")); + executableJar = project.getObjects().fileProperty(); + timeToListenUrlMillis = project.getObjects().property(Long.class).convention(0L); + timeToStopMillis = project.getObjects().property(Long.class).convention(0L); + } + + /** System properties for the Polaris JVM. */ + public MapProperty getSystemProperties() { + return systemProperties; + } + + /** System properties for the Polaris JVM, not respected for Gradle build caching. */ + public MapProperty getSystemPropertiesNonInput() { + return systemPropertiesNonInput; + } + + /** Environment variables for the Polaris JVM. */ + public MapProperty getEnvironment() { + return environment; + } + + /** Environment variables for the Polaris JVM, not respected for Gradle build caching. */ + public MapProperty getEnvironmentNonInput() { + return environmentNonInput; + } + + /** Arguments used to start the Polaris JVM. */ + public ListProperty getArguments() { + return arguments; + } + + /** Arguments used to start the Polaris JVM, not respected for Gradle build caching. */ + public ListProperty getArgumentsNonInput() { + return argumentsNonInput; + } + + /** JVM arguments used to start the Polaris JVM. */ + public ListProperty getJvmArguments() { + return jvmArguments; + } + + /** JVM arguments used to start the Polaris JVM, not respected for Gradle build caching. */ + public ListProperty getJvmArgumentsNonInput() { + return jvmArgumentsNonInput; + } + + /** The Java version to use to run Polaris, defaults to 21. */ + public Property getJavaVersion() { + return javaVersion; + } + + /** The name of the property that will receive the HTTP port number. */ + public Property getHttpListenPortProperty() { + return httpListenPortProperty; + } + + /** + * The name of the property that will receive the HTTP listen URL, in the exact form as emitted by + * Quarkus, likely containing {@code 0.0.0.0} has the host. + */ + public Property getHttpListenUrlProperty() { + return httpListenUrlProperty; + } + + /** The name of the property that will receive the management port number. */ + public Property getManagementListenPortProperty() { + return managementListenPortProperty; + } + + /** + * The name of the property that will receive the management listen URL, in the exact form as + * emitted by Quarkus, likely containing {@code 0.0.0.0} has the host. + */ + public Property getManagementListenUrlProperty() { + return managementListenUrlProperty; + } + + /** The file of the executable jar to run Polaris. */ + public RegularFileProperty getExecutableJar() { + return executableJar; + } + + /** + * Working directory used when starting Polaris, defaults to {@code build/polaris-quarkus-server} + * in the current Gradle project. + */ + public RegularFileProperty getWorkingDirectory() { + return workingDirectory; + } + + /** + * Time to wait until the plugin expects Quarkus to emit the listen URLs, defaults to 30 seconds. + */ + public Property getTimeToListenUrlMillis() { + return timeToListenUrlMillis; + } + + /** + * Time to wait until Polaris has stopped after the termination signal, defaults to 15 seconds. + */ + public Property getTimeToStopMillis() { + return timeToStopMillis; + } + + /** + * The Gradle tasks of the current Gradle project to "decorate" with a running Polaris server, + * with the HTTP and management URL and port properties. + */ + public PolarisRunnerExtension includeTasks(TaskCollection taskCollection) { + return includeTasks(taskCollection, null); + } + + /** + * The Gradle tasks of the current Gradle project to "decorate" with a running Polaris server, + * with the HTTP and management URL and port properties. + */ + public PolarisRunnerExtension includeTasks( + TaskCollection taskCollection, Action postStartAction) { + taskCollection.configureEach( + new PolarisRunnerTaskConfigurer<>(postStartAction, polarisRunnerServiceProvider)); + return this; + } + + /** + * The Gradle tasks of the current Gradle project to "decorate" with a running Polaris server, + * with the HTTP and management URL and port properties. + */ + public PolarisRunnerExtension includeTask(TaskProvider taskProvider) { + return includeTask(taskProvider, null); + } + + /** + * The Gradle tasks of the current Gradle project to "decorate" with a running Polaris server, + * with the HTTP and management URL and port properties. + */ + public PolarisRunnerExtension includeTask( + TaskProvider taskProvider, Action postStartAction) { + taskProvider.configure( + new PolarisRunnerTaskConfigurer<>(postStartAction, polarisRunnerServiceProvider)); + return this; + } +} diff --git a/tools/apprunner/gradle-plugin/src/main/java/org/apache/polaris/apprunner/plugin/PolarisRunnerPlugin.java b/tools/apprunner/gradle-plugin/src/main/java/org/apache/polaris/apprunner/plugin/PolarisRunnerPlugin.java new file mode 100644 index 0000000000..a6d538e264 --- /dev/null +++ b/tools/apprunner/gradle-plugin/src/main/java/org/apache/polaris/apprunner/plugin/PolarisRunnerPlugin.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.apprunner.plugin; + +import java.util.concurrent.ThreadLocalRandom; +import org.gradle.api.Plugin; +import org.gradle.api.Project; + +public class PolarisRunnerPlugin implements Plugin { + + static final String EXTENSION_NAME = "polarisQuarkusApp"; + + /** + * The name of the Gradle configuration that contains the Quarkus server application as the only + * dependency. + */ + static final String APP_CONFIG_NAME = "polarisQuarkusServer"; + + @Override + public void apply(Project project) { + project + .getConfigurations() + .register( + APP_CONFIG_NAME, + c -> + c.setTransitive(false) + .setDescription( + "References the Polaris-Quarkus server dependency, only a single dependency allowed.")); + + var runnerService = + project + .getGradle() + .getSharedServices() + .registerIfAbsent( + // Make the build-service unique per project to prevent Gradle class-cast + // exceptions when the plugin's reloaded within the same build using different + // class loaders. + "polaris-quarkus-runner-" + ThreadLocalRandom.current().nextLong(), + PolarisRunnerService.class, + spec -> {}); + + project + .getExtensions() + .create(EXTENSION_NAME, PolarisRunnerExtension.class, project, runnerService); + } +} diff --git a/tools/apprunner/gradle-plugin/src/main/java/org/apache/polaris/apprunner/plugin/PolarisRunnerService.java b/tools/apprunner/gradle-plugin/src/main/java/org/apache/polaris/apprunner/plugin/PolarisRunnerService.java new file mode 100644 index 0000000000..1ec12cdbd9 --- /dev/null +++ b/tools/apprunner/gradle-plugin/src/main/java/org/apache/polaris/apprunner/plugin/PolarisRunnerService.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.apprunner.plugin; + +import java.util.IdentityHashMap; +import java.util.Map; +import org.gradle.api.Task; +import org.gradle.api.services.BuildService; +import org.gradle.api.services.BuildServiceParameters; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class PolarisRunnerService + implements BuildService, AutoCloseable { + private static final Logger LOGGER = LoggerFactory.getLogger(PolarisRunnerService.class); + + private final Map processes = new IdentityHashMap<>(); + + @Override + public void close() { + synchronized (processes) { + if (!processes.isEmpty()) { + LOGGER.warn("Cleaning up {} Polaris Quarkus servers", processes.size()); + } + for (var state : processes.values()) { + state.quarkusStop(LOGGER); + } + } + } + + public void register(ProcessState processState, Task task) { + synchronized (processes) { + processes.put(task, processState); + } + } + + public void finished(Task task) { + ProcessState state; + synchronized (processes) { + state = processes.remove(task); + } + if (state != null) { + state.quarkusStop(task.getLogger()); + } + } +} diff --git a/tools/apprunner/gradle-plugin/src/main/java/org/apache/polaris/apprunner/plugin/PolarisRunnerTaskConfigurer.java b/tools/apprunner/gradle-plugin/src/main/java/org/apache/polaris/apprunner/plugin/PolarisRunnerTaskConfigurer.java new file mode 100644 index 0000000000..69b61b08da --- /dev/null +++ b/tools/apprunner/gradle-plugin/src/main/java/org/apache/polaris/apprunner/plugin/PolarisRunnerTaskConfigurer.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.apprunner.plugin; + +import static org.apache.polaris.apprunner.plugin.PolarisRunnerPlugin.APP_CONFIG_NAME; + +import jakarta.annotation.Nonnull; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; +import org.gradle.api.Action; +import org.gradle.api.Task; +import org.gradle.api.file.RegularFile; +import org.gradle.api.plugins.ExtraPropertiesExtension; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.PathSensitivity; + +/** Configures the task for which the Polaris-Quarkus process shall be started. */ +public class PolarisRunnerTaskConfigurer implements Action { + + private final Action postStartAction; + private final Provider polarisRunnerServiceProvider; + + public PolarisRunnerTaskConfigurer( + Action postStartAction, Provider polarisRunnerServiceProvider) { + this.postStartAction = postStartAction; + this.polarisRunnerServiceProvider = polarisRunnerServiceProvider; + } + + @SuppressWarnings( + "Convert2Lambda") // Gradle complains when using lambdas (build-cache won't wonk) + @Override + public void execute(T task) { + var project = task.getProject(); + + var appConfig = project.getConfigurations().getByName(APP_CONFIG_NAME); + var extension = project.getExtensions().getByType(PolarisRunnerExtension.class); + + // Add the StartTask's properties as "inputs" to the Test task, so the Test task is + // executed, when those properties change. + var inputs = task.getInputs(); + inputs.properties(extension.getEnvironment().get()); + inputs.properties(extension.getSystemProperties().get()); + inputs.property("polaris.quarkus.arguments", extension.getArguments().get().toString()); + inputs.property("polaris.quarkus.jvmArguments", extension.getJvmArguments().get().toString()); + RegularFile execJar = extension.getExecutableJar().getOrNull(); + if (execJar != null) { + inputs.file(execJar).withPathSensitivity(PathSensitivity.RELATIVE); + } + inputs.property("polaris.quarkus.javaVersion", extension.getJavaVersion().get()); + + inputs.files(appConfig); + + var dependencies = appConfig.getDependencies(); + // Although we assert that only a single artifact is used (later), collect all dependencies + // for a nicer error message. + var dependenciesString = + dependencies.stream() + .map(d -> String.format("%s:%s:%s", d.getGroup(), d.getName(), d.getVersion())) + .collect(Collectors.joining(", ")); + var files = + !dependencies.isEmpty() + ? appConfig.getIncoming().artifactView(viewConfiguration -> {}).getFiles() + : null; + + var extra = task.getExtensions().findByType(ExtraPropertiesExtension.class); + BiConsumer httpUrlAndPortConsumer = + extra != null + ? (listenUrl, listenPort) -> { + extra.set(extension.getHttpListenUrlProperty().get(), listenUrl); + extra.set(extension.getHttpListenPortProperty().get(), listenPort); + } + : (listenUrl, listenPort) -> {}; + BiConsumer managementUrlAndPortConsumer = + extra != null + ? (listenUrl, listenPort) -> { + extra.set(extension.getManagementListenUrlProperty().get(), listenUrl); + extra.set(extension.getManagementListenPortProperty().get(), listenPort); + } + : (listenUrl, listenPort) -> {}; + + if (extra != null) { + task.notCompatibleWithConfigurationCache( + "PolarisRunner needs Gradle's extra-properties, which is incompatible with the configuration cache"); + } + + // Start the Polaris-Quarkus-App only when the Test task actually runs + + task.usesService(polarisRunnerServiceProvider); + task.doFirst( + new Action<>() { + @SuppressWarnings("unchecked") + @Override + public void execute(@Nonnull Task t) { + var processState = new ProcessState(); + polarisRunnerServiceProvider.get().register(processState, t); + + processState.quarkusStart( + t, + extension, + files, + dependenciesString, + httpUrlAndPortConsumer, + managementUrlAndPortConsumer); + if (postStartAction != null) { + postStartAction.execute((T) t); + } + } + }); + task.doLast( + new Action<>() { + @Override + public void execute(@Nonnull Task t) { + polarisRunnerServiceProvider.get().finished(t); + } + }); + } +} diff --git a/tools/apprunner/gradle-plugin/src/main/java/org/apache/polaris/apprunner/plugin/ProcessState.java b/tools/apprunner/gradle-plugin/src/main/java/org/apache/polaris/apprunner/plugin/ProcessState.java new file mode 100644 index 0000000000..854d5e2c65 --- /dev/null +++ b/tools/apprunner/gradle-plugin/src/main/java/org/apache/polaris/apprunner/plugin/ProcessState.java @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.apprunner.plugin; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeoutException; +import java.util.function.BiConsumer; +import org.apache.polaris.apprunner.common.JavaVM; +import org.apache.polaris.apprunner.common.ProcessHandler; +import org.gradle.api.GradleException; +import org.gradle.api.Task; +import org.gradle.api.file.FileCollection; +import org.gradle.api.file.RegularFile; +import org.gradle.api.tasks.TaskAction; +import org.gradle.process.JavaForkOptions; +import org.slf4j.Logger; + +public class ProcessState { + + private ProcessHandler processHandler; + + public ProcessState() { + // intentionally empty + } + + @TaskAction + public void noop() {} + + void quarkusStart( + Task task, + PolarisRunnerExtension extension, + FileCollection appConfigFiles, + String dependenciesString, + BiConsumer httpUrlAndPortConsumer, + BiConsumer managementUrlAndPortConsumer) { + + RegularFile configuredJar = extension.getExecutableJar().getOrNull(); + + File execJar; + + if (configuredJar == null) { + if (appConfigFiles != null && !appConfigFiles.isEmpty()) { + var appConfigFileSet = appConfigFiles.getFiles(); + if (appConfigFileSet.size() != 1) { + throw new GradleException( + String.format( + "Expected configuration %s to resolve to exactly one artifact, but resolves to %s (hint: do not enable transitive on the dependency)", + PolarisRunnerPlugin.APP_CONFIG_NAME, dependenciesString)); + } + execJar = appConfigFileSet.iterator().next(); + } else { + throw new GradleException( + String.format( + "Neither does the configuration %s contain exactly one dependency (preferably org.apache.polaris:polaris-quarkus-server:runner), nor is the runner jar specified in the %s extension.", + PolarisRunnerPlugin.APP_CONFIG_NAME, PolarisRunnerPlugin.EXTENSION_NAME)); + } + } else { + if (appConfigFiles != null && !appConfigFiles.isEmpty()) { + throw new GradleException( + String.format( + "Configuration %s contains a dependency and option 'executableJar' are mutually exclusive", + PolarisRunnerPlugin.APP_CONFIG_NAME)); + } + execJar = configuredJar.getAsFile(); + } + + var javaVM = JavaVM.findJavaVM(extension.getJavaVersion().get()); + if (javaVM == null) { + throw new GradleException(noJavaMessage(extension.getJavaVersion().get())); + } + + var workDir = extension.getWorkingDirectory().getAsFile().get().toPath(); + if (!Files.isDirectory(workDir)) { + try { + Files.createDirectories(workDir); + } catch (IOException e) { + throw new GradleException( + String.format("Failed to create working directory %s", workDir), e); + } + } + + var command = new ArrayList(); + command.add(javaVM.getJavaExecutable().toString()); + command.addAll(extension.getJvmArguments().get()); + command.addAll(extension.getJvmArgumentsNonInput().get()); + command.add("-Dquarkus.http.port=0"); + command.add("-Dquarkus.management.port=0"); + command.add("-Dquarkus.log.level=INFO"); + command.add("-Dquarkus.log.console.level=INFO"); + extension + .getSystemProperties() + .get() + .forEach((k, v) -> command.add(String.format("-D%s=%s", k, v))); + extension + .getSystemPropertiesNonInput() + .get() + .forEach((k, v) -> command.add(String.format("-D%s=%s", k, v))); + command.add("-jar"); + command.add(execJar.getAbsolutePath()); + command.addAll(extension.getArguments().get()); + command.addAll(extension.getArgumentsNonInput().get()); + + task.getLogger().info("Starting process: {}", command); + + var processBuilder = new ProcessBuilder().command(command); + extension.getEnvironment().get().forEach((k, v) -> processBuilder.environment().put(k, v)); + extension + .getEnvironmentNonInput() + .get() + .forEach((k, v) -> processBuilder.environment().put(k, v)); + processBuilder.directory(workDir.toFile()); + + var logger = task.getLogger(); + + try { + processHandler = new ProcessHandler(); + processHandler.setStdoutTarget(line -> logger.info("[output] {}", line)); + processHandler.start(processBuilder); + if (extension.getTimeToListenUrlMillis().get() > 0L) { + processHandler.setTimeToListenUrlMillis(extension.getTimeToListenUrlMillis().get()); + } + if (extension.getTimeToStopMillis().get() > 0L) { + processHandler.setTimeStopMillis(extension.getTimeToStopMillis().get()); + } + processHandler.getListenUrls(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new GradleException(String.format("Process-start interrupted: %s", command), e); + } catch (TimeoutException e) { + throw new GradleException( + String.format("Polaris-Server/Quarkus did not emit listen URL. Process: %s", command), e); + } catch (IOException e) { + throw new GradleException(String.format("Failed to start the process %s", command), e); + } + + List listenUrls; + try { + listenUrls = processHandler.getListenUrls(); + } catch (Exception e) { + // Can safely ignore it (it invocation does not block and therefore not throw an exception). + // But make the IDE happy with this throw. + throw new RuntimeException(e); + } + + var httpListenUrl = listenUrls.getFirst(); + String httpListenPort = Integer.toString(URI.create(httpListenUrl).getPort()); + // Add the Quarkus properties as "generic properties", so any task can use them. + httpUrlAndPortConsumer.accept(httpListenUrl, httpListenPort); + + List jvmOpts; + + var managementListenUrl = listenUrls.get(1); + if (managementListenUrl != null) { + var managementListenPort = Integer.toString(URI.create(managementListenUrl).getPort()); + managementUrlAndPortConsumer.accept(managementListenUrl, managementListenPort); + + jvmOpts = + Arrays.asList( + String.format("-D%s=%s", extension.getHttpListenUrlProperty().get(), httpListenUrl), + String.format("-D%s=%s", extension.getHttpListenPortProperty().get(), httpListenPort), + String.format( + "-D%s=%s", extension.getManagementListenUrlProperty().get(), managementListenUrl), + String.format( + "-D%s=%s", + extension.getManagementListenPortProperty().get(), managementListenPort)); + } else { + jvmOpts = + Arrays.asList( + String.format("-D%s=%s", extension.getHttpListenUrlProperty().get(), httpListenUrl), + String.format( + "-D%s=%s", extension.getHttpListenPortProperty().get(), httpListenPort)); + } + + // Do not put the "dynamic" properties (quarkus.http.test-port) to the `Test` task's + // system-properties, because those are subject to the test-task's inputs, which is used + // as the build-cache key. Instead, pass the dynamic properties via a + // CommandLineArgumentProvider. + // In other words: ensure that the `Test` tasks is cacheable. + if (task instanceof JavaForkOptions test) { + test.getJvmArgumentProviders().add(() -> jvmOpts); + } + } + + void quarkusStop(Logger logger) { + if (processHandler == null) { + logger.debug("No application found."); + return; + } + + try { + processHandler.stop(); + logger.info("Quarkus application stopped."); + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + processHandler = null; + } + } + + static String noJavaMessage(int version) { + return String.format( + "Could not find a Java-VM for Java version %d. " + + "Set the Java-Home for a compatible JVM using the environment variable JDK%d_HOME or " + + "JAVA%d_HOME.", + version, version, version); + } +} diff --git a/tools/apprunner/gradle-plugin/src/main/resources/META-INF/gradle-plugins/org.apache.polaris.apprunner b/tools/apprunner/gradle-plugin/src/main/resources/META-INF/gradle-plugins/org.apache.polaris.apprunner new file mode 100644 index 0000000000..85a30017b9 --- /dev/null +++ b/tools/apprunner/gradle-plugin/src/main/resources/META-INF/gradle-plugins/org.apache.polaris.apprunner @@ -0,0 +1 @@ +implementation=org.apache.polaris.apprunner.plugin.PolarisRunnerPlugin diff --git a/tools/apprunner/gradle-plugin/src/test/java/org/apache/polaris/apprunner/plugin/TestPolarisRunnerPlugin.java b/tools/apprunner/gradle-plugin/src/test/java/org/apache/polaris/apprunner/plugin/TestPolarisRunnerPlugin.java new file mode 100644 index 0000000000..44b226b1fe --- /dev/null +++ b/tools/apprunner/gradle-plugin/src/test/java/org/apache/polaris/apprunner/plugin/TestPolarisRunnerPlugin.java @@ -0,0 +1,326 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.apprunner.plugin; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.gradle.testkit.runner.BuildResult; +import org.gradle.testkit.runner.BuildTask; +import org.gradle.testkit.runner.GradleRunner; +import org.gradle.testkit.runner.TaskOutcome; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; + +/** + * Test for the {@link PolarisRunnerPlugin}, which basically simulates what the {@code build.gradle} + * in Apache Iceberg does. + */ +@ExtendWith(SoftAssertionsExtension.class) +class TestPolarisRunnerPlugin { + @InjectSoftAssertions SoftAssertions soft; + @TempDir Path testProjectDir; + + Path buildFile; + + String nessieVersionForTest; + + List prefix; + + @BeforeEach + void setup() throws Exception { + buildFile = testProjectDir.resolve("build.gradle"); + var localBuildCacheDirectory = testProjectDir.resolve(".local-cache"); + + // Copy our test class in the test's project test-source folder + var testTargetDir = testProjectDir.resolve("src/test/java/org/apache/polaris/apprunner/plugin"); + Files.createDirectories(testTargetDir); + Files.copy( + Paths.get( + "src/test/resources/org/apache/polaris/apprunner/plugin/TestSimulatingTestUsingThePlugin.java"), + testTargetDir.resolve("TestSimulatingTestUsingThePlugin.java")); + + Files.write( + testProjectDir.resolve("settings.gradle"), + Arrays.asList( + "buildCache {", + " local {", + " directory '" + localBuildCacheDirectory.toUri() + "'", + " }", + "}", + "", + "include 'sub'")); + + // Versions injected from build.gradle.kts - this is actually a Nessie version + nessieVersionForTest = System.getProperty("polaris-version-for-test", "0.101.3"); + var junitVersion = System.getProperty("junit-version"); + + soft.assertThat(junitVersion != null) + .withFailMessage( + "System property required for this test is missing, run this test via Gradle or set the system properties manually") + .isTrue(); + + prefix = + Arrays.asList( + "plugins {", + " id 'java'", + " id 'org.apache.polaris.apprunner'", + "}", + "", + "repositories {", + " mavenLocal()", + " mavenCentral()", + "}", + "", + "test {", + " useJUnitPlatform()", + "}", + "", + "dependencies {", + " testImplementation 'org.junit.jupiter:junit-jupiter-api:" + junitVersion + "'", + " testImplementation 'org.junit.jupiter:junit-jupiter-engine:" + junitVersion + "'", + " testImplementation 'org.projectnessie.nessie:nessie-client:" + + nessieVersionForTest + + "'"); + } + + /** + * Ensure that the plugin fails when there is no dependency specified for the {@code + * polarisQuarkusServer} configuration. + */ + @Test + void noAppConfigDeps() throws Exception { + Files.write( + buildFile, + Stream.concat( + prefix.stream(), + Stream.of( + "}", "", "polarisQuarkusApp {", " includeTask(tasks.named(\"test\"))", "}")) + .collect(Collectors.toList())); + + var result = createGradleRunner("test").buildAndFail(); + soft.assertThat(result.task(":test")) + .isNotNull() + .extracting(BuildTask::getOutcome) + .isEqualTo(TaskOutcome.FAILED); + soft.assertThat(Arrays.asList(result.getOutput().split("\n"))) + .contains( + "> Neither does the configuration polarisQuarkusServer contain exactly one dependency (preferably org.apache.polaris:polaris-quarkus-server:runner), nor is the runner jar specified in the polarisQuarkusApp extension."); + } + + /** + * Ensure that the plugin works with a declared dependency via the {@code polarisQuarkusServer} + * configuration. + */ + @Test + void withDependencyDeclaration() throws Exception { + Files.write( + buildFile, + Stream.concat( + prefix.stream(), + Stream.of( + " polarisQuarkusServer 'org.projectnessie.nessie:nessie-quarkus:" + + nessieVersionForTest + + ":runner'", + "}", + "", + "polarisQuarkusApp.includeTask(tasks.named(\"test\"))")) + .collect(Collectors.toList())); + + var result = createGradleRunner("test").build(); + soft.assertThat(result.task(":test")) + .isNotNull() + .extracting(BuildTask::getOutcome) + .isEqualTo(TaskOutcome.SUCCESS); + } + + /** + * Ensure that the plugin fails when there is more than one dependency specified for the {@code + * polarisQuarkusServer} configuration. + */ + @Test + void tooManyAppConfigDeps() throws Exception { + Files.write( + buildFile, + Stream.concat( + prefix.stream(), + Stream.of( + " polarisQuarkusServer 'org.projectnessie.nessie:nessie-quarkus:" + + nessieVersionForTest + + ":runner'", + " polarisQuarkusServer 'org.projectnessie.nessie:nessie-model:" + + nessieVersionForTest + + "'", + "}", + "", + "polarisQuarkusApp.includeTask(tasks.named(\"test\"))")) + .collect(Collectors.toList())); + + var result = createGradleRunner("test").buildAndFail(); + soft.assertThat(result.task(":test")) + .isNotNull() + .extracting(BuildTask::getOutcome) + .isEqualTo(TaskOutcome.FAILED); + soft.assertThat(Arrays.asList(result.getOutput().split("\n"))) + .contains( + "> Expected configuration polarisQuarkusServer to resolve to exactly one artifact, " + + "but resolves to org.projectnessie.nessie:nessie-quarkus:" + + nessieVersionForTest + + ", org.projectnessie.nessie:nessie-model:" + + nessieVersionForTest + + " (hint: do not enable transitive on the dependency)"); + } + + /** + * Ensure that the plugin fails when both the config-dependency and the exec-jar are specified. + */ + @Test + void configAndExecJar() throws Exception { + Files.write( + buildFile, + Stream.concat( + prefix.stream(), + Stream.of( + " polarisQuarkusServer 'org.projectnessie.nessie:nessie-quarkus:" + + nessieVersionForTest + + ":runner'", + "}", + "", + "polarisQuarkusApp {", + " executableJar.set(jar.archiveFile.get())", + " includeTask(tasks.named(\"test\"))", + "}")) + .collect(Collectors.toList())); + + soft.assertThat(createGradleRunner("jar").build().task(":jar")) + .extracting(BuildTask::getOutcome) + .isNotEqualTo(TaskOutcome.FAILED); + + var result = createGradleRunner("test").buildAndFail(); + soft.assertThat(result.task(":test")) + .isNotNull() + .extracting(BuildTask::getOutcome) + .isEqualTo(TaskOutcome.FAILED); + soft.assertThat(Arrays.asList(result.getOutput().split("\n"))) + .contains( + "> Configuration polarisQuarkusServer contains a dependency and option 'executableJar' are mutually exclusive"); + } + + /** Ensure that the plugin fails when it doesn't find a matching Java. */ + @Test + void unknownJdk() throws Exception { + Files.write( + buildFile, + Stream.concat( + prefix.stream(), + Stream.of( + " polarisQuarkusServer 'org.projectnessie.nessie:nessie-quarkus:" + + nessieVersionForTest + + ":runner'", + "}", + "", + "polarisQuarkusApp {", + " javaVersion.set(42)", + " includeTask(tasks.named(\"test\"))", + "}")) + .collect(Collectors.toList())); + + BuildResult result = createGradleRunner("test").buildAndFail(); + soft.assertThat(result.task(":test")) + .isNotNull() + .extracting(BuildTask::getOutcome) + .isEqualTo(TaskOutcome.FAILED); + soft.assertThat(Arrays.asList(result.getOutput().split("\n"))) + .contains("> " + ProcessState.noJavaMessage(42)); + } + + /** + * Starting the Polaris-Server via the Polaris-Quarkus-Gradle-Plugin must work fine, even if a + * different nessie-client version is being used (despite whether having conflicting versions + * makes any sense). + */ + @Test + void conflictingDependenciesNessie() throws Exception { + Files.write( + buildFile, + Stream.concat( + prefix.stream(), + Stream.of( + " implementation 'org.projectnessie.nessie:nessie-client:0.101.0'", + " polarisQuarkusServer 'org.projectnessie.nessie:nessie-quarkus:" + + nessieVersionForTest + + ":runner'", + "}", + "", + "polarisQuarkusApp {", + " includeTask(tasks.named(\"test\"))", + "}")) + .collect(Collectors.toList())); + + var result = createGradleRunner("test").build(); + soft.assertThat(result.task(":test")) + .isNotNull() + .extracting(BuildTask::getOutcome) + .isEqualTo(TaskOutcome.SUCCESS); + + soft.assertThat(Arrays.asList(result.getOutput().split("\n"))) + .anyMatch(l -> l.contains("Listening on: http://0.0.0.0:")) + .contains("Quarkus application stopped."); + + // 2nd run must be up-to-date + + result = createGradleRunner("test").build(); + soft.assertThat(result.task(":test")) + .isNotNull() + .extracting(BuildTask::getOutcome) + .isEqualTo(TaskOutcome.UP_TO_DATE); + + // 3rd run after a 'clean' must use the cached result + + result = createGradleRunner("clean").build(); + soft.assertThat(result.task(":clean")) + .isNotNull() + .extracting(BuildTask::getOutcome) + .isEqualTo(TaskOutcome.SUCCESS); + + result = createGradleRunner("test").build(); + soft.assertThat(result.task(":test")) + .isNotNull() + .extracting(BuildTask::getOutcome) + .isEqualTo(TaskOutcome.FROM_CACHE); + } + + private GradleRunner createGradleRunner(String task) { + return GradleRunner.create() + .withPluginClasspath() + .withProjectDir(testProjectDir.toFile()) + .withArguments("--no-configuration-cache", "--build-cache", "--info", "--stacktrace", task) + .withDebug(true) + .forwardOutput(); + } +} diff --git a/tools/apprunner/gradle-plugin/src/test/resources/org/apache/polaris/apprunner/plugin/TestSimulatingTestUsingThePlugin.java b/tools/apprunner/gradle-plugin/src/test/resources/org/apache/polaris/apprunner/plugin/TestSimulatingTestUsingThePlugin.java new file mode 100644 index 0000000000..2f09315308 --- /dev/null +++ b/tools/apprunner/gradle-plugin/src/test/resources/org/apache/polaris/apprunner/plugin/TestSimulatingTestUsingThePlugin.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.apprunner.plugin; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import org.junit.jupiter.api.Test; +import org.projectnessie.client.NessieClientBuilder; +import org.projectnessie.client.api.NessieApiV2; +import org.projectnessie.model.Branch; + +/** + * This is not a test for the plugin itself, this is a test that is run BY the test for the plugin. + */ +class TestSimulatingTestUsingThePlugin { + @Test + void pingNessie() throws Exception { + var port = System.getProperty("quarkus.http.test-port"); + assertNotNull(port); + var url = System.getProperty("quarkus.http.test-url"); + assertNotNull(url); + + var uri = String.format("http://127.0.0.1:%s/api/v2", port); + + var client = NessieClientBuilder.createClientBuilderFromSystemSettings().withUri(uri).build(NessieApiV2.class); + // Just some simple REST request to verify that Polaris is started - no fancy interactions w/ Nessie + var config = client.getConfig(); + + // We have seen that HTTP/POST requests can fail with conflicting dependencies + client.createReference().sourceRefName("main").reference(Branch.of("foo-" + System.nanoTime(), config.getNoAncestorHash())).create(); + } +} diff --git a/tools/apprunner/gradle.properties b/tools/apprunner/gradle.properties new file mode 120000 index 0000000000..03ca90c515 --- /dev/null +++ b/tools/apprunner/gradle.properties @@ -0,0 +1 @@ +../../gradle.properties \ No newline at end of file diff --git a/tools/apprunner/gradle/baselibs.versions.toml b/tools/apprunner/gradle/baselibs.versions.toml new file mode 120000 index 0000000000..4db0491d1e --- /dev/null +++ b/tools/apprunner/gradle/baselibs.versions.toml @@ -0,0 +1 @@ +../../../gradle/baselibs.versions.toml \ No newline at end of file diff --git a/tools/apprunner/gradle/libs.versions.toml b/tools/apprunner/gradle/libs.versions.toml new file mode 100644 index 0000000000..57692a9c60 --- /dev/null +++ b/tools/apprunner/gradle/libs.versions.toml @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +[versions] +soebes-itf = "0.13.1" + +[libraries] +assertj-core = { module = "org.assertj:assertj-core", version = "3.27.2" } +errorprone = { module = "com.google.errorprone:error_prone_core", version = "2.36.0" } +maven-core = { module = "org.apache.maven:maven-core", version = "3.9.9" } +maven-plugin-annotations = { module = "org.apache.maven.plugin-tools:maven-plugin-annotations", version = "3.15.1" } +jakarta-annotation-api = { module = "jakarta.annotation:jakarta.annotation-api", version = "3.0.0" } +junit-bom = { module = "org.junit:junit-bom", version = "5.11.4" } +soebes-itf-assertj = { module = "com.soebes.itf.jupiter.extension:itf-assertj", version.ref = "soebes-itf" } +soebes-itf-jupiter-extension = { module = "com.soebes.itf.jupiter.extension:itf-jupiter-extension", version.ref = "soebes-itf" } + +[plugins] +maven-plugin = { id = "org.gradlex.maven-plugin-development", version = "1.0.1" } diff --git a/tools/apprunner/gradlew b/tools/apprunner/gradlew new file mode 120000 index 0000000000..343e0d2caa --- /dev/null +++ b/tools/apprunner/gradlew @@ -0,0 +1 @@ +../../gradlew \ No newline at end of file diff --git a/tools/apprunner/maven-plugin/build.gradle.kts b/tools/apprunner/maven-plugin/build.gradle.kts new file mode 100644 index 0000000000..14a1fde514 --- /dev/null +++ b/tools/apprunner/maven-plugin/build.gradle.kts @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.tools.ant.filters.ReplaceTokens + +plugins { + id("polaris-apprunner-java") + alias(libs.plugins.maven.plugin) +} + +val deps by configurations.creating +val maven by configurations.creating + +configurations.implementation.get().extendsFrom(deps) + +dependencies { + deps(project(":polaris-apprunner-common")) + implementation(libs.maven.core) + compileOnly(libs.maven.plugin.annotations) + compileOnly(libs.jakarta.annotation.api) + testImplementation(libs.soebes.itf.jupiter.extension) + testImplementation(libs.soebes.itf.assertj) + maven( + group = "org.apache.maven", + name = "apache-maven", + version = libs.maven.core.get().version, + classifier = "bin", + ext = "tar.gz", + ) +} + +mavenPlugin { + helpMojoPackage.set("org.apache.polaris.apprunner.maven") + artifactId = project.name + groupId = project.group.toString() + dependencies = deps +} + +// The following stuff is needed by the Maven integration tests. +// +// The Maven plugin integration tests use `com.soebes.itf.jupiter.extension`, which is admittedly a +// bit "dusty", +// but it works. That extension however is built to be run inside a Maven build, but here it is +// Gradle, so we +// have to do some things manually to get that Maven-plugin-integration-test-framework working. + +// Does what it says, download and unpack a Maven distribution, needed by the IT-framework. +val getMavenDistro by + tasks.registering(Sync::class) { + from(tarTree(maven.singleFile)) { eachFile { path = path.substring(path.indexOf('/') + 1) } } + into(layout.buildDirectory.dir("maven")) + } + +// soebes-itf expects the artifacts of/for the Maven plugin to be tested in `target/itf-repo` in the +// layout of +// a local Maven repo. Sadly, Gradle offers no standard way to publish artifacts to a "custom" local +// Maven repo, +// so this task publishes the required artifacts to the user's local Maven repo and then copies the +// published +// artifacts to `target/itf-repo`. +val itfRepo by + tasks.registering(Sync::class) { + // polaris-apprunner parent pom + dependsOn(":publishToMavenLocal") + // polaris-apprunner-common pom + jar + dependsOn(":polaris-apprunner-common:publishToMavenLocal") + // polaris-apprunner-maven-plugin pom + jar + dependsOn("publishToMavenLocal") + + // Poor-man's way to convert the group-ID to a path + val groupPath = project.group.toString().replace(".", "/") + // Note: this assumes the user has his local Maven repository in $HOME/.m2/repository. This does + // NOT work for + // any other location, whether it's configured using Maven properties or a settings.xml. If such + // a support is + // required, please add support for that and open a PR. + val localMavenRepo = "${System.getProperty("user.home")}/.m2/repository" + from(localMavenRepo) + include("$groupPath/**") + into(layout.projectDirectory.dir("target/itf-repo")) + } + +// Copy the Maven projects used by the integration-tests, while replacing the necessary placeholders +// for GAV and +// dependencies used by those tests. +val syncResourcesIts by + tasks.registering(Sync::class) { + from("src/test/resources-its") + into(project.layout.projectDirectory.dir("target/test-classes")) + filter( + ReplaceTokens::class, + mapOf( + "tokens" to + mapOf( + "projectGroupId" to project.group.toString(), + "projectArtifactId" to project.name, + "projectVersion" to project.version, + "junitVersion" to libs.junit.bom.get().version, + ) + ), + ) + } + +tasks.named("test") { + dependsOn(syncResourcesIts, itfRepo, getMavenDistro) + jvmArgumentProviders.add( + CommandLineArgumentProvider { + listOf("-Dmaven.home=${getMavenDistro.get().outputs.files.singleFile}") + } + ) + environment( + mapOf("JAVA_HOME" to this.javaLauncher.get().metadata.installationPath.asFile.toString()) + ) +} diff --git a/tools/apprunner/maven-plugin/pom.xml b/tools/apprunner/maven-plugin/pom.xml new file mode 100644 index 0000000000..774520a83b --- /dev/null +++ b/tools/apprunner/maven-plugin/pom.xml @@ -0,0 +1,32 @@ + + + + 4.0.0 + + org.apache.polaris.apprunner + polaris-apprunner-maven + 0.42-SNAPSHOT + + + UTF-8 + + diff --git a/tools/apprunner/maven-plugin/src/main/java/org/apache/polaris/apprunner/maven/AbstractPolarisRunnerMojo.java b/tools/apprunner/maven-plugin/src/main/java/org/apache/polaris/apprunner/maven/AbstractPolarisRunnerMojo.java new file mode 100644 index 0000000000..66ea151085 --- /dev/null +++ b/tools/apprunner/maven-plugin/src/main/java/org/apache/polaris/apprunner/maven/AbstractPolarisRunnerMojo.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.apprunner.maven; + +import org.apache.maven.execution.MavenSession; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.project.MavenProject; +import org.apache.polaris.apprunner.common.ProcessHandler; + +/* + * Base class to share configuration between mojo. + */ +abstract class AbstractPolarisRunnerMojo extends AbstractMojo { + private static final String CONTEXT_KEY = "polaris.quarkus.app"; + + /** Maven project. */ + @Parameter(defaultValue = "${project}", readonly = true, required = true) + private MavenProject project; + + /** Maven session. */ + @Parameter(defaultValue = "${session}", readonly = true, required = true) + private MavenSession session; + + /** Whether execution should be skipped. */ + @Parameter(property = "polaris.apprunner.skip", required = false, defaultValue = "false") + private boolean skip; + + /** Execution id for the app. */ + @Parameter(property = "polaris.apprunner.executionId", required = false, defaultValue = "default") + private String executionId; + + public boolean isSkipped() { + return skip; + } + + public String getExecutionId() { + return executionId; + } + + public MavenProject getProject() { + return project; + } + + public MavenSession getSession() { + return session; + } + + private String getContextKey() { + final String key = CONTEXT_KEY + '.' + getExecutionId(); + return key; + } + + protected ProcessHandler getApplication() { + final String key = getContextKey(); + return (ProcessHandler) project.getContextValue(key); + } + + protected void resetApplication() { + final String key = getContextKey(); + project.setContextValue(key, null); + } + + protected void setApplicationHandle(ProcessHandler application) { + final String key = getContextKey(); + final Object previous = project.getContextValue(key); + if (previous != null) { + getLog() + .warn( + String.format( + "Found a previous ProcessHandler for execution id %s.", getExecutionId())); + } + project.setContextValue(key, application); + } +} diff --git a/tools/apprunner/maven-plugin/src/main/java/org/apache/polaris/apprunner/maven/PolarisRunnerStartMojo.java b/tools/apprunner/maven-plugin/src/main/java/org/apache/polaris/apprunner/maven/PolarisRunnerStartMojo.java new file mode 100644 index 0000000000..dc9255cbc2 --- /dev/null +++ b/tools/apprunner/maven-plugin/src/main/java/org/apache/polaris/apprunner/maven/PolarisRunnerStartMojo.java @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.apprunner.maven; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Properties; +import java.util.stream.Collectors; +import javax.inject.Inject; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.descriptor.PluginDescriptor; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.plugins.annotations.ResolutionScope; +import org.apache.maven.toolchain.ToolchainManager; +import org.apache.polaris.apprunner.common.JavaVM; +import org.apache.polaris.apprunner.common.ProcessHandler; +import org.eclipse.aether.RepositorySystem; +import org.eclipse.aether.RepositorySystemSession; +import org.eclipse.aether.artifact.Artifact; +import org.eclipse.aether.artifact.DefaultArtifact; +import org.eclipse.aether.repository.RemoteRepository; +import org.eclipse.aether.resolution.ArtifactRequest; +import org.eclipse.aether.resolution.ArtifactResolutionException; +import org.eclipse.aether.resolution.ArtifactResult; + +/** Starting Quarkus application. */ +@Mojo(name = "start", requiresDependencyResolution = ResolutionScope.NONE, threadSafe = true) +public class PolarisRunnerStartMojo extends AbstractPolarisRunnerMojo { + + /** The entry point to Aether, i.e. the component doing all the work. */ + @Inject private RepositorySystem repoSystem; + + @Inject private ToolchainManager toolchainManager; + + /** The current repository/network configuration of Maven. */ + @Parameter(defaultValue = "${repositorySystemSession}", readonly = true) + private RepositorySystemSession repoSession; + + /** + * The project's remote repositories to use for the resolution of plugins and their dependencies. + */ + @Parameter(defaultValue = "${project.remotePluginRepositories}", readonly = true) + private List remoteRepos; + + /** The plugin descriptor. */ + @Parameter(defaultValue = "${plugin}", readonly = true) + @SuppressWarnings("unused") + private PluginDescriptor pluginDescriptor; + + /** + * The application artifact id. + * + *

Needs to be present as a plugin dependency, if "executableJar" is not set. + * + *

Mutually exclusive with "executableJar". + * + *

Supported format is groupId:artifactId[:type[:classifier]]:version + */ + @Parameter(property = "polaris.apprunner.appArtifactId") + private String appArtifactId; + + /** Environment variable configuration properties. */ + @Parameter private Properties systemProperties = new Properties(); + + /** + * Properties to get from Quarkus running application. + * + *

The property key is the name of the build property to set, the value is the name of the + * quarkus configuration key to get. + */ + @Parameter private Properties environment; + + @Parameter private List arguments; + + @Parameter private List jvmArguments; + + @Parameter(defaultValue = "21") + private int javaVersion; + + /** + * The path to the executable jar to run. + * + *

Mutually exclusive with "appArtifactId" + */ + @Parameter private String executableJar; + + @Parameter(defaultValue = "quarkus.http.test-port") + private String httpListenPortProperty; + + @Parameter(defaultValue = "quarkus.http.test-url") + private String httpListenUrlProperty; + + @Parameter(defaultValue = "quarkus.management.test-port") + private String managementListenPortProperty; + + @Parameter(defaultValue = "quarkus.management.test-url") + private String managementListenUrlProperty; + + @Parameter(defaultValue = "${build.directory}/polaris-quarkus") + private String workingDirectory; + + @Parameter private long timeToListenUrlMillis; + + @Parameter private long timeToStopMillis; + + static String noJavaVMMessage(int version) { + return String.format( + "Could not find a Java-VM for Java version %d. " + + "Either configure a type=jdk in Maven's toolchain with version=%d or " + + "set the Java-Home for a compatible JVM using the environment variable JDK%d_HOME or " + + "JAVA%d_HOME.", + version, version, version, version); + } + + @Override + public void execute() throws MojoExecutionException { + if (isSkipped()) { + getLog().debug("Execution is skipped"); + return; + } + + getLog().debug(String.format("Searching for Java %d ...", javaVersion)); + String javaExecutable = + toolchainManager + .getToolchains( + getSession(), + "jdk", + Collections.singletonMap("version", Integer.toString(javaVersion))) + .stream() + .map(tc -> tc.findTool("java")) + .filter(Objects::nonNull) + .findFirst() + .orElseGet( + () -> { + getLog() + .debug( + String.format( + "... using JavaVM as Maven toolkit returned no toolchain " + + "for type==jdk and version==%d", + javaVersion)); + JavaVM javaVM = JavaVM.findJavaVM(javaVersion); + return javaVM != null ? javaVM.getJavaExecutable().toString() : null; + }); + if (javaExecutable == null) { + throw new MojoExecutionException(noJavaVMMessage(javaVersion)); + } + getLog().debug(String.format("Using javaExecutable %s", javaExecutable)); + + Path workDir = Paths.get(workingDirectory); + if (!Files.isDirectory(workDir)) { + try { + Files.createDirectories(workDir); + } catch (IOException e) { + throw new MojoExecutionException( + String.format("Failed to create working directory %s", workingDirectory), e); + } + } + + String execJar = executableJar; + if (execJar == null && appArtifactId == null) { + throw new MojoExecutionException( + "Either appArtifactId or executableJar config option must be specified, prefer appArtifactId"); + } + if (execJar == null) { + Artifact artifact = new DefaultArtifact(appArtifactId); + ArtifactRequest artifactRequest = new ArtifactRequest(artifact, remoteRepos, null); + try { + ArtifactResult result = repoSystem.resolveArtifact(repoSession, artifactRequest); + execJar = result.getArtifact().getFile().toString(); + } catch (ArtifactResolutionException e) { + throw new MojoExecutionException( + String.format("Failed to resolve artifact %s", appArtifactId), e); + } + } else if (appArtifactId != null) { + throw new MojoExecutionException( + "The options appArtifactId and executableJar are mutually exclusive"); + } + + List command = new ArrayList<>(); + command.add(javaExecutable); + if (jvmArguments != null) { + command.addAll(jvmArguments); + } + if (systemProperties != null) { + systemProperties.forEach( + (k, v) -> command.add(String.format("-D%s=%s", k.toString(), v.toString()))); + } + command.add("-Dquarkus.http.port=0"); + command.add("-Dquarkus.management.port=0"); + command.add("-Dquarkus.log.level=INFO"); + command.add("-Dquarkus.log.console.level=INFO"); + command.add("-jar"); + command.add(execJar); + if (arguments != null) { + command.addAll(arguments); + } + + getLog() + .info( + String.format( + "Starting process: %s, additional env: %s", + String.join(" ", command), + environment != null + ? environment.entrySet().stream() + .map(e -> String.format("%s=%s", e.getKey(), e.getValue())) + .collect(Collectors.joining(", ")) + : "")); + + ProcessBuilder processBuilder = new ProcessBuilder().command(command); + if (environment != null) { + environment.forEach((k, v) -> processBuilder.environment().put(k.toString(), v.toString())); + } + processBuilder.directory(workDir.toFile()); + + try { + ProcessHandler processHandler = new ProcessHandler(); + if (timeToListenUrlMillis > 0L) { + processHandler.setTimeToListenUrlMillis(timeToListenUrlMillis); + } + if (timeToStopMillis > 0L) { + processHandler.setTimeStopMillis(timeToStopMillis); + } + processHandler.setStdoutTarget(line -> getLog().info(String.format("[output] %s", line))); + processHandler.start(processBuilder); + + setApplicationHandle(processHandler); + + List listenUrls = processHandler.getListenUrls(); + + Properties projectProperties = getProject().getProperties(); + projectProperties.setProperty(httpListenUrlProperty, listenUrls.get(0)); + projectProperties.setProperty( + httpListenPortProperty, Integer.toString(URI.create(listenUrls.get(0)).getPort())); + if (listenUrls.get(1) != null) { + projectProperties.setProperty(managementListenUrlProperty, listenUrls.get(1)); + projectProperties.setProperty( + managementListenPortProperty, + Integer.toString(URI.create(listenUrls.get(1)).getPort())); + } + + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new MojoExecutionException(String.format("Process-start interrupted: %s", command), e); + } catch (Exception e) { + throw new MojoExecutionException(String.format("Failed to start the process %s", command), e); + } + } +} diff --git a/tools/apprunner/maven-plugin/src/main/java/org/apache/polaris/apprunner/maven/PolarisRunnerStopMojo.java b/tools/apprunner/maven-plugin/src/main/java/org/apache/polaris/apprunner/maven/PolarisRunnerStopMojo.java new file mode 100644 index 0000000000..a90d4c3f7f --- /dev/null +++ b/tools/apprunner/maven-plugin/src/main/java/org/apache/polaris/apprunner/maven/PolarisRunnerStopMojo.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.apprunner.maven; + +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.ResolutionScope; +import org.apache.polaris.apprunner.common.ProcessHandler; + +/** Stop Quarkus application. */ +@Mojo(name = "stop", requiresDependencyResolution = ResolutionScope.NONE, threadSafe = true) +public class PolarisRunnerStopMojo extends AbstractPolarisRunnerMojo { + /** Mojo execution. */ + @Override + public void execute() throws MojoExecutionException { + if (isSkipped()) { + getLog().info("Stopping Quarkus application."); + return; + } + + ProcessHandler application = getApplication(); + if (application == null) { + getLog().warn(String.format("No application found for execution id '%s'.", getExecutionId())); + return; + } + + try { + application.stop(); + getLog().info("Quarkus application stopped."); + } catch (Exception e) { + throw new MojoExecutionException("Error while stopping Quarkus application", e); + } finally { + resetApplication(); + } + } +} diff --git a/tools/apprunner/maven-plugin/src/test/java/org/apache/polaris/apprunner/maven/ITPolarisMavenPlugin.java b/tools/apprunner/maven-plugin/src/test/java/org/apache/polaris/apprunner/maven/ITPolarisMavenPlugin.java new file mode 100644 index 0000000000..ad4a805726 --- /dev/null +++ b/tools/apprunner/maven-plugin/src/test/java/org/apache/polaris/apprunner/maven/ITPolarisMavenPlugin.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.apprunner.maven; + +import static com.soebes.itf.extension.assertj.MavenITAssertions.assertThat; + +import com.soebes.itf.jupiter.extension.MavenCLIOptions; +import com.soebes.itf.jupiter.extension.MavenGoal; +import com.soebes.itf.jupiter.extension.MavenJupiterExtension; +import com.soebes.itf.jupiter.extension.MavenOption; +import com.soebes.itf.jupiter.extension.MavenRepository; +import com.soebes.itf.jupiter.extension.MavenTest; +import com.soebes.itf.jupiter.maven.MavenExecutionResult; + +@MavenJupiterExtension +@MavenRepository +class ITPolarisMavenPlugin { + + @MavenTest + @MavenGoal("verify") + @MavenOption(MavenCLIOptions.ERRORS) + void executionJarAndApplicationIdMissing(MavenExecutionResult result) { + assertThat(result) + .isFailure() + .out() + .error() + .anyMatch( + s -> + s.contains( + "Either appArtifactId or executableJar config option must be specified, prefer appArtifactId")); + } + + @MavenTest + @MavenGoal("verify") + @MavenOption(MavenCLIOptions.ERRORS) + void executionJarAndApplicationIdSpecified(MavenExecutionResult result) { + assertThat(result) + .isFailure() + .out() + .error() + .anyMatch( + s -> s.contains("The options appArtifactId and executableJar are mutually exclusive")); + } + + @MavenTest + @MavenGoal("verify") + @MavenOption(MavenCLIOptions.ERRORS) + void applicationIdSpecified(MavenExecutionResult result) { + assertThat(result) + .isSuccessful() + .out() + .info() + .anyMatch( + s -> + s.matches( + "Starting process: .*-jar .*/nessie-quarkus-.*-runner.jar, additional env: .*HELLO=world.*")) + .anyMatch(s -> s.matches("Starting process: .*java.* -Dfoo=bar .*")) + .anyMatch(s -> s.matches("Starting process: .*java.* -Dquarkus.http.port=0 .*")) + .anyMatch(s -> s.matches("Quarkus application stopped.")); + } + + @MavenTest + @MavenGoal("verify") + @MavenOption(MavenCLIOptions.ERRORS) + @MavenOption(value = MavenCLIOptions.TOOLCHAINS, parameter = "non-java-toolchains.xml") + void unknownJdk(MavenExecutionResult result) { + assertThat(result) + .isFailure() + .out() + .error() + .anyMatch(s -> s.contains(PolarisRunnerStartMojo.noJavaVMMessage(42))); + } +} diff --git a/tools/apprunner/maven-plugin/src/test/resources-its/org/apache/polaris/apprunner/maven/ITPolarisMavenPlugin/applicationIdSpecified/pom.xml b/tools/apprunner/maven-plugin/src/test/resources-its/org/apache/polaris/apprunner/maven/ITPolarisMavenPlugin/applicationIdSpecified/pom.xml new file mode 100644 index 0000000000..a1a5e82698 --- /dev/null +++ b/tools/apprunner/maven-plugin/src/test/resources-its/org/apache/polaris/apprunner/maven/ITPolarisMavenPlugin/applicationIdSpecified/pom.xml @@ -0,0 +1,108 @@ + + + + 4.0.0 + + org.apache.polaris.maven.it-test + polaris-apprunner-maven-it-applicationIdSpecified + 0.42-SNAPSHOT + + + 8 + 0.49.0 + UTF-8 + + + + + org.junit.jupiter + junit-jupiter-api + @junitVersion@ + test + + + org.junit.jupiter + junit-jupiter-engine + @junitVersion@ + test + + + org.projectnessie + nessie-client + ${nessie.version-for-test} + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 21 + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.3.1 + + + org.apache.maven.plugins + maven-failsafe-plugin + 3.3.1 + + + @projectGroupId@ + @projectArtifactId@ + @projectVersion@ + + org.projectnessie:nessie-quarkus:jar:runner:${nessie.version-for-test} + + bar + + + world + + + + + start + pre-integration-test + + start + + + + stop + post-integration-test + + stop + + + + + + + diff --git a/tools/apprunner/maven-plugin/src/test/resources-its/org/apache/polaris/apprunner/maven/ITPolarisMavenPlugin/applicationIdSpecified/src/test/java/org/apache/polaris/appruner/maven/mavenit/ITSimulatingTestUsingThePlugin.java b/tools/apprunner/maven-plugin/src/test/resources-its/org/apache/polaris/apprunner/maven/ITPolarisMavenPlugin/applicationIdSpecified/src/test/java/org/apache/polaris/appruner/maven/mavenit/ITSimulatingTestUsingThePlugin.java new file mode 100644 index 0000000000..f54afc263f --- /dev/null +++ b/tools/apprunner/maven-plugin/src/test/resources-its/org/apache/polaris/apprunner/maven/ITPolarisMavenPlugin/applicationIdSpecified/src/test/java/org/apache/polaris/appruner/maven/mavenit/ITSimulatingTestUsingThePlugin.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.apprunner.maven.mavenit; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import org.junit.jupiter.api.Test; +import org.projectnessie.client.api.NessieApiV1; +import org.projectnessie.client.http.HttpClientBuilder; +import org.projectnessie.model.Branch; + +/** + * This is not a test for the plugin itself, this is a test that is run BY the test for the plugin. + */ +class ITSimulatingTestUsingThePlugin { + @Test + void pingNessie() throws Exception { + String port = System.getProperty("quarkus.http.test-port"); + assertNotNull(port, "quarkus.http.test-port"); + String url = System.getProperty("quarkus.http.test-url"); + assertNotNull(url, "quarkus.http.test-url"); + + String uri = String.format("http://127.0.0.1:%s/api/v1", port); + + NessieApiV1 client = HttpClientBuilder.builder().withUri(uri).build(NessieApiV1.class); + // Just some simple REST request to verify that Nessie is started - no fancy interactions w/ Nessie + client.getConfig(); + + // We have seen that HTTP/POST requests can fail with conflicting dependencies + client.createReference().sourceRefName("main").reference(Branch.of("foo-" + System.nanoTime(), null)).create(); + } +} diff --git a/tools/apprunner/maven-plugin/src/test/resources-its/org/apache/polaris/apprunner/maven/ITPolarisMavenPlugin/executionJarAndApplicationIdMissing/pom.xml b/tools/apprunner/maven-plugin/src/test/resources-its/org/apache/polaris/apprunner/maven/ITPolarisMavenPlugin/executionJarAndApplicationIdMissing/pom.xml new file mode 100644 index 0000000000..8b31ce1908 --- /dev/null +++ b/tools/apprunner/maven-plugin/src/test/resources-its/org/apache/polaris/apprunner/maven/ITPolarisMavenPlugin/executionJarAndApplicationIdMissing/pom.xml @@ -0,0 +1,76 @@ + + + + 4.0.0 + + org.apache.polaris.maven.it-test + polaris-apprunner-maven-it-executionJarAndApplicationIdMissing + 0.42-SNAPSHOT + + + UTF-8 + + + + + org.junit.jupiter + junit-jupiter-api + @junitVersion@ + test + + + org.junit.jupiter + junit-jupiter-engine + @junitVersion@ + test + + + + + + + @projectGroupId@ + @projectArtifactId@ + @projectVersion@ + + + + + + start + pre-integration-test + + start + + + + stop + post-integration-test + + stop + + + + + + + diff --git a/tools/apprunner/maven-plugin/src/test/resources-its/org/apache/polaris/apprunner/maven/ITPolarisMavenPlugin/executionJarAndApplicationIdSpecified/pom.xml b/tools/apprunner/maven-plugin/src/test/resources-its/org/apache/polaris/apprunner/maven/ITPolarisMavenPlugin/executionJarAndApplicationIdSpecified/pom.xml new file mode 100644 index 0000000000..8f753564ce --- /dev/null +++ b/tools/apprunner/maven-plugin/src/test/resources-its/org/apache/polaris/apprunner/maven/ITPolarisMavenPlugin/executionJarAndApplicationIdSpecified/pom.xml @@ -0,0 +1,77 @@ + + + + 4.0.0 + + org.apache.polaris.maven.it-test + polaris-apprunner-maven-it-executionJarAndApplicationIdSpecified + 0.42-SNAPSHOT + + + UTF-8 + + + + + org.junit.jupiter + junit-jupiter-api + @junitVersion@ + test + + + org.junit.jupiter + junit-jupiter-engine + @junitVersion@ + test + + + + + + + @projectGroupId@ + @projectArtifactId@ + @projectVersion@ + + org.projectnessie:nessie-quarkus:jar:runner:0.101.3 + foo/bar/test.jar + + + + start + pre-integration-test + + start + + + + stop + post-integration-test + + stop + + + + + + + diff --git a/tools/apprunner/maven-plugin/src/test/resources-its/org/apache/polaris/apprunner/maven/ITPolarisMavenPlugin/unknownJdk/non-java-toolchains.xml b/tools/apprunner/maven-plugin/src/test/resources-its/org/apache/polaris/apprunner/maven/ITPolarisMavenPlugin/unknownJdk/non-java-toolchains.xml new file mode 100644 index 0000000000..3e3791912d --- /dev/null +++ b/tools/apprunner/maven-plugin/src/test/resources-its/org/apache/polaris/apprunner/maven/ITPolarisMavenPlugin/unknownJdk/non-java-toolchains.xml @@ -0,0 +1,33 @@ + + + + + + jdk + + 42 + test + + + /tmp + + + diff --git a/tools/apprunner/maven-plugin/src/test/resources-its/org/apache/polaris/apprunner/maven/ITPolarisMavenPlugin/unknownJdk/pom.xml b/tools/apprunner/maven-plugin/src/test/resources-its/org/apache/polaris/apprunner/maven/ITPolarisMavenPlugin/unknownJdk/pom.xml new file mode 100644 index 0000000000..d75b21fabf --- /dev/null +++ b/tools/apprunner/maven-plugin/src/test/resources-its/org/apache/polaris/apprunner/maven/ITPolarisMavenPlugin/unknownJdk/pom.xml @@ -0,0 +1,77 @@ + + + + 4.0.0 + + org.apache.polaris.maven.it-test + polaris-apprunner-maven-it-unknownJdk + 0.42-SNAPSHOT + + + UTF-8 + + + + + org.junit.jupiter + junit-jupiter-api + @junitVersion@ + test + + + org.junit.jupiter + junit-jupiter-engine + @junitVersion@ + test + + + + + + + @projectGroupId@ + @projectArtifactId@ + @projectVersion@ + + org.projectnessie:nessie-quarkus:jar:runner:0.21.2 + 42 + + + + start + pre-integration-test + + start + + + + stop + post-integration-test + + stop + + + + + + + diff --git a/tools/apprunner/settings.gradle.kts b/tools/apprunner/settings.gradle.kts new file mode 100644 index 0000000000..bb26d632e3 --- /dev/null +++ b/tools/apprunner/settings.gradle.kts @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +includeBuild("apprunner-build-logic") { name = "polaris-apprunner-build-logic" } + +if (!JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_21)) { + throw GradleException( + """ + + Build aborted... + + The Apache Polaris build requires Java 21. + Detected Java version: ${JavaVersion.current()} + + """ + ) +} + +rootProject.name = "polaris-apprunner" + +val baseVersion = file("../../version.txt").readText().trim() + +fun addProject(name: String) { + var fullName = "polaris-apprunner-$name" + include(fullName) + val prj = project(":$fullName") + prj.projectDir = file(name) +} + +listOf("common", "gradle-plugin", "maven-plugin").forEach { addProject(it) } + +pluginManagement { + repositories { + mavenCentral() // prefer Maven Central, in case Gradle's repo has issues + gradlePluginPortal() + } +} + +dependencyResolutionManagement { + repositoriesMode = RepositoriesMode.FAIL_ON_PROJECT_REPOS + repositories { + mavenCentral() + gradlePluginPortal() + } +} + +gradle.beforeProject { + version = baseVersion + group = "org.apache.polaris.apprunner" +}