Skip to content

Commit

Permalink
Adding configuration cache compatibility for QuarkusGenerateCode, Qua…
Browse files Browse the repository at this point in the history
…rkusBuildDependencies, QuarkusBuildCacheableAppParts, QuarkusBuild and QuarkusShowEffectiveConfig
  • Loading branch information
cdsap committed Mar 8, 2024
1 parent b40997f commit 6469240
Show file tree
Hide file tree
Showing 14 changed files with 657 additions and 407 deletions.
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
package io.quarkus.gradle;

import static io.quarkus.gradle.tasks.QuarkusGradleUtils.getSourceSet;

import java.io.File;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -34,16 +38,21 @@
import org.gradle.tooling.provider.model.ToolingModelBuilderRegistry;
import org.gradle.util.GradleVersion;

import io.quarkus.bootstrap.model.ApplicationModel;
import io.quarkus.bootstrap.resolver.AppModelResolverException;
import io.quarkus.deployment.pkg.PackageConfig;
import io.quarkus.gradle.dependency.ApplicationDeploymentClasspathBuilder;
import io.quarkus.gradle.extension.QuarkusPluginExtension;
import io.quarkus.gradle.extension.SourceSetExtension;
import io.quarkus.gradle.tasks.BaseConfig;
import io.quarkus.gradle.tasks.Deploy;
import io.quarkus.gradle.tasks.ImageBuild;
import io.quarkus.gradle.tasks.ImagePush;
import io.quarkus.gradle.tasks.QuarkusAddExtension;
import io.quarkus.gradle.tasks.QuarkusBuild;
import io.quarkus.gradle.tasks.QuarkusBuildCacheableAppParts;
import io.quarkus.gradle.tasks.QuarkusBuildDependencies;
import io.quarkus.gradle.tasks.QuarkusBuildTaskWithConfigurationCache;
import io.quarkus.gradle.tasks.QuarkusDev;
import io.quarkus.gradle.tasks.QuarkusGenerateCode;
import io.quarkus.gradle.tasks.QuarkusGoOffline;
Expand All @@ -64,7 +73,10 @@
import io.quarkus.gradle.tooling.dependency.DependencyUtils;
import io.quarkus.gradle.tooling.dependency.ExtensionDependency;
import io.quarkus.gradle.tooling.dependency.ProjectExtensionDependency;
import io.quarkus.maven.dependency.GACTV;
import io.quarkus.runtime.LaunchMode;
import io.smallrye.config.Expressions;
import io.smallrye.config.SmallRyeConfig;

public class QuarkusPlugin implements Plugin<Project> {

Expand Down Expand Up @@ -167,37 +179,45 @@ private void registerTasks(Project project, QuarkusPluginExtension quarkusExt) {
// quarkusGenerateCode
TaskProvider<QuarkusGenerateCode> quarkusGenerateCode = tasks.register(QUARKUS_GENERATE_CODE_TASK_NAME,
QuarkusGenerateCode.class, LaunchMode.NORMAL, SourceSet.MAIN_SOURCE_SET_NAME);
quarkusGenerateCode.configure(task -> configureGenerateCodeTask(task, QuarkusGenerateCode.QUARKUS_GENERATED_SOURCES));
quarkusGenerateCode.configure(task -> configureGenerateCodeTask(task, QuarkusGenerateCode.QUARKUS_GENERATED_SOURCES,
project, quarkusExt, LaunchMode.NORMAL));
// quarkusGenerateCodeDev
TaskProvider<QuarkusGenerateCode> quarkusGenerateCodeDev = tasks.register(QUARKUS_GENERATE_CODE_DEV_TASK_NAME,
QuarkusGenerateCode.class, LaunchMode.DEVELOPMENT, SourceSet.MAIN_SOURCE_SET_NAME);
quarkusGenerateCodeDev.configure(task -> {
task.dependsOn(quarkusGenerateCode);
configureGenerateCodeTask(task, QuarkusGenerateCode.QUARKUS_GENERATED_SOURCES);
configureGenerateCodeTask(task, QuarkusGenerateCode.QUARKUS_GENERATED_SOURCES, project, quarkusExt,
LaunchMode.DEVELOPMENT);
});
// quarkusGenerateCodeTests
TaskProvider<QuarkusGenerateCode> quarkusGenerateCodeTests = tasks.register(QUARKUS_GENERATE_CODE_TESTS_TASK_NAME,
QuarkusGenerateCode.class, LaunchMode.TEST, SourceSet.TEST_SOURCE_SET_NAME);
quarkusGenerateCodeTests.configure(task -> {
task.dependsOn("compileQuarkusTestGeneratedSourcesJava");
configureGenerateCodeTask(task, QuarkusGenerateCode.QUARKUS_TEST_GENERATED_SOURCES);
configureGenerateCodeTask(task, QuarkusGenerateCode.QUARKUS_TEST_GENERATED_SOURCES, project, quarkusExt,
LaunchMode.TEST);
});

tasks.register(QUARKUS_SHOW_EFFECTIVE_CONFIG_TASK_NAME,
QuarkusShowEffectiveConfig.class, task -> {
task.setDescription("Show effective Quarkus build configuration.");
configureBuildTask(project, quarkusExt, task);
});

TaskProvider<QuarkusBuildDependencies> quarkusBuildDependencies = tasks.register(QUARKUS_BUILD_DEP_TASK_NAME,
QuarkusBuildDependencies.class,
task -> task.getOutputs().doNotCacheIf("Dependencies are never cached", t -> true));
task -> {
task.getOutputs().doNotCacheIf("Dependencies are never cached", t -> true);
configureBuildTask(project, quarkusExt, task);
});

Property<Boolean> cacheLargeArtifacts = quarkusExt.getCacheLargeArtifacts();

TaskProvider<QuarkusBuildCacheableAppParts> quarkusBuildCacheableAppParts = tasks.register(
QUARKUS_BUILD_APP_PARTS_TASK_NAME,
QuarkusBuildCacheableAppParts.class, task -> {
task.dependsOn(quarkusGenerateCode);
configureBuildTask(project, quarkusExt, task);
task.getOutputs().doNotCacheIf(
"Not adding uber-jars, native binaries and mutable-jar package type to Gradle " +
"build cache by default. To allow caching of uber-jars, native binaries and mutable-jar " +
Expand All @@ -213,6 +233,7 @@ public boolean isSatisfiedBy(Task t) {

TaskProvider<QuarkusBuild> quarkusBuild = tasks.register(QUARKUS_BUILD_TASK_NAME, QuarkusBuild.class, build -> {
build.dependsOn(quarkusBuildDependencies, quarkusBuildCacheableAppParts);
configureBuildTask(project, quarkusExt, build);
build.getOutputs().doNotCacheIf(
"Only collects and combines the outputs of " + QUARKUS_BUILD_APP_PARTS_TASK_NAME + " and "
+ QUARKUS_BUILD_DEP_TASK_NAME + ", see 'cacheLargeArtifacts' in the 'quarkus' Gradle extension " +
Expand Down Expand Up @@ -414,14 +435,70 @@ public void execute(Task task) {
});
}

private static void configureGenerateCodeTask(QuarkusGenerateCode task, String generateSourcesDir) {
private static void configureGenerateCodeTask(QuarkusGenerateCode task, String generateSourcesDir, Project project,
QuarkusPluginExtension quarkusExtension,
LaunchMode launchMode) {
ApplicationModel applicationModel = quarkusExtension.getApplicationModel(launchMode);

SourceSet generatedSources = QuarkusGradleUtils.getSourceSet(task.getProject(), generateSourcesDir);
Set<File> sourceSetOutput = generatedSources.getOutput().filter(f -> f.getName().equals(generateSourcesDir)).getFiles();
if (sourceSetOutput.isEmpty()) {
throw new GradleException("Failed to configure " + task.getPath() + ": sourceSet " + generateSourcesDir
+ " has no output");
}
task.getGeneratedOutputDirectory().set(generatedSources.getJava().getClassesDirectory().get().getAsFile());
task.getAppModel().set(applicationModel);
task.getFinalName().set(quarkusExtension.finalName());
task.getEffectiveConfigurationValues()
.set(quarkusExtension.buildEffectiveConfiguration(applicationModel.getAppArtifact()).getValues());
task.getGradleVersion().set(project.getGradle().getGradleVersion());
task.getCachingRelevantInput()
.set(quarkusExtension.baseConfig()
.cachingRelevantProperties(quarkusExtension.getCachingRelevantProperties().get()));
}

private static void configureBuildTask(Project project, QuarkusPluginExtension quarkusExt,
QuarkusBuildTaskWithConfigurationCache task) {
BaseConfig baseConfig = quarkusExt.baseConfig();
PackageConfig packageConfig = baseConfig.packageConfig();
ApplicationModel appModel;

try {
appModel = quarkusExt.getAppModelResolver().resolveModel(new GACTV(project.getGroup().toString(), project.getName(),
project.getVersion().toString()));
} catch (AppModelResolverException e) {
throw new GradleException("Failed to resolve Quarkus application model for " + task.getPath(), e);
}

SmallRyeConfig smallRyeConfig = quarkusExt.buildEffectiveConfiguration(appModel.getAppArtifact()).getConfig();
Map<String, String> smallRyeConfigValues = smallRyeConfig.getValues("quarkus", String.class, String.class);
Map<String, String> quarkusProperties = Expressions.withoutExpansion(() -> {
Map<String, String> values = new HashMap<>();
smallRyeConfig.getValues("quarkus", String.class, String.class)
.forEach((key, value) -> values.put("quarkus." + key, value));
return values;
});

task.getResolveAppModelForBuild().set(appModel);
task.getRunnerSuffix().set(packageConfig.getRunnerSuffix());
task.getQuarkusProperties().set(quarkusProperties);
task.getOutputDirectory().set(packageConfig.outputDirectory.orElse(QuarkusPlugin.DEFAULT_OUTPUT_DIRECTORY));
task.getFinalName().set(quarkusExt.finalName());
task.getRunnerName().set(packageConfig.outputName.orElseGet(() -> quarkusExt.finalName()));
task.getCachingRelevantInput()
.set(baseConfig.cachingRelevantProperties(quarkusExt.getCachingRelevantProperties().get()));
task.getBuildSystemProperties().set(quarkusExt.buildSystemProperties(appModel.getAppArtifact(), quarkusProperties));
task.getForcedProperties().set(quarkusExt.forcedPropertiesProperty());

SourceSet mainSourceSet = getSourceSet(project, SourceSet.MAIN_SOURCE_SET_NAME);
task.setCompileClasspath(mainSourceSet.getCompileClasspath().plus(mainSourceSet.getRuntimeClasspath())
.plus(mainSourceSet.getAnnotationProcessorPath())
.plus(mainSourceSet.getResources()));

task.getSmallRyeConfig().set(smallRyeConfig);
task.getSmallRyeConfigValues().set(smallRyeConfigValues);
task.packageType().set(baseConfig.packageType());
task.getGradleVersion().set(project.getGradle().getGradleVersion());
}

private void createSourceSets(Project project) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,28 +78,28 @@ private BaseConfig buildBaseConfig() {
return new BaseConfig(effectiveConfig);
}

protected BaseConfig baseConfig() {
public BaseConfig baseConfig() {
this.baseConfig.finalizeValue();
return this.baseConfig.get();
}

protected MapProperty<String, String> forcedPropertiesProperty() {
public MapProperty<String, String> forcedPropertiesProperty() {
return forcedPropertiesProperty;
}

protected ListProperty<String> ignoredEntriesProperty() {
return ignoredEntries;
}

protected FileCollection classpath() {
public FileCollection classpath() {
return classpath;
}

protected Manifest manifest() {
return baseConfig().manifest();
}

protected EffectiveConfig buildEffectiveConfiguration(ResolvedDependency appArtifact) {
public EffectiveConfig buildEffectiveConfiguration(ResolvedDependency appArtifact) {
Map<String, Object> properties = new HashMap<>();

exportCustomManifestProperties(properties);
Expand Down Expand Up @@ -144,7 +144,7 @@ private EffectiveConfig buildEffectiveConfiguration(Map<String, Object> properti
* @param appArtifact the application dependency to retrive the quarkus application name and version.
* @return a filtered view of the configuration only with <code>quarkus.</code> names.
*/
protected Map<String, String> buildSystemProperties(ResolvedDependency appArtifact, Map<String, String> quarkusProperties) {
public Map<String, String> buildSystemProperties(ResolvedDependency appArtifact, Map<String, String> quarkusProperties) {
Map<String, String> buildSystemProperties = new HashMap<>();
buildSystemProperties.putIfAbsent("quarkus.application.name", appArtifact.getArtifactId());
buildSystemProperties.putIfAbsent("quarkus.application.version", appArtifact.getVersion());
Expand Down Expand Up @@ -214,7 +214,7 @@ private String quarkusProfile() {
return profile;
}

private static FileCollection dependencyClasspath(SourceSet mainSourceSet) {
public static FileCollection dependencyClasspath(SourceSet mainSourceSet) {
return mainSourceSet.getCompileClasspath().plus(mainSourceSet.getRuntimeClasspath())
.plus(mainSourceSet.getAnnotationProcessorPath())
.plus(mainSourceSet.getResources());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
* Configuration from system properties, environment, application.properties/yaml/yml, project properties is
* available in a Gradle task's configuration phase.
*/
final class BaseConfig {
public final class BaseConfig {
private final Manifest manifest;
private final PackageConfig packageConfig;
private final Map<String, String> values;
Expand All @@ -37,19 +37,19 @@ final class BaseConfig {
values = config.getValues();
}

PackageConfig packageConfig() {
public PackageConfig packageConfig() {
return packageConfig;
}

PackageConfig.BuiltInType packageType() {
public PackageConfig.BuiltInType packageType() {
return PackageConfig.BuiltInType.fromString(packageConfig.type);
}

Manifest manifest() {
return manifest;
}

Map<String, String> cachingRelevantProperties(List<String> propertyPatterns) {
public Map<String, String> cachingRelevantProperties(List<String> propertyPatterns) {
List<Pattern> patterns = propertyPatterns.stream().map(s -> "^(" + s + ")$").map(Pattern::compile)
.collect(Collectors.toList());
Predicate<Map.Entry<String, ?>> keyPredicate = e -> patterns.stream().anyMatch(p -> p.matcher(e.getKey()).matches());
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
package io.quarkus.gradle.tasks;

import java.io.File;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;

import javax.inject.Inject;

import org.gradle.api.Action;
import org.gradle.api.DefaultTask;
import org.gradle.process.JavaForkOptions;
import org.gradle.workers.ProcessWorkerSpec;
import org.gradle.workers.WorkQueue;
import org.gradle.workers.WorkerExecutor;

import io.quarkus.utilities.OS;

public abstract class DefaultQuarkusTask extends DefaultTask {
private static final List<String> WORKER_BUILD_FORK_OPTIONS = List.of("quarkus.");

protected final File projectDir;
protected final File buildDir;

DefaultQuarkusTask(String description) {
setDescription(description);
setGroup("quarkus");
this.projectDir = getProject().getProjectDir();
this.buildDir = getProject().getBuildDir();
}

@Inject
protected abstract WorkerExecutor getWorkerExecutor();

WorkQueue workQueue(Map<String, String> configMap, Supplier<List<Action<? super JavaForkOptions>>> forkOptionsActions) {
WorkerExecutor workerExecutor = getWorkerExecutor();

// Use process isolation by default, unless Gradle's started with its debugging system property or the
// system property `quarkus.gradle-worker.no-process is set to `true`.
if (Boolean.getBoolean("org.gradle.debug") || Boolean.getBoolean("quarkus.gradle-worker.no-process")) {
return workerExecutor.classLoaderIsolation();
}

return workerExecutor.processIsolation(processWorkerSpec -> configureProcessWorkerSpec(processWorkerSpec,
configMap, forkOptionsActions.get()));
}

private void configureProcessWorkerSpec(ProcessWorkerSpec processWorkerSpec, Map<String, String> configMap,
List<Action<? super JavaForkOptions>> customizations) {
JavaForkOptions forkOptions = processWorkerSpec.getForkOptions();
customizations.forEach(a -> a.execute(forkOptions));

// Propagate user.dir to load config sources that use it (instead of the worker user.dir)
String userDir = configMap.get("user.dir");
if (userDir != null) {
forkOptions.systemProperty("user.dir", userDir);
}

String quarkusWorkerMaxHeap = System.getProperty("quarkus.gradle-worker.max-heap");
if (quarkusWorkerMaxHeap != null && forkOptions.getAllJvmArgs().stream().noneMatch(arg -> arg.startsWith("-Xmx"))) {
forkOptions.jvmArgs("-Xmx" + quarkusWorkerMaxHeap);
}

// Pass all environment variables
forkOptions.environment(System.getenv());

if (OS.determineOS() == OS.WINDOWS) {
// On Windows, gRPC code generation is sometimes(?) unable to find "java.exe". Feels (not proven) that
// the grpc code generation tool looks up "java.exe" instead of consulting the 'JAVA_HOME' environment.
// Might be, that Gradle's process isolation "loses" some information down to the worker process, so add
// both JAVA_HOME and updated PATH environment from the 'java' executable chosen by Gradle (could be from
// a different toolchain than the one running the build, in theory at least).
// Linux is fine though, so no need to add a hack for Linux.
String java = forkOptions.getExecutable();
Path javaBinPath = Paths.get(java).getParent().toAbsolutePath();
String javaBin = javaBinPath.toString();
String javaHome = javaBinPath.getParent().toString();
forkOptions.environment("JAVA_HOME", javaHome);
forkOptions.environment("PATH", javaBin + File.pathSeparator + System.getenv("PATH"));
}

// It's kind of a "very big hammer" here, but this way we ensure that all necessary properties
// "quarkus.*" from all configuration sources
// are (forcefully) used in the Quarkus build - even properties defined on the QuarkusPluginExtension.
// This prevents that settings from e.g. a application.properties takes precedence over an explicit
// setting in Gradle project properties, the Quarkus extension or even via the environment or system
// properties.
// see https://github.com/quarkusio/quarkus/issues/33321 why not all properties are passed as system properties
// Note that we MUST NOT mess with the system properties of the JVM running the build! And that is the
// main reason why build and code generation happen in a separate process.
configMap.entrySet().stream()
.filter(e -> WORKER_BUILD_FORK_OPTIONS.stream().anyMatch(e.getKey().toLowerCase()::startsWith))
.forEach(e -> forkOptions.systemProperty(e.getKey(), e.getValue()));

// populate worker classpath with additional content?
// or maybe remove some dependencies from the plugin and make those exclusively available to the worker?
// processWorkerSpec.getClasspath().from();
}
}
Loading

0 comments on commit 6469240

Please sign in to comment.