diff --git a/bundles/com.salesforce.bazel.eclipse.core/build.properties b/bundles/com.salesforce.bazel.eclipse.core/build.properties index b0dbe001..24689006 100644 --- a/bundles/com.salesforce.bazel.eclipse.core/build.properties +++ b/bundles/com.salesforce.bazel.eclipse.core/build.properties @@ -8,4 +8,5 @@ output.. = bin/ additional.bundles = com.github.ben-manes.caffeine,\ com.google.gson,\ org.apache.commons.text,\ - org.osgi.service.event + org.osgi.service.event,\ + com.google.errorprone.annotations diff --git a/bundles/com.salesforce.bazel.eclipse.core/plugin.xml b/bundles/com.salesforce.bazel.eclipse.core/plugin.xml index ebaad167..656b4097 100644 --- a/bundles/com.salesforce.bazel.eclipse.core/plugin.xml +++ b/bundles/com.salesforce.bazel.eclipse.core/plugin.xml @@ -120,6 +120,10 @@ class="com.salesforce.bazel.eclipse.core.model.discovery.ProjectPerPackageProvisioningStrategy" name="project-per-package"> + + diff --git a/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/BazelProject.java b/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/BazelProject.java index 5b4950e8..5e4acea0 100644 --- a/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/BazelProject.java +++ b/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/BazelProject.java @@ -30,6 +30,7 @@ import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.stream.Stream; import org.eclipse.core.resources.ICommand; import org.eclipse.core.resources.IFile; @@ -44,6 +45,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.idea.blaze.base.model.primitives.TargetName; import com.salesforce.bazel.eclipse.core.BazelCore; import com.salesforce.bazel.eclipse.core.projectview.BazelProjectView; import com.salesforce.bazel.sdk.model.BazelLabel; @@ -592,24 +594,12 @@ boolean removeBazelBuilder(IProject project, IProgressMonitor monitor) throws Co return true; } - public void setBazelTargets(List targets, IProgressMonitor monitor) throws CoreException { - // the list can become extremely huge - // thus, instead of storing it as a persistent property on the project - // we put it into the .bazeltargets file - - List lines = new ArrayList<>(); - lines.add("# targets used to setup the project"); - lines.add("# (do not modify manually; synchronize projects to update)"); - targets.stream().map(BazelTarget::getTargetName).distinct().sorted().forEach(lines::add); + public void setBazelTargetNames(List targetNames, IProgressMonitor monitor) throws CoreException { + writeBazeltargetsFile(targetNames.stream().map(TargetName::toString), monitor); + } - var content = lines.stream().collect(joining(lineSeparator())).getBytes(UTF_8); - var bazelTargetsFile = getProject().getFile(FILE_NAME_DOT_BAZELTARGETS); - if (!bazelTargetsFile.exists()) { - bazelTargetsFile.create(new ByteArrayInputStream(content), IResource.FORCE | IResource.DERIVED, monitor); - } else { - bazelTargetsFile - .setContents(new ByteArrayInputStream(content), IResource.FORCE | IResource.KEEP_HISTORY, monitor); - } + public void setBazelTargets(List targets, IProgressMonitor monitor) throws CoreException { + writeBazeltargetsFile(targets.stream().map(BazelTarget::getTargetName), monitor); } public void setModel(BazelModel bazelModel) { @@ -637,4 +627,24 @@ public String toString() { result.append("]"); return result.toString(); } + + private void writeBazeltargetsFile(Stream targetNames, IProgressMonitor monitor) throws CoreException { + // the list can become extremely huge + // thus, instead of storing it as a persistent property on the project + // we put it into the .bazeltargets file + + List lines = new ArrayList<>(); + lines.add("# targets used to setup the project"); + lines.add("# (do not modify manually; synchronize projects to update)"); + targetNames.distinct().sorted().forEach(lines::add); + + var content = lines.stream().collect(joining(lineSeparator())).getBytes(UTF_8); + var bazelTargetsFile = getProject().getFile(FILE_NAME_DOT_BAZELTARGETS); + if (!bazelTargetsFile.exists()) { + bazelTargetsFile.create(new ByteArrayInputStream(content), IResource.FORCE | IResource.DERIVED, monitor); + } else { + bazelTargetsFile + .setContents(new ByteArrayInputStream(content), IResource.FORCE | IResource.KEEP_HISTORY, monitor); + } + } } diff --git a/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/BazelWorkspace.java b/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/BazelWorkspace.java index d6fb9495..57601dc8 100644 --- a/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/BazelWorkspace.java +++ b/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/BazelWorkspace.java @@ -244,7 +244,7 @@ public BazelModuleFile getBazelModuleFile() throws CoreException { */ public BazelPackage getBazelPackage(BazelLabel label) { checkIsRootedAtThisWorkspace(label); - return getBazelPackage(new org.eclipse.core.runtime.Path(label.getPackagePath())); + return getBazelPackage(IPath.forPosix(label.getPackagePath())); } /** diff --git a/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/BaseProvisioningStrategy.java b/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/BaseProvisioningStrategy.java index 45753365..b5f5edf5 100644 --- a/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/BaseProvisioningStrategy.java +++ b/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/BaseProvisioningStrategy.java @@ -1124,8 +1124,8 @@ def format(target): } /** - * Called by {@link #provisionProjectsForSelectedTargets(Collection, BazelWorkspace, IProgressMonitor)} after base - * workspace information has been detected. + * Called by {@link #doProvisionProjects(Collection, BazelWorkspace, TracingSubMonitor)} after packages were opened + * and {@link BazelTarget targets} looked up. *

* Implementors are expected to map all of the targets into projects. *

@@ -1140,6 +1140,56 @@ def format(target): protected abstract List doProvisionProjects(Collection targets, TracingSubMonitor monitor) throws CoreException; + /** + * Called by {@link #provisionProjectsForSelectedTargets(Collection, BazelWorkspace, IProgressMonitor)} after base + * workspace information has been detected. + *

+ * The default implementation obtains and opens {@link BazelPackage} and {@link BazelTarget} objects. + *

+ * + * @param targetsOrPackages + * collection of {@link TargetExpression target or wildcard expressions} + * @param workspace + * the Bazel workspace + * @param monitor + * monitor for reporting progress + * @return list of provisioned projects + * @throws CoreException + */ + protected List doProvisionProjects(Collection targetsOrPackages, + BazelWorkspace workspace, TracingSubMonitor monitor) throws CoreException { + // open all packages at once + monitor.subTask("Loading packages"); + var bazelPackages = targetsOrPackages.parallelStream().map(e -> { + if (e instanceof Label l) { + return new BazelLabel(l.toString()); + } + var w = WildcardTargetPattern.stripWildcardSuffix(e.toString()); + if (w != null) { + return new BazelLabel(w); + } + return null; + }).filter(Predicate.not(Objects::isNull)).map(workspace::getBazelPackage).distinct().toList(); + workspace.open(bazelPackages); + + // collect targets + monitor.subTask("Collecting targets"); + List targets = new ArrayList<>(); + for (TargetExpression targetExpression : targetsOrPackages) { + if (targetExpression instanceof Label l) { + // we don't check for no-ide tag here because we assume this was done already when discovering targets + targets.add(workspace.getBazelTarget(new BazelLabel(l.toString()))); + } else { + LOG.warn( + "Ignoring target expression '{}' for provisioning because this is not supported by the current implementation.", + targetExpression); + } + } + + // create projects + return doProvisionProjects(targets, monitor); + } + private void ensureFolderLinksToTarget(IFolder folderWhichShouldBeALink, IPath linkTarget, SubMonitor monitor) throws CoreException { if (folderWhichShouldBeALink.exists() && !folderWhichShouldBeALink.isLinked()) { @@ -1674,34 +1724,6 @@ public List provisionProjectsForSelectedTargets(Collection { - if (e instanceof Label l) { - return new BazelLabel(l.toString()); - } - var w = WildcardTargetPattern.stripWildcardSuffix(e.toString()); - if (w != null) { - return new BazelLabel(w); - } - return null; - }).filter(Predicate.not(Objects::isNull)).map(workspace::getBazelPackage).distinct().toList(); - workspace.open(bazelPackages); - - // collect targets - monitor.subTask("Collecting targets"); - List targets = new ArrayList<>(); - for (TargetExpression targetExpression : targetsOrPackages) { - if (targetExpression instanceof Label l) { - // we don't check for no-ide tag here because we assume this was done already when discovering targets - targets.add(workspace.getBazelTarget(new BazelLabel(l.toString()))); - } else { - LOG.warn( - "Ignoring target expression '{}' for provisioning because this is not supported by the current implementation.", - targetExpression); - } - } - // ensure there is a mapper fileSystemMapper = new BazelProjectFileSystemMapper(workspace); @@ -1715,8 +1737,7 @@ public List provisionProjectsForSelectedTargets(Collection + * This is recommended with the {@link BuildfileDrivenProvisioningStrategy}. + *

+ */ +public class BazelBuildfileTargetDiscovery extends BazelQueryTargetDiscovery implements TargetDiscoveryStrategy { + + public static final String STRATEGY_NAME = "buildfiles"; + + @Override + public Collection discoverTargets(BazelWorkspace bazelWorkspace, + Collection bazelPackages, IProgressMonitor progress) throws CoreException { + return bazelPackages.stream().map(TargetExpression::allFromPackageNonRecursive).toList(); + } + +} diff --git a/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/BuildFileAndVisibilityDrivenProvisioningStrategy.java b/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/BuildFileAndVisibilityDrivenProvisioningStrategy.java index bebc82ce..71d2e261 100644 --- a/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/BuildFileAndVisibilityDrivenProvisioningStrategy.java +++ b/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/BuildFileAndVisibilityDrivenProvisioningStrategy.java @@ -5,7 +5,6 @@ import static java.lang.String.format; import static java.nio.file.Files.isRegularFile; import static java.util.function.Predicate.not; -import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.toCollection; import static org.eclipse.core.resources.IResource.DEPTH_ZERO; @@ -18,7 +17,6 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import org.eclipse.core.runtime.CoreException; @@ -42,12 +40,8 @@ import com.salesforce.bazel.eclipse.core.model.BazelProject; import com.salesforce.bazel.eclipse.core.model.BazelTarget; import com.salesforce.bazel.eclipse.core.model.BazelWorkspace; -import com.salesforce.bazel.eclipse.core.model.buildfile.FunctionCall; import com.salesforce.bazel.eclipse.core.model.discovery.classpath.ClasspathEntry; import com.salesforce.bazel.eclipse.core.model.discovery.classpath.libs.ExternalLibrariesDiscovery; -import com.salesforce.bazel.eclipse.core.model.discovery.projects.JavaProjectInfo; -import com.salesforce.bazel.eclipse.core.model.discovery.projects.JavaSourceEntry; -import com.salesforce.bazel.eclipse.core.util.trace.TracingSubMonitor; import com.salesforce.bazel.sdk.command.BazelQueryForLabelsCommand; import com.salesforce.bazel.sdk.model.BazelLabel; @@ -69,7 +63,7 @@ * *

*/ -public class BuildFileAndVisibilityDrivenProvisioningStrategy extends ProjectPerPackageProvisioningStrategy { +public class BuildFileAndVisibilityDrivenProvisioningStrategy extends BuildfileDrivenProvisioningStrategy { public static class CircularDependenciesHelper { @@ -191,9 +185,6 @@ public boolean isAllowedDependencyPath(BazelPackage fromPackage, BazelPackage to private static Logger LOG = LoggerFactory.getLogger(BuildFileAndVisibilityDrivenProvisioningStrategy.class); - private final TargetDiscoveryAndProvisioningExtensionLookup extensionLookup = - new TargetDiscoveryAndProvisioningExtensionLookup(); - @Override public Map computeClasspaths(Collection bazelProjects, BazelWorkspace workspace, BazelClasspathScope scope, IProgressMonitor progress) throws CoreException { @@ -367,7 +358,8 @@ public Map computeClasspaths(Collectio } } - classpathsByProject.put(bazelProject, new CompileAndRuntimeClasspath(classpath, Collections.emptyList())); + classpathsByProject + .put(bazelProject, new CompileAndRuntimeClasspath(classpath, Collections.emptyList())); monitor.worked(1); } @@ -379,120 +371,4 @@ public Map computeClasspaths(Collectio } } - @Override - protected List doProvisionProjects(Collection targets, TracingSubMonitor monitor) - throws CoreException { - // group into packages - Map> targetsByPackage = - targets.stream().collect(groupingBy(BazelTarget::getBazelPackage)); - - monitor.beginTask("Provisioning projects", targetsByPackage.size() * 3); - - var result = new ArrayList(); - for (Entry> entry : targetsByPackage.entrySet()) { - var bazelPackage = entry.getKey(); - var packageTargets = entry.getValue(); - - // skip the root package (not supported) - if (bazelPackage.isRoot()) { - createBuildPathProblem( - bazelPackage.getBazelWorkspace().getBazelProject(), - Status.warning( - format( - "The root package was skipped during sync because it's not supported by the '%s' strategy. Consider excluding it in the .bazelproject file.", - STRATEGY_NAME))); - continue; - } - - // get the top-level macro calls - var topLevelMacroCalls = bazelPackage.getBazelBuildFile().getTopLevelCalls(); - - // build the project information as we traverse the macros - var javaInfo = new JavaProjectInfo(bazelPackage); - var relevantTargets = new ArrayList(); - for (FunctionCall macroCall : topLevelMacroCalls) { - var relevant = processMacroCall(macroCall, javaInfo); - if (!relevant) { - if (LOG.isDebugEnabled()) { - LOG.debug("Skipping not relevant macro call '{}'.", macroCall); - } - continue; - } - var name = macroCall.getName(); - if (name != null) { - packageTargets.stream().filter(t -> t.getTargetName().equals(name)).forEach(t -> { - if (LOG.isDebugEnabled()) { - LOG.debug("Found relevant target '{}' for macro call '{}'", t, macroCall); - } - relevantTargets.add(t); - }); - } - } - - // create the project for the package - var project = provisionPackageProject(bazelPackage, packageTargets, monitor.slice(1)); - - // create markers - analyzeProjectInfo(project, javaInfo, monitor); - - // sanity check - var sourceInfo = javaInfo.getSourceInfo(); - if (sourceInfo.hasSourceFilesWithoutCommonRoot()) { - for (JavaSourceEntry file : sourceInfo.getSourceFilesWithoutCommonRoot()) { - createBuildPathProblem( - project, - Status.warning( - format( - "File '%s' could not be mapped into a common source directory. The project may not build successful in Eclipse.", - file.getPath()))); - } - } - if (!sourceInfo.hasSourceDirectories()) { - createBuildPathProblem( - project, - Status.error( - format( - "No source directories detected when analyzing package '%s' using targets '%s'", - bazelPackage.getLabel().getPackagePath(), - packageTargets.stream() - .map(BazelTarget::getLabel) - .map(BazelLabel::getLabelPath) - .collect(joining(", "))))); - } - - // configure links - linkGeneratedSourcesIntoProject(project, javaInfo, monitor.slice(1)); - - // configure classpath - configureRawClasspath(project, javaInfo, monitor.slice(1)); - - result.add(project); - } - return result; - } - - private boolean processMacroCall(FunctionCall macroCall, JavaProjectInfo javaInfo) throws CoreException { - var analyzers = extensionLookup.createMacroCallAnalyzers(macroCall.getResolvedFunctionName()); - if (analyzers.isEmpty()) { - if (LOG.isDebugEnabled()) { - LOG.debug("No analyzers available for function '{}'", macroCall.getResolvedFunctionName()); - } - return false; // no analyzers - } - - for (MacroCallAnalyzer analyzer : analyzers) { - if (LOG.isTraceEnabled()) { - LOG.trace("Processing macro call '{}' with analyzer '{}'", macroCall, analyzer); - } - var wasAnalyzed = analyzer.analyze(macroCall, javaInfo); - if (wasAnalyzed) { - if (LOG.isDebugEnabled()) { - LOG.debug("Analyzer '{}' successfully processed macro call '{}'", analyzer.getClass(), macroCall); - } - return true; // stop processing - } - } - - return false; // not analyzed - } } diff --git a/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/BuildfileDrivenProvisioningStrategy.java b/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/BuildfileDrivenProvisioningStrategy.java new file mode 100644 index 00000000..e546c98c --- /dev/null +++ b/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/BuildfileDrivenProvisioningStrategy.java @@ -0,0 +1,268 @@ +package com.salesforce.bazel.eclipse.core.model.discovery; + +import static com.salesforce.bazel.eclipse.core.BazelCoreSharedContstants.BUILDPATH_PROBLEM_MARKER; +import static java.lang.String.format; +import static java.nio.file.Files.isRegularFile; +import static java.util.stream.Collectors.joining; +import static org.eclipse.core.resources.IResource.DEPTH_ZERO; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; + +import org.eclipse.core.runtime.CoreException; +import org.eclipse.core.runtime.IPath; +import org.eclipse.core.runtime.IProgressMonitor; +import org.eclipse.core.runtime.Status; +import org.eclipse.core.runtime.SubMonitor; +import org.eclipse.jdt.core.IClasspathEntry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.idea.blaze.base.model.primitives.TargetExpression; +import com.google.idea.blaze.base.model.primitives.TargetName; +import com.salesforce.bazel.eclipse.core.classpath.BazelClasspathScope; +import com.salesforce.bazel.eclipse.core.classpath.CompileAndRuntimeClasspath; +import com.salesforce.bazel.eclipse.core.model.BazelProject; +import com.salesforce.bazel.eclipse.core.model.BazelTarget; +import com.salesforce.bazel.eclipse.core.model.BazelWorkspace; +import com.salesforce.bazel.eclipse.core.model.buildfile.FunctionCall; +import com.salesforce.bazel.eclipse.core.model.discovery.classpath.ClasspathEntry; +import com.salesforce.bazel.eclipse.core.model.discovery.classpath.libs.ExternalLibrariesDiscovery; +import com.salesforce.bazel.eclipse.core.model.discovery.projects.JavaProjectInfo; +import com.salesforce.bazel.eclipse.core.model.discovery.projects.JavaSourceEntry; +import com.salesforce.bazel.eclipse.core.util.trace.TracingSubMonitor; +import com.salesforce.bazel.sdk.model.BazelLabel; + +/** + * Implementation of {@link TargetProvisioningStrategy} which provisions projects based on parsing BUILD + * files directly and computing their classpath based on visibility in the build graph. + *

+ * This strategy implements behavior which intentionally deviates from Bazel dominated strategies in favor of a better + * developer experience in IDEs. + *

    + *
  • BUILD files are parsed and macro/function calls translated into projects.
  • + *
  • The macro translation is extensible so translators for custom macros can be provided and included in the + * analysis.
  • + *
  • A heuristic is used to merge java_* targets in the same package into a single Eclipse project.
  • + *
  • The classpath is computed based on visibility, which eventually allows to compute the deps list by IDEs based on + * actual use.
  • + *
  • Projects are created directly in the package location.
  • + *
  • The root (empty) package // is not supported.
  • + *
+ *

+ */ +public class BuildfileDrivenProvisioningStrategy extends ProjectPerPackageProvisioningStrategy { + + public static final String STRATEGY_NAME = "project-per-buildfile"; + + private static Logger LOG = LoggerFactory.getLogger(BuildfileDrivenProvisioningStrategy.class); + + private final TargetDiscoveryAndProvisioningExtensionLookup extensionLookup = + new TargetDiscoveryAndProvisioningExtensionLookup(); + + @Override + public Map computeClasspaths(Collection bazelProjects, + BazelWorkspace workspace, BazelClasspathScope scope, IProgressMonitor progress) throws CoreException { + LOG.debug("Computing classpath for projects: {}", bazelProjects); + try { + var monitor = SubMonitor.convert(progress, "Computing classpaths...", 2 + bazelProjects.size()); + + Map> activeTargetsPerProject = new HashMap<>(); + for (BazelProject bazelProject : bazelProjects) { + monitor.checkCanceled(); + + if (!bazelProject.isPackageProject()) { + throw new CoreException( + Status.error( + format( + "Unable to compute classpath for project '%s'. Please check the setup. This is not a Bazel package project created by the project per package strategy.", + bazelProject))); + } + + var targetsToBuild = bazelProject.getBazelTargets(); + if (targetsToBuild.isEmpty()) { + throw new CoreException( + Status.error( + format( + "Unable to compute classpath for project '%s'. Please check the setup. This Bazel package project is missing information about the relevant targets.", + bazelProject))); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Found targets for project '{}': {}", bazelProject, targetsToBuild); + } + + var packageTargets = targetsToBuild.stream().map(BazelTarget::getLabel).toList(); + activeTargetsPerProject.put(bazelProject, packageTargets); + } + + var jarResolver = new JavaClasspathJarLocationResolver(workspace); + var externalLibrariesDiscovery = new ExternalLibrariesDiscovery(workspace); + var externalLibraries = externalLibrariesDiscovery.query(monitor.split(1)); + + var workspaceRoot = workspace.getLocation().toPath(); + + Map classpathsByProject = new HashMap<>(); + for (BazelProject bazelProject : bazelProjects) { + monitor.subTask("Analyzing: " + bazelProject); + monitor.checkCanceled(); + + // cleanup markers from previous runs + bazelProject.getProject().deleteMarkers(BUILDPATH_PROBLEM_MARKER, true, DEPTH_ZERO); + + // compute the classpath + var classpath = new LinkedHashSet<>(externalLibraries); + + // check for non existing jars + for (ClasspathEntry entry : classpath) { + if (entry.getEntryKind() != IClasspathEntry.CPE_LIBRARY) { + continue; + } + + if (!isRegularFile(entry.getPath().toPath())) { + createBuildPathProblem( + bazelProject, + Status.error( + format( + "Library '%s' is missing. Please consider running 'bazel fetch'", + entry.getPath()))); + break; + } + } + + classpathsByProject + .put(bazelProject, new CompileAndRuntimeClasspath(classpath, Collections.emptyList())); + monitor.worked(1); + } + + return classpathsByProject; + } finally { + if (progress != null) { + progress.done(); + } + } + } + + @Override + protected List doProvisionProjects(Collection targetsOrPackages, + BazelWorkspace workspace, TracingSubMonitor monitor) throws CoreException { + + // obtain package paths + var packages = targetsOrPackages.parallelStream().map(this::extractPackagePath).distinct().toList(); + + monitor.beginTask("Provisioning projects", packages.size() * 3); + var result = new ArrayList(); + for (Path packagePath : packages) { + var bazelPackage = workspace.getBazelPackage(IPath.fromPath(packagePath)); + + // skip the root package (not supported) + if (bazelPackage.isRoot()) { + createBuildPathProblem( + bazelPackage.getBazelWorkspace().getBazelProject(), + Status.warning( + format( + "The root package was skipped during sync because it's not supported by the '%s' strategy. Consider excluding it in the .bazelproject file.", + STRATEGY_NAME))); + continue; + } + + // get the top-level macro calls + var topLevelMacroCalls = bazelPackage.getBazelBuildFile().getTopLevelCalls(); + + // build the project information as we traverse the macros + var javaInfo = new JavaProjectInfo(bazelPackage); + var relevantTargets = new ArrayList(); + for (FunctionCall macroCall : topLevelMacroCalls) { + var relevant = processMacroCall(macroCall, javaInfo); + if (!relevant) { + if (LOG.isDebugEnabled()) { + LOG.debug("Skipping not relevant macro call '{}'.", macroCall); + } + continue; + } + var name = macroCall.getName(); + if (name != null) { + relevantTargets.add(TargetName.create(name)); + } + } + + // create the project for the package + var project = provisionPackageProject(bazelPackage, monitor.slice(1)); + + // remember/update the targets to build for the project + project.setBazelTargetNames(relevantTargets, monitor.slice(1)); + + // create markers + analyzeProjectInfo(project, javaInfo, monitor); + + // sanity check + var sourceInfo = javaInfo.getSourceInfo(); + if (sourceInfo.hasSourceFilesWithoutCommonRoot()) { + for (JavaSourceEntry file : sourceInfo.getSourceFilesWithoutCommonRoot()) { + createBuildPathProblem( + project, + Status.warning( + format( + "File '%s' could not be mapped into a common source directory. The project may not build successful in Eclipse.", + file.getPath()))); + } + } + if (!sourceInfo.hasSourceDirectories()) { + createBuildPathProblem( + project, + Status.error( + format( + "No source directories detected when analyzing package '%s' using targets '%s'", + bazelPackage.getLabel().getPackagePath(), + relevantTargets.stream().map(TargetName::toString).collect(joining(", "))))); + } + + // configure links + linkGeneratedSourcesIntoProject(project, javaInfo, monitor.slice(1)); + + // configure classpath + configureRawClasspath(project, javaInfo, monitor.slice(1)); + + result.add(project); + } + return result; + } + + private Path extractPackagePath(TargetExpression targetExpression) { + var targetExpressionStr = targetExpression.toString(); + var startIndex = targetExpressionStr.indexOf("//") + "//".length(); + var colonIndex = targetExpressionStr.lastIndexOf(':'); + return Path.of(targetExpressionStr.substring(startIndex, colonIndex)); + } + + private boolean processMacroCall(FunctionCall macroCall, JavaProjectInfo javaInfo) throws CoreException { + var analyzers = extensionLookup.createMacroCallAnalyzers(macroCall.getResolvedFunctionName()); + if (analyzers.isEmpty()) { + if (LOG.isDebugEnabled()) { + LOG.debug("No analyzers available for function '{}'", macroCall.getResolvedFunctionName()); + } + return false; // no analyzers + } + + for (MacroCallAnalyzer analyzer : analyzers) { + if (LOG.isTraceEnabled()) { + LOG.trace("Processing macro call '{}' with analyzer '{}'", macroCall, analyzer); + } + var wasAnalyzed = analyzer.analyze(macroCall, javaInfo); + if (wasAnalyzed) { + if (LOG.isDebugEnabled()) { + LOG.debug("Analyzer '{}' successfully processed macro call '{}'", analyzer.getClass(), macroCall); + } + return true; // stop processing + } + } + + return false; // not analyzed + } +} diff --git a/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/ProjectPerPackageProvisioningStrategy.java b/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/ProjectPerPackageProvisioningStrategy.java index f4ee42f0..848c0c65 100644 --- a/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/ProjectPerPackageProvisioningStrategy.java +++ b/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/ProjectPerPackageProvisioningStrategy.java @@ -349,7 +349,10 @@ protected List doProvisionProjects(Collection targets } // create the project for the package - var project = provisionPackageProject(bazelPackage, packageTargets, monitor.slice(1)); + var project = provisionPackageProject(bazelPackage, monitor.slice(1)); + + // remember/update the targets to build for the project + project.setBazelTargets(packageTargets, monitor.slice(1)); // build the Java information var javaInfo = collectJavaInfo(project, packageTargets, monitor.slice(1)); @@ -453,8 +456,8 @@ private void logSourceFilesWithoutCommonRoot(BazelProject project, JavaSourceInf + " Please check the error log and reach out for help.")); } - protected BazelProject provisionPackageProject(BazelPackage bazelPackage, List targets, - IProgressMonitor monitor) throws CoreException { + protected BazelProject provisionPackageProject(BazelPackage bazelPackage, IProgressMonitor monitor) + throws CoreException { try { monitor.beginTask(format("Provisioning project for '//%s'", bazelPackage.getLabel().getPackagePath()), 2); if (!bazelPackage.hasBazelProject()) { @@ -471,18 +474,10 @@ protected BazelProject provisionPackageProject(BazelPackage bazelPackage, Listanalyze function + */ +@StarlarkBuiltin(name = "AnalyzeInfo", documented = false) +public class StarlarkAnalyzeInfo { + + private List convertToStringList(Sequence exclude, String nameForErrorMessage) throws EvalException { + List stringList = new ArrayList<>(); + for (Object value : exclude) { + if (!(value instanceof String s)) { + throw Starlark.errorf("Invalid 'glob' argument type in '%s': %s", nameForErrorMessage, value); + } + stringList.add(s); + } + return stringList; + } + + /** + * Creates the project information + */ + @StarlarkMethod(name = "ProjectInfo", documented = false, parameters = { + @Param(name = "srcDirectories", allowedTypes = { + @ParamType(type = Sequence.class, generic1 = String.class) }, defaultValue = "[]", named = true, documented = false), + @Param(name = "exclude", allowedTypes = { + @ParamType(type = Sequence.class, generic1 = String.class) }, defaultValue = "[]", named = true, documented = false), + @Param(name = "exclude_directories", defaultValue = "1", named = true, documented = false), + @Param(name = "allow_empty", defaultValue = "unbound", named = true, documented = false) }) + StarlarkGlobInfo glob(Sequence include, Sequence exclude, StarlarkInt excludeDirectories, Object allowEmpty) + throws EvalException, InterruptedException { + + var includeStringList = convertToStringList(include, "include"); + var excludeStringList = convertToStringList(exclude, "exclude"); + return new StarlarkGlobInfo(new GlobInfo(includeStringList, excludeStringList)); + + } +} diff --git a/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/analyzers/starlark/StarlarkFunctionCallInfo.java b/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/analyzers/starlark/StarlarkFunctionCallInfo.java new file mode 100644 index 00000000..4778ce5d --- /dev/null +++ b/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/analyzers/starlark/StarlarkFunctionCallInfo.java @@ -0,0 +1,94 @@ +/*- + * Copyright (c) 2024 Salesforce and others. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Salesforce - adapted from M2E, JDT or other Eclipse project + */ +package com.salesforce.bazel.eclipse.core.model.discovery.analyzers.starlark; + +import static java.lang.String.format; + +import java.util.Map; + +import org.eclipse.core.runtime.OperationCanceledException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableMap; +import com.salesforce.bazel.eclipse.core.model.buildfile.FunctionCall; + +import net.starlark.java.annot.StarlarkBuiltin; +import net.starlark.java.annot.StarlarkMethod; +import net.starlark.java.eval.Dict; +import net.starlark.java.eval.Dict.Builder; +import net.starlark.java.eval.Starlark; +import net.starlark.java.eval.StarlarkThread; +import net.starlark.java.eval.StarlarkValue; +import net.starlark.java.syntax.Argument; +import net.starlark.java.syntax.FileOptions; +import net.starlark.java.syntax.ParserInput; + +/** + * A data type which exposes {@link FunctionCall} info to Starlark . + */ +@StarlarkBuiltin(name = "function_call_info", documented = false) +class StarlarkFunctionCallInfo implements StarlarkValue { + + private static final Map predeclared; + static { + ImmutableMap.Builder predeclaredEnvBuilder = ImmutableMap.builder(); + Starlark.addMethods(predeclaredEnvBuilder, new StarlarkNativeModuleApiDummy()); + predeclared = predeclaredEnvBuilder.build(); + } + + private static final Logger LOG = LoggerFactory.getLogger(StarlarkFunctionCallInfo.class); + + private final FunctionCall functionCall; + + public StarlarkFunctionCallInfo(FunctionCall functionCall) { + this.functionCall = functionCall; + } + + private Dict evaluateArgs(StarlarkThread thread) { + var callExpression = functionCall.getCallExpression(); + + Builder result = Dict.builder(); + + for (Argument argument : callExpression.getArguments()) { + if (argument.getName() == null) { + continue; + } + try { + + var parserInput = ParserInput + .fromString(argument.getValue().prettyPrint(), format("argument %s", argument.getName())); + + result.put(argument.getName(), Starlark.eval(parserInput, FileOptions.DEFAULT, predeclared, thread)); + } catch (InterruptedException e) { + throw new OperationCanceledException("Interrupted Starlark execution"); + } catch (Exception e) { + LOG.warn("Unable to evaluate argument '{}' in function call '{}'", argument.getName(), functionCall, e); + } + + } + + return result.buildImmutable(); + } + + @StarlarkMethod(name = "args", structField = true, useStarlarkThread = true) + public Dict getArgs(StarlarkThread thread) { + return evaluateArgs(thread); + } + + @StarlarkMethod(name = "resolved_function_name", structField = true) + public String getResolvedFunctionName() { + return functionCall.getResolvedFunctionName(); + } +} diff --git a/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/analyzers/starlark/StarlarkGlobInfo.java b/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/analyzers/starlark/StarlarkGlobInfo.java new file mode 100644 index 00000000..c0c89134 --- /dev/null +++ b/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/analyzers/starlark/StarlarkGlobInfo.java @@ -0,0 +1,51 @@ +/*- + * Copyright (c) 2024 Salesforce and others. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Salesforce - adapted from M2E, JDT or other Eclipse project + */ +package com.salesforce.bazel.eclipse.core.model.discovery.analyzers.starlark; + +import com.salesforce.bazel.eclipse.core.model.buildfile.GlobInfo; + +import net.starlark.java.annot.StarlarkBuiltin; +import net.starlark.java.annot.StarlarkMethod; +import net.starlark.java.eval.Sequence; +import net.starlark.java.eval.StarlarkList; +import net.starlark.java.eval.StarlarkValue; + +/** + * Exposes {@link GlobInfo} to Starlark + */ +@StarlarkBuiltin(name = "glob_info", documented = false) +class StarlarkGlobInfo implements StarlarkValue { + + private final GlobInfo globInfo; + + public StarlarkGlobInfo(GlobInfo globInfo) { + this.globInfo = globInfo; + } + + /** + * {@return {@link GlobInfo#exclude()}} + */ + @StarlarkMethod(name = "exclude", structField = true) + public Sequence exclude() { + return StarlarkList.immutableCopyOf(globInfo.exclude()); + } + + /** + * {@return {@link GlobInfo#include()}} + */ + @StarlarkMethod(name = "info", structField = true) + public Sequence include() { + return StarlarkList.immutableCopyOf(globInfo.include()); + } +} diff --git a/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/analyzers/starlark/StarlarkMacroCallAnalyzer.java b/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/analyzers/starlark/StarlarkMacroCallAnalyzer.java new file mode 100644 index 00000000..f22f1638 --- /dev/null +++ b/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/analyzers/starlark/StarlarkMacroCallAnalyzer.java @@ -0,0 +1,139 @@ +/*- + * Copyright (c) 2024 Salesforce and others. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Salesforce - adapted from M2E, JDT or other Eclipse project + */ +package com.salesforce.bazel.eclipse.core.model.discovery.analyzers.starlark; + +import static java.lang.String.format; + +import java.io.IOException; +import java.util.Map; + +import org.eclipse.core.runtime.CoreException; +import org.eclipse.core.runtime.IPath; +import org.eclipse.core.runtime.OperationCanceledException; +import org.eclipse.core.runtime.Status; + +import com.google.common.collect.ImmutableMap; +import com.google.idea.blaze.base.model.primitives.WorkspacePath; +import com.salesforce.bazel.eclipse.core.model.BazelWorkspace; +import com.salesforce.bazel.eclipse.core.model.buildfile.FunctionCall; +import com.salesforce.bazel.eclipse.core.model.discovery.MacroCallAnalyzer; +import com.salesforce.bazel.eclipse.core.model.discovery.projects.JavaProjectInfo; + +import net.starlark.java.eval.EvalException; +import net.starlark.java.eval.Module; +import net.starlark.java.eval.Mutability; +import net.starlark.java.eval.Starlark; +import net.starlark.java.eval.StarlarkFunction; +import net.starlark.java.eval.StarlarkSemantics; +import net.starlark.java.eval.StarlarkThread; +import net.starlark.java.syntax.FileOptions; +import net.starlark.java.syntax.ParserInput; +import net.starlark.java.syntax.SyntaxError; + +/** + * A generic analyzer for {@link FunctionCall} which delegates analysis to a Starlark function. + *

+ * This allows contributing {@link MacroCallAnalyzer} in Starlark outside the IDE. The Starlark execution is limited, + * though. Bazel specific Starlark globals are not supported. + *

+ *

+ * The analyzer must be defined in a .bzl file. The file must define a function named analyze. + * The function will be called with the following named parameters: + *

    + *
  • function_info - info about the function call (see {@link StarlarkFunctionCallInfo} for details)
  • + *
+ *

+ *

+ * The following limitations apply: + *

    + *
  • Function arguments will only be detected if they evaluate to a simple value, i.e. complex expressions may not + * yield a value and thus need to be skipped.
  • + *
+ *

+ */ +public class StarlarkMacroCallAnalyzer implements MacroCallAnalyzer { + + private static final StarlarkSemantics starlarkSemantics = + StarlarkSemantics.builder().setBool(StarlarkSemantics.EXPERIMENTAL_ENABLE_STARLARK_SET, true).build(); + + private final IPath analyzeFile; + private final StarlarkFunction analyzeFunction; + + public StarlarkMacroCallAnalyzer(BazelWorkspace bazelWorkspace, WorkspacePath bzlFile) + throws CoreException, OperationCanceledException { + analyzeFile = bazelWorkspace.getLocation().append(bzlFile.relativePath()); + ParserInput input; + try { + input = ParserInput.readFile(analyzeFile.toOSString()); + } catch (IOException e) { + throw new CoreException(Status.error(format("Failed to read file '%s'", analyzeFile), e)); + } + + try (var mu = Mutability.create("analyzer")) { + ImmutableMap.Builder env = ImmutableMap.builder(); + //Starlark.addMethods(env, new CqueryDialectGlobals(), starlarkSemantics); + var module = Module.withPredeclared(starlarkSemantics, env.buildOrThrow()); + + var thread = StarlarkThread.createTransient(mu, starlarkSemantics); + Starlark.execFile(input, FileOptions.DEFAULT, module, thread); + var analyzeFn = module.getGlobal("analyze"); + if (analyzeFn == null) { + throw new CoreException( + Status.error(format("File '%s' does not define 'analyze' function", analyzeFile))); + } + if (!(analyzeFn instanceof StarlarkFunction)) { + throw new CoreException( + Status.error( + format( + "File '%s' 'analyze' is not a function. Got '%s'.", + analyzeFile, + Starlark.type(analyzeFn)))); + } + analyzeFunction = (StarlarkFunction) analyzeFn; + if (analyzeFunction.getParameterNames().size() != 1) { + throw new CoreException( + Status.error(format("File '%s' 'format' function must take exactly 1 argument", analyzeFile))); + } + } catch (SyntaxError.Exception e) { + throw new CoreException( + Status.error(format("Syntax error in file '%s': %s", analyzeFile, e.getMessage()), e)); + } catch (EvalException e) { + throw new CoreException( + Status.error(format("Evaluation error in file '%s': %s", analyzeFile, e.getMessage()), e)); + } catch (InterruptedException e) { + throw new OperationCanceledException("Interrupted while executing Starlark"); + } + } + + @Override + public boolean analyze(FunctionCall macroCall, JavaProjectInfo javaInfo) throws CoreException { + try { + var thread = StarlarkThread.createTransient(Mutability.create("analyze evaluation"), starlarkSemantics); + thread.setMaxExecutionSteps(500_000L); + + var kwargs = Map. of("macro_info", new StarlarkFunctionCallInfo(macroCall)); + var result = Starlark.call(thread, analyzeFunction, null, kwargs); + } catch (EvalException e) { + throw new CoreException( + Status.error( + format("Error executiong 'analyze' in file '%s': %s", analyzeFile, e.getMessage()), + e)); + } catch (InterruptedException e) { + throw new OperationCanceledException("Interrupted Starlark execution"); + } + + return false; + } + +} diff --git a/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/analyzers/starlark/StarlarkNativeModuleApiDummy.java b/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/analyzers/starlark/StarlarkNativeModuleApiDummy.java new file mode 100644 index 00000000..e8b9065c --- /dev/null +++ b/bundles/com.salesforce.bazel.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/discovery/analyzers/starlark/StarlarkNativeModuleApiDummy.java @@ -0,0 +1,67 @@ +/*- + * Copyright (c) 2024 Salesforce and others. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Salesforce - adapted from M2E, JDT or other Eclipse project + */ +package com.salesforce.bazel.eclipse.core.model.discovery.analyzers.starlark; + +import java.util.ArrayList; +import java.util.List; + +import com.salesforce.bazel.eclipse.core.model.buildfile.GlobInfo; + +import net.starlark.java.annot.Param; +import net.starlark.java.annot.ParamType; +import net.starlark.java.annot.StarlarkBuiltin; +import net.starlark.java.annot.StarlarkMethod; +import net.starlark.java.eval.EvalException; +import net.starlark.java.eval.Sequence; +import net.starlark.java.eval.Starlark; +import net.starlark.java.eval.StarlarkInt; + +/** + * A dummy of the Bazel Build API native module to allow intercepting glob in evaluations. + */ +@StarlarkBuiltin(name = "native", documented = false) +public class StarlarkNativeModuleApiDummy { + + private List convertToStringList(Sequence exclude, String nameForErrorMessage) throws EvalException { + List stringList = new ArrayList<>(); + for (Object value : exclude) { + if (!(value instanceof String s)) { + throw Starlark.errorf("Invalid 'glob' argument type in '%s': %s", nameForErrorMessage, value); + } + stringList.add(s); + } + return stringList; + } + + /** + * Support for glob to turn into {@link StarlarkGlobInfo}. + * + * @see https://github.com/bazelbuild/bazel/blob/984d1bad444797db0d60692c9dfaadc6d450752e/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/StarlarkNativeModuleApi.java#L91 + */ + @StarlarkMethod(name = "glob", documented = false, parameters = { + @Param(name = "include", allowedTypes = { + @ParamType(type = Sequence.class, generic1 = String.class) }, defaultValue = "[]", named = true, documented = false), + @Param(name = "exclude", allowedTypes = { + @ParamType(type = Sequence.class, generic1 = String.class) }, defaultValue = "[]", named = true, documented = false), + @Param(name = "exclude_directories", defaultValue = "1", named = true, documented = false), + @Param(name = "allow_empty", defaultValue = "unbound", named = true, documented = false) }) + StarlarkGlobInfo glob(Sequence include, Sequence exclude, StarlarkInt excludeDirectories, Object allowEmpty) + throws EvalException, InterruptedException { + + var includeStringList = convertToStringList(include, "include"); + var excludeStringList = convertToStringList(exclude, "exclude"); + return new StarlarkGlobInfo(new GlobInfo(includeStringList, excludeStringList)); + + } +}