Add macro() .bzl callable, for defining symbolic macros

The ability to define symbolic macros in the build is guarded by --experimental_enable_first_class_macros. (For the moment, we're using "first class macros" in user documentation and "symbolic macros" or sometimes just "macros" in internal code.)

Symbolic macro types follow an analogous pattern to rules:

- A MacroFunction (StarlarkRuleFunction) is the Starlark callable object, returned by `macro()`, that can be invoked during package loading to instantiate the symbolic macro. MacroFunctions accept the name of the instance (like a target name) and return `None`.
- A MacroClass (RuleClass) represents the information needed to actually instantiate the macro, such as the name the macro was exported with, and its implementation function.
- A MacroInstance (Rule) represents the result of calling the macro during package loading, and is the object tracked by Package.Builder.

Note that MacroFunction references MacroInstance references MacroClass.

Package / Package.Builder gain new fields and methods to track macro instances created during BUILD evaluation. A follow-up CL will actually evaluate macro implementation functions. A different follow-up will add name conflict checking so that macro instance names are guaranteed to not clash with targets in the package.

StarlarkRuleClassFunctions:
- Update javadoc on StarlarkRuleFunction; add @nullable, field comments, and other comments

Work toward #19922.

PiperOrigin-RevId: 597571980
Change-Id: I1789d94e02c45e23b25c80990fb6ccef82e13da9

src/main/java/com/google/devtools/build/lib/analysis/starlark/StarlarkRuleClassFunctions.java

@@ -75,6 +75,8 @@
 import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.StarlarkImplicitOutputsFunctionWithCallback;
 import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.StarlarkImplicitOutputsFunctionWithMap;
 import com.google.devtools.build.lib.packages.LabelConverter;
+import com.google.devtools.build.lib.packages.MacroClass;
+import com.google.devtools.build.lib.packages.MacroInstance;
 import com.google.devtools.build.lib.packages.Package;
 import com.google.devtools.build.lib.packages.Package.NameConflictException;
 import com.google.devtools.build.lib.packages.PredicateWithMessage;
@@ -311,6 +313,22 @@ private static void failIf(boolean condition, String message, Object... args)
     }
   }
 
+  @Override
+  public StarlarkCallable macro(StarlarkFunction implementation, StarlarkThread thread)
+      throws EvalException {
+    // Ordinarily we would use StarlarkMethod#enableOnlyWithFlag, but this doesn't work for
+    // top-level symbols (due to StarlarkGlobalsImpl relying on the Starlark#addMethods overload
+    // that uses default StarlarkSemantics), so enforce it here instead.
+    if (!thread
+        .getSemantics()
+        .getBool(BuildLanguageOptions.EXPERIMENTAL_ENABLE_FIRST_CLASS_MACROS)) {
+      throw Starlark.errorf("Use of `macro()` requires --experimental_enable_first_class_macros");
+    }
+
+    MacroClass.Builder builder = new MacroClass.Builder(implementation);
+    return new MacroFunction(builder);
+  }
+
   // TODO(bazel-team): implement attribute copy and other rule properties
   @Override
   public StarlarkRuleFunction rule(
@@ -434,7 +452,7 @@ public StarlarkRuleFunction rule(
   }
 
   /**
-   * Returns a new function representing a Starlark-defined rule.
+   * Returns a new callable representing a Starlark-defined rule.
    *
    * <p>This is public for the benefit of {@link StarlarkTestingModule}, which has the unusual use
    * case of creating new rule types to house analysis-time test assertions ({@code analysis_test}).
@@ -974,26 +992,143 @@ private static ImmutableSet<String> getLegacyAnyTypeAttrs(RuleClass ruleClass) {
   }
 
   /**
-   * The implementation for the magic function "rule" that creates Starlark rule classes.
+   * A callable Starlark object representing a symbolic macro, which may be invoked during package
+   * construction time to instantiate the macro.
+   *
+   * <p>Instantiating the macro does not necessarily imply that the macro's implementation function
+   * will run synchronously with the call to this object. Just like a rule, a macro's implementation
+   * function is evaluated in its own context separate from the caller.
+   *
+   * <p>This object is not usable until it has been {@link #export exported}. Calling an unexported
+   * macro function results in an {@link EvalException}.
+   */
+  public static final class MacroFunction implements StarlarkExportable, StarlarkCallable {
+
+    // Initially non-null, then null once exported.
+    @Nullable private MacroClass.Builder builder;
+
+    // Initially null, then non-null once exported.
+    @Nullable private MacroClass macroClass = null;
+
+    public MacroFunction(MacroClass.Builder builder) {
+      this.builder = builder;
+    }
+
+    @Override
+    public String getName() {
+      return macroClass != null ? macroClass.getName() : "unexported macro";
+    }
+
+    // TODO(#19922): Define getDocumentation() and interaction with ModuleInfoExtractor, analogous
+    // to StarlarkRuleFunction.
+
+    @Override
+    public Object call(StarlarkThread thread, Tuple args, Dict<String, Object> kwargs)
+        throws EvalException, InterruptedException {
+      BazelStarlarkContext.checkLoadingPhase(thread, getName());
+      Package.Builder pkgBuilder = thread.getThreadLocal(Package.Builder.class);
+      if (pkgBuilder == null) {
+        throw new EvalException(
+            "Cannot instantiate a macro when loading a .bzl file. "
+                + "Macros may only be instantiated while evaluating a BUILD file.");
+      }
+
+      if (macroClass == null) {
+        throw Starlark.errorf(
+            "Cannot instantiate a macro that has not been exported (assign it to a global variable"
+                + " in the .bzl where it's defined)");
+      }
+
+      if (!args.isEmpty()) {
+        throw Starlark.errorf("unexpected positional arguments");
+      }
+
+      Object nameUnchecked = kwargs.get("name");
+      if (nameUnchecked == null) {
+        throw Starlark.errorf("macro requires a `name` attribute");
+      }
+      if (!(nameUnchecked instanceof String)) {
+        throw Starlark.errorf(
+            "Expected a String for attribute 'name'; got %s",
+            nameUnchecked.getClass().getSimpleName());
+      }
+      String instanceName = (String) nameUnchecked;
+
+      MacroInstance macroInstance = new MacroInstance(macroClass, instanceName);
+      try {
+        pkgBuilder.addMacro(macroInstance);
+      } catch (NameConflictException e) {
+        throw new EvalException(e);
+      }
+      return Starlark.NONE;
+    }
+
+    @Override
+    public void export(EventHandler handler, Label label, String exportedName) {
+      checkState(builder != null && macroClass == null);
+      builder.setName(exportedName);
+      this.macroClass = builder.build();
+      this.builder = null;
+    }
+
+    @Override
+    public boolean isExported() {
+      return macroClass != null;
+    }
+
+    @Override
+    public void repr(Printer printer) {
+      if (isExported()) {
+        printer.append("<macro ").append(macroClass.getName()).append(">");
+      } else {
+        printer.append("<macro>");
+      }
+    }
+
+    @Override
+    public String toString() {
+      return "macro(...)";
+    }
+
+    @Override
+    public boolean isImmutable() {
+      // TODO(bazel-team): This seems technically wrong, analogous to
+      // StarlarkRuleFunction#isImmutable.
+      return true;
+    }
+  }
+
+  /**
+   * A callable Starlark object representing a Starlark-defined rule, which may be invoked during
+   * package construction time to instantiate the rule.
    *
-   * <p>Exactly one of {@link #builder} or {@link #ruleClass} is null except inside {@link #export}.
+   * <p>This is the object returned by calling {@code rule()}, e.g. the value that is bound in
+   * {@code my_rule = rule(...)}}.
    */
   public static final class StarlarkRuleFunction implements StarlarkExportable, RuleFunction {
-    private RuleClass.Builder builder;
+    // Initially non-null, then null once exported.
+    @Nullable private RuleClass.Builder builder;
+
+    // Initially null, then non-null once exported.
+    @Nullable private RuleClass ruleClass;
 
-    private RuleClass ruleClass;
     private final Location definitionLocation;
     @Nullable private final String documentation;
-    private Label starlarkLabel;
+
+    // Set upon export.
+    @Nullable private Label starlarkLabel;
 
     // TODO(adonovan): merge {Starlark,Builtin}RuleFunction and RuleClass,
     // making the latter a callable, StarlarkExportable value.
     // (Making RuleClasses first-class values will help us to build a
     // rich query output mode that includes values from loaded .bzl files.)
+    // [Note from brandjon: Even if we merge RuleFunction and RuleClass, it may still be useful to
+    // carry a distinction between loading-time vs analysis-time information about a rule type,
+    // particularly when it comes to the possibility of lazy .bzl loading. For example, you can in
+    // principle evaluate a BUILD file without loading and digesting .bzls that are only used by the
+    // implementation function.]
     public StarlarkRuleFunction(
-        RuleClass.Builder builder,
-        Location definitionLocation,
-        Optional<String> documentation) {
+        RuleClass.Builder builder, Location definitionLocation, Optional<String> documentation) {
       this.builder = builder;
       this.definitionLocation = definitionLocation;
       this.documentation = documentation.orElse(null);
@@ -1245,6 +1380,7 @@ public String toString() {
 
     @Override
     public boolean isImmutable() {
+      // TODO(bazel-team): It shouldn't be immutable until it's exported, no?
       return true;
     }
   }

src/main/java/com/google/devtools/build/lib/packages/MacroClass.java

@@ -0,0 +1,68 @@
+// Copyright 2024 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.base.Preconditions;
+import com.google.errorprone.annotations.CanIgnoreReturnValue;
+import javax.annotation.Nullable;
+import net.starlark.java.eval.StarlarkFunction;
+
+/**
+ * Represents a symbolic macro, defined in a .bzl file, that may be instantiated during Package
+ * evaluation.
+ *
+ * <p>This is analogous to {@link RuleClass}. In essence, a {@code MacroClass} consists of the
+ * macro's schema and its implementation function.
+ */
+public final class MacroClass {
+
+  private final String name;
+  private final StarlarkFunction implementation;
+
+  public MacroClass(String name, StarlarkFunction implementation) {
+    this.name = name;
+    this.implementation = implementation;
+  }
+
+  /** Returns the macro's exported name. */
+  public String getName() {
+    return name;
+  }
+
+  public StarlarkFunction getImplementation() {
+    return implementation;
+  }
+
+  /** Builder for {@link MacroClass}. */
+  public static final class Builder {
+    private final StarlarkFunction implementation;
+    @Nullable private String name = null;
+
+    public Builder(StarlarkFunction implementation) {
+      this.implementation = implementation;
+    }
+
+    @CanIgnoreReturnValue
+    public Builder setName(String name) {
+      this.name = name;
+      return this;
+    }
+
+    public MacroClass build() {
+      Preconditions.checkNotNull(name);
+      return new MacroClass(name, implementation);
+    }
+  }
+}

src/main/java/com/google/devtools/build/lib/packages/MacroInstance.java

@@ -0,0 +1,47 @@
+// Copyright 2024 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+/**
+ * Represents a use of a symbolic macro in a package.
+ *
+ * <p>There is one {@code MacroInstance} for each call to a {@link
+ * StarlarkRuleClassFunctions#MacroFunction} that is executed during a package's evaluation. Just as
+ * a {@link MacroClass} is analogous to a {@link RuleClass}, {@code MacroInstance} is analogous to a
+ * {@link Rule} (i.e. a rule target).
+ */
+public final class MacroInstance {
+
+  private final MacroClass macroClass;
+  private final String name;
+
+  public MacroInstance(MacroClass macroClass, String name) {
+    this.macroClass = macroClass;
+    this.name = name;
+  }
+
+  /** Returns the {@link MacroClass} (i.e. schema info) that this instance parameterizes. */
+  public MacroClass getMacroClass() {
+    return macroClass;
+  }
+
+  /**
+   * Returns the name of this instance, as given in the {@code name = ...} attribute in the calling
+   * BUILD file or macro.
+   */
+  public String getName() {
+    return name;
+  }
+}