Merge pull request NixOS#8813 from obsidiansystems/outputOf

Create (experimental) `outputOf` primop.

doc/manual/src/release-notes/rl-next.md

@@ -19,3 +19,6 @@
 
 - The JSON output for derived paths with are store paths is now a string, not an object with a single `path` field.
  This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op.
+
+- Introduce a new [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) builtin.
+ It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature.

src/libexpr/eval.cc

@@ -1027,24 +1027,67 @@ void EvalState::mkStorePathString(const StorePath & p, Value & v)
 }
 
 
+std::string EvalState::mkOutputStringRaw(
+ const SingleDerivedPath::Built & b,
+ std::optional<StorePath> optStaticOutputPath,
+ const ExperimentalFeatureSettings & xpSettings)
+{
+ /* In practice, this is testing for the case of CA derivations, or
+ dynamic derivations. */
+ return optStaticOutputPath
+ ? store->printStorePath(*std::move(optStaticOutputPath))
+ /* Downstream we would substitute this for an actual path once
+ we build the floating CA derivation */
+ : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b, xpSettings).render();
+}
+
+
 void EvalState::mkOutputString(
  Value & value,
- const StorePath & drvPath,
- const std::string outputName,
- std::optional<StorePath> optOutputPath,
+ const SingleDerivedPath::Built & b,
+ std::optional<StorePath> optStaticOutputPath,
  const ExperimentalFeatureSettings & xpSettings)
 {
  value.mkString(
- optOutputPath
- ? store->printStorePath(*std::move(optOutputPath))
- /* Downstream we would substitute this for an actual path once
- we build the floating CA derivation */
- : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName, xpSettings).render(),
+ mkOutputStringRaw(b, optStaticOutputPath, xpSettings),
+ NixStringContext { b });
+}
+
+
+std::string EvalState::mkSingleDerivedPathStringRaw(
+ const SingleDerivedPath & p)
+{
+ return std::visit(overloaded {
+ [&](const SingleDerivedPath::Opaque & o) {
+ return store->printStorePath(o.path);
+ },
+ [&](const SingleDerivedPath::Built & b) {
+ auto optStaticOutputPath = std::visit(overloaded {
+ [&](const SingleDerivedPath::Opaque & o) {
+ auto drv = store->readDerivation(o.path);
+ auto i = drv.outputs.find(b.output);
+ if (i == drv.outputs.end())
+ throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*store), b.output);
+ return i->second.path(*store, drv.name, b.output);
+ },
+ [&](const SingleDerivedPath::Built & o) -> std::optional<StorePath> {
+ return std::nullopt;
+ },
+ }, b.drvPath->raw());
+ return mkOutputStringRaw(b, optStaticOutputPath);
+ }
+ }, p.raw());
+}
+
+
+void EvalState::mkSingleDerivedPathString(
+ const SingleDerivedPath & p,
+ Value & v)
+{
+ v.mkString(
+ mkSingleDerivedPathStringRaw(p),
  NixStringContext {
- NixStringContextElem::Built {
- .drvPath = makeConstantStorePathRef(drvPath),
- .output = outputName,
- }
+ std::visit([](auto && v) -> NixStringContextElem { return v; }, p),
  });
 }
 
@@ -2333,39 +2376,25 @@ SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value &
 {
  auto [derivedPath, s_] = coerceToSingleDerivedPathUnchecked(pos, v, errorCtx);
  auto s = s_;
- std::visit(overloaded {
- [&](const SingleDerivedPath::Opaque & o) {
- auto sExpected = store->printStorePath(o.path);
- if (s != sExpected)
+ auto sExpected = mkSingleDerivedPathStringRaw(derivedPath);
+ if (s != sExpected) {
+ /* `std::visit` is used here just to provide a more precise
+ error message. */
+ std::visit(overloaded {
+ [&](const SingleDerivedPath::Opaque & o) {
  error(
  "path string '%s' has context with the different path '%s'",
  s, sExpected)
  .withTrace(pos, errorCtx).debugThrow<EvalError>();
- },
- [&](const SingleDerivedPath::Built & b) {
- auto sExpected = std::visit(overloaded {
- [&](const SingleDerivedPath::Opaque & o) {
- auto drv = store->readDerivation(o.path);
- auto i = drv.outputs.find(b.output);
- if (i == drv.outputs.end())
- throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*store), b.output);
- auto optOutputPath = i->second.path(*store, drv.name, b.output);
- // This is testing for the case of CA derivations
- return optOutputPath
- ? store->printStorePath(*optOutputPath)
- : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render();
- },
- [&](const SingleDerivedPath::Built & o) {
- return DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render();
- },
- }, b.drvPath->raw());
- if (s != sExpected)
+ },
+ [&](const SingleDerivedPath::Built & b) {
  error(
  "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'",
  s, b.output, b.drvPath->to_string(*store), sExpected)
  .withTrace(pos, errorCtx).debugThrow<EvalError>();
- }
- }, derivedPath.raw());
+ }
+ }, derivedPath.raw());
+ }
  return derivedPath;
 }
 

src/libexpr/eval.hh

@@ -668,37 +668,46 @@ public:
  /**
  * Create a string representing a store path.
  *
- * The string is the printed store path with a context containing a single
- * `NixStringContextElem::Opaque` element of that store path.
+ * The string is the printed store path with a context containing a
+ * single `NixStringContextElem::Opaque` element of that store path.
  */
  void mkStorePathString(const StorePath & storePath, Value & v);
 
  /**
- * Create a string representing a `DerivedPath::Built`.
+ * Create a string representing a `SingleDerivedPath::Built`.
  *
- * The string is the printed store path with a context containing a single
- * `NixStringContextElem::Built` element of the drv path and output name.
+ * The string is the printed store path with a context containing a
+ * single `NixStringContextElem::Built` element of the drv path and
+ * output name.
  *
  * @param value Value we are settings
  *
- * @param drvPath Path the drv whose output we are making a string for
+ * @param b the drv whose output we are making a string for, and the
+ * output
  *
- * @param outputName Name of the output
- *
- * @param optOutputPath Optional output path for that string. Must
- * be passed if and only if output store object is input-addressed.
- * Will be printed to form string if passed, otherwise a placeholder
- * will be used (see `DownstreamPlaceholder`).
+ * @param optStaticOutputPath Optional output path for that string.
+ * Must be passed if and only if output store object is
+ * input-addressed or fixed output. Will be printed to form string
+ * if passed, otherwise a placeholder will be used (see
+ * `DownstreamPlaceholder`).
  *
  * @param xpSettings Stop-gap to avoid globals during unit tests.
  */
  void mkOutputString(
  Value & value,
- const StorePath & drvPath,
- const std::string outputName,
- std::optional<StorePath> optOutputPath,
+ const SingleDerivedPath::Built & b,
+ std::optional<StorePath> optStaticOutputPath,
  const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
+ /**
+ * Create a string representing a `SingleDerivedPath`.
+ *
+ * A combination of `mkStorePathString` and `mkOutputString`.
+ */
+ void mkSingleDerivedPathString(
+ const SingleDerivedPath & p,
+ Value & v);
+
  void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx);
 
  /**
@@ -714,6 +723,22 @@ public:
 
 private:
 
+ /**
+ * Like `mkOutputString` but just creates a raw string, not an
+ * string Value, which would also have a string context.
+ */
+ std::string mkOutputStringRaw(
+ const SingleDerivedPath::Built & b,
+ std::optional<StorePath> optStaticOutputPath,
+ const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
+
+ /**
+ * Like `mkSingleDerivedPathStringRaw` but just creates a raw string
+ * Value, which would also have a string context.
+ */
+ std::string mkSingleDerivedPathStringRaw(
+ const SingleDerivedPath & p);
+
  unsigned long nrEnvs = 0;
  unsigned long nrValuesInEnvs = 0;
  unsigned long nrValues = 0;

src/libexpr/primops.cc

@@ -156,8 +156,10 @@ static void mkOutputString(
 {
  state.mkOutputString(
  attrs.alloc(o.first),
- drvPath,
- o.first,
+ SingleDerivedPath::Built {
+ .drvPath = makeConstantStorePathRef(drvPath),
+ .output = o.first,
+ },
  o.second.path(*state.store, Derivation::nameFromPath(drvPath), o.first));
 }
 
@@ -1836,6 +1838,45 @@ static RegisterPrimOp primop_readDir({
  .fun = prim_readDir,
 });
 
+/* Extend single element string context with another output. */
+static void prim_outputOf(EvalState & state, const PosIdx pos, Value * * args, Value & v)
+{
+ SingleDerivedPath drvPath = state.coerceToSingleDerivedPath(pos, *args[0], "while evaluating the first argument to builtins.outputOf");
+
+ std::string_view outputName = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument to builtins.outputOf");
+
+ state.mkSingleDerivedPathString(
+ SingleDerivedPath::Built {
+ .drvPath = make_ref<SingleDerivedPath>(drvPath),
+ .output = std::string { outputName },
+ },
+ v);
+}
+
+static RegisterPrimOp primop_outputOf({
+ .name = "__outputOf",
+ .args = {"derivation-reference", "output-name"},
+ .doc = R"(
+ Return the output path of a derivation, literally or using a placeholder if needed.
+
+ If the derivation has a statically-known output path (i.e. the derivation output is input-addressed, or fixed content-addresed), the output path will just be returned.
+ But if the derivation is content-addressed or if the derivation is itself not-statically produced (i.e. is the output of another derivation), a placeholder will be returned instead.
+
+ *`derivation reference`* must be a string that may contain a regular store path to a derivation, or may be a placeholder reference. If the derivation is produced by a derivation, you must explicitly select `drv.outPath`.
+ This primop can be chained arbitrarily deeply.
+ For instance,
+ ```nix
+ builtins.outputOf
+ (builtins.outputOf myDrv "out)
+ "out"
+ ```
+ will return a placeholder for the output of the output of `myDrv`.
+
+ This primop corresponds to the `^` sigil for derivable paths, e.g. as part of installable syntax on the command line.
+ )",
+ .fun = prim_outputOf,
+ .experimentalFeature = Xp::DynamicDerivations,
+});
 
 /*************************************************************
  * Creating files

src/libexpr/tests/derived-path.cc

@@ -34,8 +34,8 @@ RC_GTEST_FIXTURE_PROP(
 
 RC_GTEST_FIXTURE_PROP(
  DerivedPathExpressionTest,
- prop_built_path_placeholder_round_trip,
- (const StorePath & drvPath, const StorePathName & outputName))
+ prop_derived_path_built_placeholder_round_trip,
+ (const SingleDerivedPath::Built & b))
 {
  /**
  * We set these in tests rather than the regular globals so we don't have
@@ -45,27 +45,19 @@ RC_GTEST_FIXTURE_PROP(
  mockXpSettings.set("experimental-features", "ca-derivations");
 
  auto * v = state.allocValue();
- state.mkOutputString(*v, drvPath, outputName.name, std::nullopt, mockXpSettings);
+ state.mkOutputString(*v, b, std::nullopt, mockXpSettings);
  auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, "");
- SingleDerivedPath::Built b {
- .drvPath = makeConstantStorePathRef(drvPath),
- .output = outputName.name,
- };
  RC_ASSERT(SingleDerivedPath { b } == d);
 }
 
 RC_GTEST_FIXTURE_PROP(
  DerivedPathExpressionTest,
- prop_built_path_out_path_round_trip,
- (const StorePath & drvPath, const StorePathName & outputName, const StorePath & outPath))
+ prop_derived_path_built_out_path_round_trip,
+ (const SingleDerivedPath::Built & b, const StorePath & outPath))
 {
  auto * v = state.allocValue();
- state.mkOutputString(*v, drvPath, outputName.name, outPath);
+ state.mkOutputString(*v, b, outPath);
  auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, "");
- SingleDerivedPath::Built b {
- .drvPath = makeConstantStorePathRef(drvPath),
- .output = outputName.name,
- };
  RC_ASSERT(SingleDerivedPath { b } == d);
 }
 

src/libstore/downstream-placeholder.cc

@@ -39,16 +39,18 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation(
 }
 
 DownstreamPlaceholder DownstreamPlaceholder::fromSingleDerivedPathBuilt(
- const SingleDerivedPath::Built & b)
+ const SingleDerivedPath::Built & b,
+ const ExperimentalFeatureSettings & xpSettings)
 {
  return std::visit(overloaded {
  [&](const SingleDerivedPath::Opaque & o) {
- return DownstreamPlaceholder::unknownCaOutput(o.path, b.output);
+ return DownstreamPlaceholder::unknownCaOutput(o.path, b.output, xpSettings);
  },
  [&](const SingleDerivedPath::Built & b2) {
  return DownstreamPlaceholder::unknownDerivation(
- DownstreamPlaceholder::fromSingleDerivedPathBuilt(b2),
- b.output);
+ DownstreamPlaceholder::fromSingleDerivedPathBuilt(b2, xpSettings),
+ b.output,
+ xpSettings);
  },
  }, b.drvPath->raw());
 }

src/libstore/downstream-placeholder.hh

@@ -84,7 +84,8 @@ public:
  * `SingleDerivedPath::Built.drvPath` chain.
  */
  static DownstreamPlaceholder fromSingleDerivedPathBuilt(
- const SingleDerivedPath::Built & built);
+ const SingleDerivedPath::Built & built,
+ const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 };
 
 }

src/nix/main.cc

@@ -359,6 +359,7 @@ void mainWrapped(int argc, char * * argv)
  experimentalFeatureSettings.experimentalFeatures = {
  Xp::Flakes,
  Xp::FetchClosure,
+ Xp::DynamicDerivations,
  };
  evalSettings.pureEval = false;
  EvalState state({}, openStore("dummy://"));

tests/dyn-drv/dep-built-drv.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+source common.sh
+
+out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link)
+
+clearStore
+
+expectStderr 1 nix-build ./text-hashed-output.nix -A wrapper --no-out-link | grepQuiet "Dependencies on the outputs of dynamic derivations are not yet supported"