Merge pull request #8084 from edolstra/store-docs

Auto-generate store documentation

doc/manual/generate-manpage.nix

@@ -1,4 +1,4 @@
-{ toplevel }:
+cliDumpStr:
 
 with builtins;
 with import ./utils.nix;
@@ -63,7 +63,10 @@ let
         * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description}
       '';
 
-      maybeDocumentation = if details ? doc then details.doc else "";
+      maybeDocumentation =
+        if details ? doc
+        then replaceStrings ["@stores@"] [storeDocs] details.doc
+        else "";
 
       maybeOptions = if details.flags == {} then "" else ''
         # Options
@@ -110,18 +113,32 @@ let
       };
     in [ cmd ] ++ concatMap subcommand (attrNames details.commands or {});
 
-  parsedToplevel = builtins.fromJSON toplevel;
+  cliDump = builtins.fromJSON cliDumpStr;
 
   manpages = processCommand {
     command = "nix";
-    details = parsedToplevel;
+    details = cliDump.args;
     filename = "nix";
-    toplevel = parsedToplevel;
+    toplevel = cliDump.args;
   };
 
   tableOfContents = let
     showEntry = page:
       "    - [${page.command}](command-ref/new-cli/${page.name})";
     in concatStringsSep "\n" (map showEntry manpages) + "\n";
 
+  storeDocs =
+    let
+      showStore = name: { settings, doc }:
+        ''
+          ## ${name}
+
+          ${doc}
+
+          **Settings**:
+
+          ${showSettings { useAnchors = false; } settings}
+        '';
+    in concatStrings (attrValues (mapAttrs showStore cliDump.stores));
+
 in (listToAttrs manpages) // { "SUMMARY.md" = tableOfContents; }

doc/manual/local.mk

@@ -60,17 +60,17 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli
 	@$(call process-includes,$@,$@)
 
 $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
-	@rm -rf $@
-	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix { toplevel = builtins.readFile $<; }'
+	@rm -rf $@ $@.tmp
+	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)'
 	@mv $@.tmp $@
 
-$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
+$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
 	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
-	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
+	$(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
 	@mv $@.tmp $@
 
 $(d)/nix.json: $(bindir)/nix
-	$(trace-gen) $(dummy-env) $(bindir)/nix __dump-args > $@.tmp
+	$(trace-gen) $(dummy-env) $(bindir)/nix __dump-cli > $@.tmp
 	@mv $@.tmp $@
 
 $(d)/conf-file.json: $(bindir)/nix

doc/manual/src/package-management/s3-substituter.md

@@ -1,41 +1,11 @@
 # Serving a Nix store via S3
 
-Nix has built-in support for storing and fetching store paths from
+Nix has [built-in support](@docroot@/command-ref/new-cli/nix3-help-stores.md#s3-binary-cache-store)
+for storing and fetching store paths from
 Amazon S3 and S3-compatible services. This uses the same *binary*
 cache mechanism that Nix usually uses to fetch prebuilt binaries from
 [cache.nixos.org](https://cache.nixos.org/).
 
-The following options can be specified as URL parameters to the S3 URL:
-
-  - `profile`\
-    The name of the AWS configuration profile to use. By default Nix
-    will use the `default` profile.
-
-  - `region`\
-    The region of the S3 bucket. `us–east-1` by default.
-
-    If your bucket is not in `us–east-1`, you should always explicitly
-    specify the region parameter.
-
-  - `endpoint`\
-    The URL to your S3-compatible service, for when not using Amazon S3.
-    Do not specify this value if you're using Amazon S3.
-
-    > **Note**
-    > 
-    > This endpoint must support HTTPS and will use path-based
-    > addressing instead of virtual host based addressing.
-
-  - `scheme`\
-    The scheme used for S3 requests, `https` (default) or `http`. This
-    option allows you to disable HTTPS for binary caches which don't
-    support it.
-
-    > **Note**
-    > 
-    > HTTPS should be used if the cache might contain sensitive
-    > information.
-
 In this example we will use the bucket named `example-nix-cache`.
 
 ## Anonymous Reads to your S3-compatible binary cache

doc/manual/src/release-notes/rl-next.md

@@ -26,4 +26,8 @@
   ```shell-session
   $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^*
   ```
-  provides information about each of its outputs.
+  provides information about each of its outputs.
+
+* The experimental command `nix describe-stores` has been removed.
+
+* Nix stores and their settings are now documented in [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md).

doc/manual/utils.nix

@@ -38,4 +38,46 @@ rec {
 
   filterAttrs = pred: set:
     listToAttrs (concatMap (name: let v = set.${name}; in if pred name v then [(nameValuePair name v)] else []) (attrNames set));
+
+  showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value }:
+    let
+      result = squash ''
+          - ${if useAnchors
+              then ''<span id="conf-${name}">[`${name}`](#conf-${name})</span>''
+              else ''`${name}`''}
+
+          ${indent "  " body}
+        '';
+
+      # separate body to cleanly handle indentation
+      body = ''
+          ${description}
+
+          **Default:** ${showDefault documentDefault defaultValue}
+
+          ${showAliases aliases}
+        '';
+
+      showDefault = documentDefault: defaultValue:
+        if documentDefault then
+          # a StringMap value type is specified as a string, but
+          # this shows the value type. The empty stringmap is `null` in
+          # JSON, but that converts to `{ }` here.
+          if defaultValue == "" || defaultValue == [] || isAttrs defaultValue
+            then "*empty*"
+            else if isBool defaultValue then
+              if defaultValue then "`true`" else "`false`"
+            else "`${toString defaultValue}`"
+        else "*machine-specific*";
+
+      showAliases = aliases:
+          if aliases == [] then "" else
+            "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}";
+
+      indent = prefix: s:
+        concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s));
+
+    in result;
+
+  showSettings = args: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting args) settingsInfo));
 }

src/libcmd/command.hh

@@ -18,6 +18,7 @@ class EvalState;
 struct Pos;
 class Store;
 
+static constexpr Command::Category catHelp = -1;
 static constexpr Command::Category catSecondary = 100;
 static constexpr Command::Category catUtility = 101;
 static constexpr Command::Category catNixInstallation = 102;

src/libcmd/common-eval-args.cc

@@ -136,7 +136,11 @@ MixEvalArgs::MixEvalArgs()
 
     addFlag({
         .longName = "eval-store",
-        .description = "The Nix store to use for evaluations.",
+        .description =
+          R"(
+            The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
+            to use for evaluation, i.e. to store derivations (`.drv` files) and inputs referenced by them.
+          )",
         .category = category,
         .labels = {"store-url"},
         .handler = {&evalStoreUrl},

src/libstore/binary-cache-store.hh

@@ -16,17 +16,33 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
 {
     using StoreConfig::StoreConfig;
 
-    const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', 'gzip', 'zstd', or 'none')"};
-    const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
-    const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
-    const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"};
-    const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"};
+    const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression",
+        "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."};
+
+    const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing",
+        "Whether to write a JSON file that lists the files in each NAR."};
+
+    const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info",
+        R"(
+          Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to
+          fetch debug info on demand
+        )"};
+
+    const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key",
+        "Path to the secret key used to sign the binary cache."};
+
+    const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache",
+        "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."};
+
     const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression",
-        "enable multi-threading compression for NARs, available for xz and zstd only currently"};
+        "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."};
+
     const Setting<int> compressionLevel{(StoreConfig*) this, -1, "compression-level",
-        "specify 'preset level' of compression to be used with NARs: "
-        "meaning and accepted range of values depends on compression method selected, "
-        "other than -1 which we reserve to indicate Nix defaults should be used"};
+        R"(
+          The *preset level* to be used when compressing NARs.
+          The meaning and accepted values depend on the compression method selected.
+          `-1` specifies that the default compression level should be used.
+        )"};
 };
 
 class BinaryCacheStore : public virtual BinaryCacheStoreConfig,

src/libstore/dummy-store.cc

@@ -7,6 +7,13 @@ struct DummyStoreConfig : virtual StoreConfig {
     using StoreConfig::StoreConfig;
 
     const std::string name() override { return "Dummy Store"; }
+
+    std::string doc() override
+    {
+        return
+          #include "dummy-store.md"
+          ;
+    }
 };
 
 struct DummyStore : public virtual DummyStoreConfig, public virtual Store

src/libstore/dummy-store.md

@@ -0,0 +1,13 @@
+R"(
+
+**Store URL format**: `dummy://`
+
+This store type represents a store that contains no store paths and
+cannot be written to. It's useful when you want to use the Nix
+evaluator when no actual Nix store exists, e.g.
+
+```console
+# nix eval --store dummy:// --expr '1 + 2'
+```
+
+)"

src/libstore/globals.hh

@@ -98,7 +98,12 @@ public:
     Path nixDaemonSocketFile;
 
     Setting<std::string> storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store",
-        "The default Nix store to use."};
+        R"(
+          The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
+          to use for most operations.
+          See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md)
+          for supported store types and settings.
+        )"};
 
     Setting<bool> keepFailed{this, false, "keep-failed",
         "Whether to keep temporary directories of failed builds."};
@@ -679,8 +684,9 @@ public:
         Strings{"https://cache.nixos.org/"},
         "substituters",
         R"(
-          A list of URLs of substituters, separated by whitespace. Substituters
-          are tried based on their Priority value, which each substituter can set
+          A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
+          to be used as substituters, separated by whitespace.
+          Substituters are tried based on their Priority value, which each substituter can set
           independently. Lower value means higher priority.
           The default is `https://cache.nixos.org`, with a Priority of 40.
 
@@ -698,7 +704,8 @@ public:
     Setting<StringSet> trustedSubstituters{
         this, {}, "trusted-substituters",
         R"(
-          A list of URLs of substituters, separated by whitespace. These are
+          A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format),
+          separated by whitespace. These are
           not used by default, but can be enabled by users of the Nix daemon
           by specifying `--option substituters urls` on the command
           line. Unprivileged users are only allowed to pass a subset of the

src/libstore/http-binary-cache-store.cc

@@ -12,7 +12,14 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
 {
     using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
 
-    const std::string name() override { return "Http Binary Cache Store"; }
+    const std::string name() override { return "HTTP Binary Cache Store"; }
+
+    std::string doc() override
+    {
+        return
+          #include "http-binary-cache-store.md"
+          ;
+    }
 };
 
 class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore

src/libstore/http-binary-cache-store.md

@@ -0,0 +1,8 @@
+R"(
+
+**Store URL format**: `http://...`, `https://...`
+
+This store allows a binary cache to be accessed via the HTTP
+protocol.
+
+)"