nix hash path, and preperatory refactors

- Proper `parse` and `render` functions for `FileIngestionMethod` and `ContentAddressMethod` Including unit tests! Older methods with same names that operate on on method + algo pair (for old-style `<method>:algo`) are renamed to `*WithAlgo`.) - `nix store add` supports text hashing With functional test ensuring it matches `builtins.toFile`. - Factored-out flags for both - Move all common reusable flags to `libcmd` - They are not part of the *definition* of the CLI infra, just a usag of it. - The `libstore` flag couldn't go in `args.hh` in libutil anyways, would be awkward for it to live alone - Shuffle around `Cmd*` hierarchy so flags for deprecated commands don't end up on the new ones - Split `tests/functional/hash.sh` into `hash-path.sh` and `has-convert.sh` for performance and conceptual clarity.
NixOS · Jan 21, 2024 · ddb5764 · ddb5764
1 parent da1aae2
commit ddb5764
Show file tree

Hide file tree

Showing 21 changed files with 589 additions and 357 deletions.
diff --git a/src/libcmd/misc-store-flags.cc b/src/libcmd/misc-store-flags.cc
@@ -0,0 +1,120 @@
+#include "misc-store-flags.hh"
+
+namespace nix::flag
+{
+
+static void hashFormatCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
+{
+    for (auto & format : hashFormats) {
+        if (hasPrefix(format, prefix)) {
+            completions.add(format);
+        }
+    }
+}
+
+Args::Flag hashFormatWithDefault(std::string && longName, HashFormat * hf)
+{
+    assert(*hf == nix::HashFormat::SRI);
+    return Args::Flag {
+            .longName = std::move(longName),
+            .description = "hash format ('base16', 'nix32', 'base64', 'sri'). Default: 'sri'",
+            .labels = {"hash-format"},
+            .handler = {[hf](std::string s) {
+                *hf = parseHashFormat(s);
+            }},
+            .completer = hashFormatCompleter,
+    };
+}
+
+Args::Flag hashFormatOpt(std::string && longName, std::optional<HashFormat> * ohf)
+{
+    return Args::Flag {
+            .longName = std::move(longName),
+            .description = "hash format ('base16', 'nix32', 'base64', 'sri').",
+            .labels = {"hash-format"},
+            .handler = {[ohf](std::string s) {
+                *ohf = std::optional<HashFormat>{parseHashFormat(s)};
+            }},
+            .completer = hashFormatCompleter,
+    };
+}
+
+static void hashAlgoCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
+{
+    for (auto & algo : hashAlgorithms)
+        if (hasPrefix(algo, prefix))
+            completions.add(algo);
+}
+
+Args::Flag hashAlgo(std::string && longName, HashAlgorithm * ha)
+{
+    return Args::Flag {
+            .longName = std::move(longName),
+            .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')",
+            .labels = {"hash-algo"},
+            .handler = {[ha](std::string s) {
+                *ha = parseHashAlgo(s);
+            }},
+            .completer = hashAlgoCompleter,
+    };
+}
+
+Args::Flag hashAlgoOpt(std::string && longName, std::optional<HashAlgorithm> * oha)
+{
+    return Args::Flag {
+            .longName = std::move(longName),
+            .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.",
+            .labels = {"hash-algo"},
+            .handler = {[oha](std::string s) {
+                *oha = std::optional<HashAlgorithm>{parseHashAlgo(s)};
+            }},
+            .completer = hashAlgoCompleter,
+    };
+}
+
+Args::Flag fileIngestionMethod(FileIngestionMethod * method)
+{
+    return Args::Flag {
+        .longName  = "mode",
+        // FIXME indentation carefully made for context, this is messed up.
+        .description = R"(
+    How to compute the hash of the input.
+    One of:
+
+    - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function.
+
+    - `flat`: Assumes that the input is a single file and directly passes it to the hash function;
+        )",
+        .labels = {"file-ingestion-method"},
+        .handler = {[method](std::string s) {
+            *method = parseFileIngestionMethod(s);
+        }},
+    };
+}
+
+Args::Flag contentAddressMethod(ContentAddressMethod * method)
+{
+    return Args::Flag {
+        .longName  = "mode",
+        // FIXME indentation carefully made for context, this is messed up.
+        .description = R"(
+    How to compute the content-address of the store object.
+    One of:
+
+    - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function.
+
+    - `flat`: Assumes that the input is a single file and directly passes it to the hash function;
+
+    - `text`: Like `flat`, but used for misc things like
+      [derivations](@docroot@/glossary.md#store-derivation) serialized in store object,
+      [`builtins.toFile`](@docroot@/language/builtins.html#builtins-toFile).
+      Less common.
+        )",
+        .labels = {"content-address-method"},
+        .handler = {[method](std::string s) {
+            *method = ContentAddressMethod::parse(s);
+        }},
+    };
+}
+
+}
diff --git a/src/libcmd/misc-store-flags.hh b/src/libcmd/misc-store-flags.hh
@@ -0,0 +1,13 @@
+#include "args.hh"
+#include "content-address.hh"
+
+namespace nix::flag {
+
+Args::Flag hashAlgo(std::string && longName, HashAlgorithm * ha);
+Args::Flag hashAlgoOpt(std::string && longName, std::optional<HashAlgorithm> * oha);
+Args::Flag hashFormatWithDefault(std::string && longName, HashFormat * hf);
+Args::Flag hashFormatOpt(std::string && longName, std::optional<HashFormat> * ohf);
+Args::Flag fileIngestionMethod(FileIngestionMethod * method);
+Args::Flag contentAddressMethod(ContentAddressMethod * method);
+
+}
diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc
@@ -4,7 +4,7 @@
 
 namespace nix {
 
-std::string makeFileIngestionPrefix(FileIngestionMethod m)
+std::string_view makeFileIngestionPrefix(FileIngestionMethod m)
 {
     switch (m) {
     case FileIngestionMethod::Flat:
@@ -16,10 +16,29 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m)
     }
 }
 
-std::string ContentAddressMethod::renderPrefix() const
+std::string_view ContentAddressMethod::render() const
 {
     return std::visit(overloaded {
-        [](TextIngestionMethod) -> std::string { return "text:"; },
+        [](TextIngestionMethod) -> std::string_view { return "text"; },
+        [](FileIngestionMethod m2) {
+             /* Not prefixed for back compat with things that couldn't produce text before. */
+            return renderFileIngestionMethod(m2);
+        },
+    }, raw);
+}
+
+ContentAddressMethod ContentAddressMethod::parse(std::string_view m)
+{
+    if (m == "text")
+        return TextIngestionMethod {};
+    else
+        return parseFileIngestionMethod(m);
+}
+
+std::string_view ContentAddressMethod::renderPrefix() const
+{
+    return std::visit(overloaded {
+        [](TextIngestionMethod) -> std::string_view { return "text:"; },
         [](FileIngestionMethod m2) {
              /* Not prefixed for back compat with things that couldn't produce text before. */
             return makeFileIngestionPrefix(m2);
@@ -38,7 +57,7 @@ ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m)
     return FileIngestionMethod::Flat;
 }
 
-std::string ContentAddressMethod::render(HashAlgorithm ha) const
+std::string ContentAddressMethod::renderWithAlgo(HashAlgorithm ha) const
 {
     return std::visit(overloaded {
         [&](const TextIngestionMethod & th) {
@@ -133,7 +152,7 @@ ContentAddress ContentAddress::parse(std::string_view rawCa)
     };
 }
 
-std::pair<ContentAddressMethod, HashAlgorithm> ContentAddressMethod::parse(std::string_view caMethod)
+std::pair<ContentAddressMethod, HashAlgorithm> ContentAddressMethod::parseWithAlgo(std::string_view caMethod)
 {
     std::string asPrefix = std::string{caMethod} + ":";
     // parseContentAddressMethodPrefix takes its argument by reference
@@ -155,7 +174,7 @@ std::string renderContentAddress(std::optional<ContentAddress> ca)
 
 std::string ContentAddress::printMethodAlgo() const
 {
-    return method.renderPrefix()
+    return std::string { method.renderPrefix() }
         + printHashAlgo(hash.algo);
 }
 

diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh
@@ -36,7 +36,7 @@ struct TextIngestionMethod : std::monostate { };
  * Compute the prefix to the hash algorithm which indicates how the
  * files were ingested.
  */
-std::string makeFileIngestionPrefix(FileIngestionMethod m);
+std::string_view makeFileIngestionPrefix(FileIngestionMethod m);
 
 /**
  * An enumeration of all the ways we can content-address store objects.
@@ -59,6 +59,20 @@ struct ContentAddressMethod
 
     MAKE_WRAPPER_CONSTRUCTOR(ContentAddressMethod);
 
+    /**
+     * Parse a content addressing method (name).
+     *
+     * The inverse of `render`.
+     */
+    static ContentAddressMethod parse(std::string_view rawCaMethod);
+
+    /**
+     * Render a content addressing method (name).
+     *
+     * The inverse of `parse`.
+     */
+    std::string_view render() const;
+
     /**
      * Parse the prefix tag which indicates how the files
      * were ingested, with the fixed output case not prefixed for back
@@ -74,20 +88,20 @@ struct ContentAddressMethod
      *
      * The rough inverse of `parsePrefix()`.
      */
-    std::string renderPrefix() const;
+    std::string_view renderPrefix() const;
 
     /**
      * Parse a content addressing method and hash type.
      */
-    static std::pair<ContentAddressMethod, HashAlgorithm> parse(std::string_view rawCaMethod);
+    static std::pair<ContentAddressMethod, HashAlgorithm> parseWithAlgo(std::string_view rawCaMethod);
 
     /**
      * Render a content addressing method and hash type in a
      * nicer way, prefixing both cases.
      *
      * The rough inverse of `parse()`.
      */
-    std::string render(HashAlgorithm ht) const;
+    std::string renderWithAlgo(HashAlgorithm ht) const;
 
     /**
      * Get the underlying way to content-address file system objects.

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
@@ -400,7 +400,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             logger->startWork();
             auto pathInfo = [&]() {
                 // NB: FramedSource must be out of scope before logger->stopWork();
-                auto [contentAddressMethod, hashAlgo_] = ContentAddressMethod::parse(camStr);
+                auto [contentAddressMethod, hashAlgo_] = ContentAddressMethod::parseWithAlgo(camStr);
                 auto hashAlgo = hashAlgo_; // work around clang bug
                 FramedSource source(from);
                 // TODO these two steps are essentially RemoteStore::addCAToStore. Move it up to Store.

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
@@ -601,7 +601,7 @@ std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs,
             },
             [&](const DerivationOutput::CAFloating & dof) {
                 s += ','; printUnquotedString(s, "");
-                s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo));
+                s += ','; printUnquotedString(s, std::string { dof.method.renderPrefix() } + printHashAlgo(dof.hashAlgo));
                 s += ','; printUnquotedString(s, "");
             },
             [&](const DerivationOutput::Deferred &) {
@@ -612,7 +612,7 @@ std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs,
             [&](const DerivationOutput::Impure & doi) {
                 // FIXME
                 s += ','; printUnquotedString(s, "");
-                s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo));
+                s += ','; printUnquotedString(s, std::string { doi.method.renderPrefix() } + printHashAlgo(doi.hashAlgo));
                 s += ','; printUnquotedString(s, "impure");
             }
         }, i.second.raw);
@@ -984,7 +984,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva
             },
             [&](const DerivationOutput::CAFloating & dof) {
                 out << ""
-                    << (dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo))
+                    << (std::string { dof.method.renderPrefix() } + printHashAlgo(dof.hashAlgo))
                     << "";
             },
             [&](const DerivationOutput::Deferred &) {
@@ -994,7 +994,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva
             },
             [&](const DerivationOutput::Impure & doi) {
                 out << ""
-                    << (doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo))
+                    << (std::string { doi.method.renderPrefix() } + printHashAlgo(doi.hashAlgo))
                     << "impure";
             },
         }, i.second.raw);
@@ -1221,11 +1221,11 @@ nlohmann::json DerivationOutput::toJSON(
             // FIXME print refs?
         },
         [&](const DerivationOutput::CAFloating & dof) {
-            res["hashAlgo"] = dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo);
+            res["hashAlgo"] = std::string { dof.method.renderPrefix() } + printHashAlgo(dof.hashAlgo);
         },
         [&](const DerivationOutput::Deferred &) {},
         [&](const DerivationOutput::Impure & doi) {
-            res["hashAlgo"] = doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo);
+            res["hashAlgo"] = std::string { doi.method.renderPrefix() } + printHashAlgo(doi.hashAlgo);
             res["impure"] = true;
         },
     }, raw);

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
@@ -435,7 +435,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
         conn->to
             << WorkerProto::Op::AddToStore
             << name
-            << caMethod.render(hashAlgo);
+            << caMethod.renderWithAlgo(hashAlgo);
         WorkerProto::write(*this, *conn, references);
         conn->to << repair;
 

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
@@ -544,73 +544,6 @@ nlohmann::json Args::toJSON()
     return res;
 }
 
-static void hashFormatCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
-{
-    for (auto & format : hashFormats) {
-        if (hasPrefix(format, prefix)) {
-            completions.add(format);
-        }
-    }
-}
-
-Args::Flag Args::Flag::mkHashFormatFlagWithDefault(std::string &&longName, HashFormat * hf) {
-    assert(*hf == nix::HashFormat::SRI);
-    return Flag{
-            .longName = std::move(longName),
-            .description = "hash format ('base16', 'nix32', 'base64', 'sri'). Default: 'sri'",
-            .labels = {"hash-format"},
-            .handler = {[hf](std::string s) {
-                *hf = parseHashFormat(s);
-            }},
-            .completer = hashFormatCompleter,
-    };
-}
-
-Args::Flag Args::Flag::mkHashFormatOptFlag(std::string && longName, std::optional<HashFormat> * ohf) {
-    return Flag{
-            .longName = std::move(longName),
-            .description = "hash format ('base16', 'nix32', 'base64', 'sri').",
-            .labels = {"hash-format"},
-            .handler = {[ohf](std::string s) {
-                *ohf = std::optional<HashFormat>{parseHashFormat(s)};
-            }},
-            .completer = hashFormatCompleter,
-    };
-}
-
-static void hashAlgoCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
-{
-    for (auto & algo : hashAlgorithms)
-        if (hasPrefix(algo, prefix))
-            completions.add(algo);
-}
-
-Args::Flag Args::Flag::mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha)
-{
-    return Flag{
-            .longName = std::move(longName),
-            .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')",
-            .labels = {"hash-algo"},
-            .handler = {[ha](std::string s) {
-                *ha = parseHashAlgo(s);
-            }},
-            .completer = hashAlgoCompleter,
-    };
-}
-
-Args::Flag Args::Flag::mkHashAlgoOptFlag(std::string && longName, std::optional<HashAlgorithm> * oha)
-{
-    return Flag{
-            .longName = std::move(longName),
-            .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.",
-            .labels = {"hash-algo"},
-            .handler = {[oha](std::string s) {
-                *oha = std::optional<HashAlgorithm>{parseHashAlgo(s)};
-            }},
-            .completer = hashAlgoCompleter,
-    };
-}
-
 static void _completePath(AddCompletions & completions, std::string_view prefix, bool onlyDirs)
 {
     completions.setType(Completions::Type::Filenames);