diff --git a/.editorconfig b/.editorconfig index 887ecadba59..86360e6582d 100644 --- a/.editorconfig +++ b/.editorconfig @@ -17,7 +17,7 @@ indent_style = space indent_size = 2 # Match c++/shell/perl, set indent to spaces with width of four -[*.{hpp,cc,hh,sh,pl}] +[*.{hpp,cc,hh,sh,pl,xs}] indent_style = space indent_size = 4 diff --git a/.github/labeler.yml b/.github/labeler.yml index 12120bdb367..7544f07a610 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -20,4 +20,4 @@ # Unit tests - src/*/tests/**/* # Functional and integration tests - - tests/**/* + - tests/functional/**/* diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 12c60c64975..975c90b9178 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v1.4.0 + uses: zeebe-io/backport-action@v2.1.1 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c9c24dade2..afe4dc2e35e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,6 +101,9 @@ jobs: docker_push_image: needs: [check_secrets, tests] + permissions: + contents: read + packages: write if: >- github.event_name == 'push' && github.ref_name == 'master' && @@ -126,6 +129,9 @@ jobs: - run: docker load -i ./result/image.tar.gz - run: docker tag nix:$NIX_VERSION nixos/nix:$NIX_VERSION - run: docker tag nix:$NIX_VERSION nixos/nix:master + # We'll deploy the newly built image to both Docker Hub and Github Container Registry. + # + # Push to Docker Hub first - name: Login to Docker Hub uses: docker/login-action@v3 with: @@ -133,3 +139,20 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - run: docker push nixos/nix:$NIX_VERSION - run: docker push nixos/nix:master + # Push to GitHub Container Registry as well + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Push image + run: | + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/nix + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + + docker tag nix:$NIX_VERSION $IMAGE_ID:$NIX_VERSION + docker tag nix:$NIX_VERSION $IMAGE_ID:master + docker push $IMAGE_ID:$NIX_VERSION + docker push $IMAGE_ID:master diff --git a/.gitignore b/.gitignore index bf4f33564ad..23ba1077f68 100644 --- a/.gitignore +++ b/.gitignore @@ -41,14 +41,14 @@ perl/Makefile.config /src/libexpr/parser-tab.hh /src/libexpr/parser-tab.output /src/libexpr/nix.tbl -/src/libexpr/tests/libnixexpr-tests +/tests/unit/libexpr/libnixexpr-tests # /src/libstore/ *.gen.* -/src/libstore/tests/libnixstore-tests +/tests/unit/libstore/libnixstore-tests # /src/libutil/ -/src/libutil/tests/libnixutil-tests +/tests/unit/libutil/libnixutil-tests /src/nix/nix @@ -79,24 +79,24 @@ perl/Makefile.config /src/build-remote/build-remote -# /tests/ -/tests/test-tmp -/tests/common/vars-and-functions.sh -/tests/result* -/tests/restricted-innocent -/tests/shell -/tests/shell.drv -/tests/config.nix -/tests/ca/config.nix -/tests/dyn-drv/config.nix -/tests/repl-result-out -/tests/test-libstoreconsumer/test-libstoreconsumer - -# /tests/lang/ -/tests/lang/*.out -/tests/lang/*.out.xml -/tests/lang/*.err -/tests/lang/*.ast +# /tests/functional/ +/tests/functional/test-tmp +/tests/functional/common/vars-and-functions.sh +/tests/functional/result* +/tests/functional/restricted-innocent +/tests/functional/shell +/tests/functional/shell.drv +/tests/functional/config.nix +/tests/functional/ca/config.nix +/tests/functional/dyn-drv/config.nix +/tests/functional/repl-result-out +/tests/functional/test-libstoreconsumer/test-libstoreconsumer + +# /tests/functional/lang/ +/tests/functional/lang/*.out +/tests/functional/lang/*.out.xml +/tests/functional/lang/*.err +/tests/functional/lang/*.ast /perl/lib/Nix/Config.pm /perl/lib/Nix/Store.cc @@ -138,7 +138,9 @@ nix-rust/target result +# IDE .vscode/ +.idea/ # clangd and possibly more .cache/ diff --git a/.version b/.version index cf8690732fe..6ef6ef5ac16 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.18.0 +2.19.3 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index facbf0eb051..ffcc0268f56 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,25 +24,33 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). ## Making changes to Nix -1. Check for [pull requests](https://github.com/NixOS/nix/pulls) that might already cover the contribution you are about to make. - There are many open pull requests that might already do what you intent to work on. - You can use [labels](https://github.com/NixOS/nix/labels) to filter for relevant topics. +1. Search for related issues that cover what you're going to work on. + It could help to mention there that you will work on the issue. + + Issues labeled [good first issue](https://github.com/NixOS/nix/labels/good%20first%20issue) should be relatively easy to fix and are likely to get merged quickly. + Pull requests addressing issues labeled [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) or [RFC](https://github.com/NixOS/nix/labels/RFC) are especially welcomed by maintainers and will receive prioritised review. -2. Search for related issues that cover what you're going to work on. It could help to mention there that you will work on the issue. + If you are proficient with C++, addressing one of the [popular issues](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc) will be highly appreciated by maintainers and Nix users all over the world. + For far-reaching changes, please investigate possible blockers and design implications, and coordinate with maintainers before investing too much time in writing code that may not end up getting merged. - Issues labeled [good first issue](https://github.com/NixOS/nix/labels/good-first-issue) should be relatively easy to fix and are likely to get merged quickly. - Pull requests addressing issues labeled [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) are especially welcomed by maintainers and will receive prioritised review. + If there is no relevant issue yet and you're not sure whether your change is likely to be accepted, [open an issue](https://github.com/NixOS/nix/issues/new/choose) yourself. + +2. Check for [pull requests](https://github.com/NixOS/nix/pulls) that might already cover the contribution you are about to make. + There are many open pull requests that might already do what you intend to work on. + You can use [labels](https://github.com/NixOS/nix/labels) to filter for relevant topics. 3. Check the [Nix reference manual](https://nixos.org/manual/nix/unstable/contributing/hacking.html) for information on building Nix and running its tests. For contributions to the command line interface, please check the [CLI guidelines](https://nixos.org/manual/nix/unstable/contributing/cli-guideline.html). -4. Make your changes! +4. Make your change! 5. [Create a pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request) for your changes. - * Link related issues in your pull request to inform interested parties and future contributors about your change. + * Clearly explain the problem that you're solving. + + Link related issues to inform interested parties and future contributors about your change. + If your pull request closes one or multiple issues, mention that in the description using `Closes: #`, as it will then happen automatically when your change is merged. * Make sure to have [a clean history of commits on your branch by using rebase](https://www.digitalocean.com/community/tutorials/how-to-rebase-and-update-a-pull-request). - If your pull request closes one or multiple issues, note that in the description using `Closes: #`, as it will then happen automatically when your change is merged. * [Mark the pull request as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you're not done with the changes. 6. Do not expect your pull request to be reviewed immediately. @@ -52,7 +60,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). - [ ] Fixes an [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) issue - [ ] Tests, as appropriate: - - Functional tests – [`tests/**.sh`](./tests) + - Functional tests – [`tests/functional/**.sh`](./tests/functional) - Unit tests – [`src/*/tests`](./src/) - Integration tests – [`tests/nixos/*`](./tests/nixos) - [ ] User documentation in the [manual](..doc/manual/src) diff --git a/Makefile b/Makefile index 31b54b93d20..8e0719b86a8 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,7 @@ +-include Makefile.config +clean-files += Makefile.config + +ifeq ($(ENABLE_BUILD), yes) makefiles = \ mk/precompiled-headers.mk \ local.mk \ @@ -18,19 +22,25 @@ makefiles = \ misc/upstart/local.mk \ doc/manual/local.mk \ doc/internal-api/local.mk +endif --include Makefile.config +ifeq ($(ENABLE_BUILD)_$(ENABLE_TESTS), yes_yes) +makefiles += \ + tests/unit/libutil/local.mk \ + tests/unit/libutil-support/local.mk \ + tests/unit/libstore/local.mk \ + tests/unit/libstore-support/local.mk \ + tests/unit/libexpr/local.mk \ + tests/unit/libexpr-support/local.mk +endif -ifeq ($(tests), yes) +ifeq ($(ENABLE_TESTS), yes) makefiles += \ - src/libutil/tests/local.mk \ - src/libstore/tests/local.mk \ - src/libexpr/tests/local.mk \ - tests/local.mk \ - tests/ca/local.mk \ - tests/dyn-drv/local.mk \ - tests/test-libstoreconsumer/local.mk \ - tests/plugins/local.mk + tests/functional/local.mk \ + tests/functional/ca/local.mk \ + tests/functional/dyn-drv/local.mk \ + tests/functional/test-libstoreconsumer/local.mk \ + tests/functional/plugins/local.mk else makefiles += \ mk/disable-tests.mk diff --git a/Makefile.config.in b/Makefile.config.in index 707cfe0e398..1482db81f53 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -28,6 +28,8 @@ SODIUM_LIBS = @SODIUM_LIBS@ SQLITE3_LIBS = @SQLITE3_LIBS@ bash = @bash@ bindir = @bindir@ +checkbindir = @checkbindir@ +checklibdir = @checklibdir@ datadir = @datadir@ datarootdir = @datarootdir@ doc_generate = @doc_generate@ @@ -46,5 +48,7 @@ sandbox_shell = @sandbox_shell@ storedir = @storedir@ sysconfdir = @sysconfdir@ system = @system@ -tests = @tests@ +ENABLE_BUILD = @ENABLE_BUILD@ +ENABLE_TESTS = @ENABLE_TESTS@ +INSTALL_UNIT_TESTS = @INSTALL_UNIT_TESTS@ internal_api_docs = @internal_api_docs@ diff --git a/README.md b/README.md index 85b0902b107..e1cace3b4d5 100644 --- a/README.md +++ b/README.md @@ -7,21 +7,20 @@ Nix is a powerful package manager for Linux and other Unix systems that makes pa management reliable and reproducible. Please refer to the [Nix manual](https://nixos.org/nix/manual) for more details. -## Installation +## Installation and first steps -On Linux and macOS the easiest way to install Nix is to run the following shell command -(as a user other than root): +Visit [nix.dev](https://nix.dev) for [installation instructions](https://nix.dev/tutorials/install-nix) and [beginner tutorials](https://nix.dev/tutorials/first-steps). -```console -$ curl -L https://nixos.org/nix/install | sh -``` - -Information on additional installation methods is available on the [Nix download page](https://nixos.org/download.html). +Full reference documentation can be found in the [Nix manual](https://nixos.org/nix/manual). ## Building And Developing See our [Hacking guide](https://nixos.org/manual/nix/unstable/contributing/hacking.html) in our manual for instruction on how to -to set up a development environment and build Nix from source. + set up a development environment and build Nix from source. + +## Contributing + +Check the [contributing guide](./CONTRIBUTING.md) if you want to get involved with developing Nix. ## Additional Resources @@ -29,7 +28,6 @@ to set up a development environment and build Nix from source. - [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix) - [NixOS Discourse](https://discourse.nixos.org/) - [Matrix - #nix:nixos.org](https://matrix.to/#/#nix:nixos.org) -- [IRC - #nixos on libera.chat](irc://irc.libera.chat/#nixos) ## License diff --git a/boehmgc-coroutine-sp-fallback.diff b/boehmgc-coroutine-sp-fallback.diff index 5066d8278ac..2afbe96712b 100644 --- a/boehmgc-coroutine-sp-fallback.diff +++ b/boehmgc-coroutine-sp-fallback.diff @@ -59,12 +59,18 @@ index b5d71e62..aed7b0bf 100644 GC_bool found_me = FALSE; size_t nthreads = 0; int i; -@@ -851,6 +853,31 @@ GC_INNER void GC_push_all_stacks(void) +@@ -851,6 +853,37 @@ GC_INNER void GC_push_all_stacks(void) hi = p->altstack + p->altstack_size; /* FIXME: Need to scan the normal stack too, but how ? */ /* FIXME: Assume stack grows down */ + } else { -+ if (pthread_getattr_np(p->id, &pattr)) { ++#ifdef HAVE_PTHREAD_ATTR_GET_NP ++ if (!pthread_attr_init(&pattr) ++ || !pthread_attr_get_np(p->id, &pattr)) ++#else /* HAVE_PTHREAD_GETATTR_NP */ ++ if (pthread_getattr_np(p->id, &pattr)) ++#endif ++ { + ABORT("GC_push_all_stacks: pthread_getattr_np failed!"); + } + if (pthread_attr_getstacksize(&pattr, &stack_limit)) { diff --git a/bootstrap.sh b/bootstrap.sh deleted file mode 100755 index e3e25935167..00000000000 --- a/bootstrap.sh +++ /dev/null @@ -1,4 +0,0 @@ -#! /bin/sh -e -rm -f aclocal.m4 -mkdir -p config -exec autoreconf -vfi diff --git a/configure.ac b/configure.ac index 6d78237f043..281ba2c32e2 100644 --- a/configure.ac +++ b/configure.ac @@ -68,6 +68,9 @@ case "$host_os" in esac +ENSURE_NO_GCC_BUG_80431 + + # Check for pubsetbuf. AC_MSG_CHECKING([for pubsetbuf]) AC_LANG_PUSH(C++) @@ -152,12 +155,29 @@ if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then LDFLAGS="-latomic $LDFLAGS" fi +# Running the functional tests without building Nix is useful for testing +# different pre-built versions of Nix against each other. +AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]), + ENABLE_BUILD=$enableval, ENABLE_BUILD=yes) +AC_SUBST(ENABLE_BUILD) # Building without tests is useful for bootstrapping with a smaller footprint # or running the tests in a separate derivation. Otherwise, we do compile and # run them. AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[Do not build the tests]), - tests=$enableval, tests=yes) -AC_SUBST(tests) + ENABLE_TESTS=$enableval, ENABLE_TESTS=yes) +AC_SUBST(ENABLE_TESTS) + +AC_ARG_ENABLE(install-unit-tests, AS_HELP_STRING([--enable-install-unit-tests],[Install the unit tests for running later (default no)]), + INSTALL_UNIT_TESTS=$enableval, INSTALL_UNIT_TESTS=no) +AC_SUBST(INSTALL_UNIT_TESTS) + +AC_ARG_WITH(check-bin-dir, AS_HELP_STRING([--with-check-bin-dir=PATH],[path to install unit tests for running later (defaults to $libexecdir/nix)]), + checkbindir=$withval, checkbindir=$libexecdir/nix) +AC_SUBST(checkbindir) + +AC_ARG_WITH(check-lib-dir, AS_HELP_STRING([--with-check-lib-dir=PATH],[path to install unit tests for running later (defaults to $libdir)]), + checklibdir=$withval, checklibdir=$libdir) +AC_SUBST(checklibdir) # Building without API docs is the default as Nix' C++ interfaces are internal and unstable. AC_ARG_ENABLE(internal_api_docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]), @@ -289,7 +309,7 @@ if test "$gc" = yes; then fi -if test "$tests" = yes; then +if test "$ENABLE_TESTS" = yes; then # Look for gtest. PKG_CHECK_MODULES([GTEST], [gtest_main]) diff --git a/doc/internal-api/doxygen.cfg.in b/doc/internal-api/doxygen.cfg.in index 8f526536d53..ad5af97e6fe 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/doc/internal-api/doxygen.cfg.in @@ -39,21 +39,42 @@ INPUT = \ src/libcmd \ src/libexpr \ src/libexpr/flake \ - src/libexpr/tests \ - src/libexpr/tests/value \ + tests/unit/libexpr \ + tests/unit/libexpr/value \ + tests/unit/libexpr/test \ + tests/unit/libexpr/test/value \ src/libexpr/value \ src/libfetchers \ src/libmain \ src/libstore \ src/libstore/build \ src/libstore/builtins \ - src/libstore/tests \ + tests/unit/libstore \ + tests/unit/libstore/test \ src/libutil \ - src/libutil/tests \ + tests/unit/libutil \ + tests/unit/libutil/test \ src/nix \ src/nix-env \ src/nix-store +# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names +# in the source code. If set to NO, only conditional compilation will be +# performed. Macro expansion can be done in a controlled way by setting +# EXPAND_ONLY_PREDEF to YES. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +MACRO_EXPANSION = YES + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then +# the macro expansion is limited to the macros specified with the PREDEFINED and +# EXPAND_AS_DEFINED tags. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +EXPAND_ONLY_PREDEF = YES + # The INCLUDE_PATH tag can be used to specify one or more directories that # contain include files that are not input files but should be processed by the # preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of @@ -61,3 +82,16 @@ INPUT = \ # This tag requires that the tag SEARCH_INCLUDES is set to YES. INCLUDE_PATH = @RAPIDCHECK_HEADERS@ + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this +# tag can be used to specify a list of macro names that should be expanded. The +# macro definition that is found in the sources will be used. Use the PREDEFINED +# tag if you want to use a different macro definition that overrules the +# definition found in the source code. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +EXPAND_AS_DEFINED = \ + DECLARE_COMMON_SERIALISER \ + DECLARE_WORKER_SERIALISER \ + DECLARE_SERVE_SERIALISER \ + LENGTH_PREFIXED_PROTO_HELPER diff --git a/doc/manual/_redirects b/doc/manual/_redirects new file mode 100644 index 00000000000..4ea289d86ec --- /dev/null +++ b/doc/manual/_redirects @@ -0,0 +1,30 @@ +# redirect rules for paths (server-side) to prevent link rot. +# see ./redirects.js for redirects based on URL fragments (client-side) +# +# concrete user story this supports: +# - user finds URL to the manual for Nix x.y +# - Nix x.z (z > y) is the most recent release +# - updating the version in the URL will show the right thing +# +# format documentation: +# - https://docs.netlify.com/routing/redirects/#syntax-for-the-redirects-file +# - https://docs.netlify.com/routing/redirects/redirect-options/ +# +# conventions: +# - always force (!) since this allows re-using file names +# - group related paths to ease readability +# - always append new redirects to the end of the file +# - redirects that should have been there but are missing can be inserted where they belong + +/expressions/expression-language /language/ 301! +/expressions/language-values /language/values 301! +/expressions/language-constructs /language/constructs 301! +/expressions/language-operators /language/operators 301! +/expressions/* /language/:splat 301! + +/package-management/basic-package-mgmt /command-ref/nix-env 301! + +/package-management/channels* /command-ref/nix-channel 301! + +/package-management/s3-substituter* /command-ref/new-cli/nix3-help-stores#s3-binary-cache-store 301! + diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index bc4d2c5bcc0..14136016dfc 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -1,11 +1,12 @@ let inherit (builtins) - attrNames attrValues fromJSON listToAttrs mapAttrs + attrNames attrValues fromJSON listToAttrs mapAttrs groupBy concatStringsSep concatMap length lessThan replaceStrings sort; - inherit (import ./utils.nix) concatStrings optionalString filterAttrs trim squash unique showSettings; + inherit (import ) attrsToList concatStrings optionalString filterAttrs trim squash unique; + showStoreDocs = import ./generate-store-info.nix; in -commandDump: +inlineHTML: commandDump: let @@ -30,7 +31,7 @@ let ${maybeSubcommands} - ${maybeDocumentation} + ${maybeStoreDocs} ${maybeOptions} ''; @@ -40,15 +41,15 @@ let showArgument = arg: "*${arg.label}*" + optionalString (! arg ? arity) "..."; arguments = concatStringsSep " " (map showArgument args); in '' - `${command}` [*option*...] ${arguments} + `${command}` [*option*...] ${arguments} ''; maybeSubcommands = optionalString (details ? commands && details.commands != {}) - '' - where *subcommand* is one of the following: + '' + where *subcommand* is one of the following: - ${subcommands} - ''; + ${subcommands} + ''; subcommands = if length categories > 1 then listCategories @@ -70,40 +71,57 @@ let * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description} ''; - maybeDocumentation = optionalString - (details ? doc) - (replaceStrings ["@stores@"] [storeDocs] details.doc); - - maybeOptions = optionalString (details.flags != {}) '' + # FIXME: this is a hack. + # store parameters should not be part of command documentation to begin + # with, but instead be rendered on separate pages. + maybeStoreDocs = optionalString (details ? doc) + (replaceStrings [ "@stores@" ] [ (showStoreDocs inlineHTML commandInfo.stores) ] details.doc); + + maybeOptions = let + allVisibleOptions = filterAttrs + (_: o: ! o.hiddenCategory) + (details.flags // toplevel.flags); + in optionalString (allVisibleOptions != {}) '' # Options - ${showOptions details.flags toplevel.flags} + ${showOptions inlineHTML allVisibleOptions} + + > **Note** + > + > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. ''; - showOptions = options: commonOptions: + showOptions = inlineHTML: allOptions: let - allOptions = options // commonOptions; - showCategory = cat: '' - ${optionalString (cat != "") "**${cat}:**"} + showCategory = cat: opts: '' + ${optionalString (cat != "") "## ${cat}"} - ${listOptions (filterAttrs (n: v: v.category == cat) allOptions)} + ${concatStringsSep "\n" (attrValues (mapAttrs showOption opts))} ''; - listOptions = opts: concatStringsSep "\n" (attrValues (mapAttrs showOption opts)); showOption = name: option: let + result = trim '' + - ${item} + + ${option.description} + ''; + item = if inlineHTML + then ''[`--${name}`](#opt-${name}) ${shortName} ${labels}'' + else "`--${name}` ${shortName} ${labels}"; shortName = optionalString (option ? shortName) ("/ `-${option.shortName}`"); labels = optionalString (option ? labels) (concatStringsSep " " (map (s: "*${s}*") option.labels)); - in trim '' - - [`--${name}`](#opt-${name}) ${shortName} ${labels} - - ${option.description} - ''; - categories = sort lessThan (unique (map (cmd: cmd.category) (attrValues allOptions))); - in concatStrings (map showCategory categories); + in result; + categories = mapAttrs + # Convert each group from a list of key-value pairs back to an attrset + (_: listToAttrs) + (groupBy + (cmd: cmd.value.category) + (attrsToList allOptions)); + in concatStrings (attrValues (mapAttrs showCategory categories)); in squash result; appendName = filename: name: (if filename == "nix" then "nix3" else filename) + "-" + name; @@ -135,35 +153,4 @@ let " - [${page.command}](command-ref/new-cli/${page.name})"; in concatStringsSep "\n" (map showEntry manpages) + "\n"; - storeDocs = - let - showStore = name: { settings, doc, experimentalFeature }: - let - experimentalFeatureNote = optionalString (experimentalFeature != null) '' - > **Warning** - > This store is part of an - > [experimental feature](@docroot@/contributing/experimental-features.md). - - To use this store, you need to make sure the corresponding experimental feature, - [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), - is enabled. - For example, include the following in [`nix.conf`](#): - - ``` - extra-experimental-features = ${experimentalFeature} - ``` - ''; - in '' - ## ${name} - - ${doc} - - ${experimentalFeatureNote} - - **Settings**: - - ${showSettings { useAnchors = false; } settings} - ''; - in concatStrings (attrValues (mapAttrs showStore commandInfo.stores)); - in (listToAttrs manpages) // { "SUMMARY.md" = tableOfContents; } diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix new file mode 100644 index 00000000000..8736bb7933f --- /dev/null +++ b/doc/manual/generate-settings.nix @@ -0,0 +1,66 @@ +let + inherit (builtins) attrValues concatStringsSep isAttrs isBool mapAttrs; + inherit (import ./utils.nix) concatStrings indent optionalString squash; +in + +# `inlineHTML` is a hack to accommodate inconsistent output from `lowdown` +{ prefix, inlineHTML ? true }: settingsInfo: + +let + + showSetting = prefix: setting: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }: + let + result = squash '' + - ${item} + + ${indent " " body} + ''; + item = if inlineHTML + then ''[`${setting}`](#${prefix}-${setting})'' + else "`${setting}`"; + # separate body to cleanly handle indentation + body = '' + ${description} + + ${experimentalFeatureNote} + + **Default:** ${showDefault documentDefault defaultValue} + + ${showAliases aliases} + ''; + + experimentalFeatureNote = optionalString (experimentalFeature != null) '' + > **Warning** + > This setting is part of an + > [experimental feature](@docroot@/contributing/experimental-features.md). + + To change this setting, you need to make sure the corresponding experimental feature, + [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), + is enabled. + For example, include the following in [`nix.conf`](#): + + ``` + extra-experimental-features = ${experimentalFeature} + ${setting} = ... + ``` + ''; + + showDefault = documentDefault: defaultValue: + if documentDefault then + # a StringMap value type is specified as a string, but + # this shows the value type. The empty stringmap is `null` in + # JSON, but that converts to `{ }` here. + if defaultValue == "" || defaultValue == [] || isAttrs defaultValue + then "*empty*" + else if isBool defaultValue then + if defaultValue then "`true`" else "`false`" + else "`${toString defaultValue}`" + else "*machine-specific*"; + + showAliases = aliases: + optionalString (aliases != []) + "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}"; + + in result; + +in concatStrings (attrValues (mapAttrs (showSetting prefix) settingsInfo)) diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix new file mode 100644 index 00000000000..36215aadf73 --- /dev/null +++ b/doc/manual/generate-store-info.nix @@ -0,0 +1,45 @@ +let + inherit (builtins) attrValues mapAttrs; + inherit (import ./utils.nix) concatStrings optionalString; + showSettings = import ./generate-settings.nix; +in + +inlineHTML: storesInfo: + +let + + showStore = name: { settings, doc, experimentalFeature }: + let + + result = '' + ## ${name} + + ${doc} + + ${experimentalFeatureNote} + + ### Settings + + ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings} + ''; + + # markdown doesn't like spaces in URLs + slug = builtins.replaceStrings [ " " ] [ "-" ] name; + + experimentalFeatureNote = optionalString (experimentalFeature != null) '' + > **Warning** + > This store is part of an + > [experimental feature](@docroot@/contributing/experimental-features.md). + + To use this store, you need to make sure the corresponding experimental feature, + [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), + is enabled. + For example, include the following in [`nix.conf`](#): + + ``` + extra-experimental-features = ${experimentalFeature} + ``` + ''; + in result; + +in concatStrings (attrValues (mapAttrs showStore storesInfo)) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index abdfd6a622b..db3daf2524a 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -32,7 +32,7 @@ dummy-env = env -i \ NIX_STATE_DIR=/dummy \ NIX_CONFIG='cores = 0' -nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw +nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw # re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution define process-includes @@ -96,14 +96,14 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/sr @cp $< $@ @$(call process-includes,$@,$@) -$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(bindir)/nix +$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(bindir)/nix @rm -rf $@ $@.tmp - $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)' + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)' @mv $@.tmp $@ -$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix +$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "conf"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @mv $@.tmp $@ $(d)/nix.json: $(bindir)/nix @@ -125,7 +125,7 @@ $(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $ @mv $@.tmp $@ $(d)/xp-features.json: $(bindir)/nix - $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp + $(trace-gen) $(dummy-env) $(bindir)/nix __dump-xp-features > $@.tmp @mv $@.tmp $@ $(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix @@ -141,7 +141,7 @@ $(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin @mv $@.tmp $@ $(d)/language.json: $(bindir)/nix - $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-language > $@.tmp + $(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp @mv $@.tmp $@ # Generate the HTML manual. @@ -173,6 +173,10 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ +# the `! -name 'contributing.md'` filter excludes the one place where +# `@docroot@` is to be preserved for documenting the mechanism +# FIXME: maybe contributing guides should live right next to the code +# instead of in the manual $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(trace-gen) \ tmp="$$(mktemp -d)"; \ @@ -180,7 +184,7 @@ $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/ find "$$tmp" -name '*.md' | while read -r file; do \ $(call process-includes,$$file,$$file); \ done; \ - find "$$tmp" -name '*.md' | while read -r file; do \ + find "$$tmp" -name '*.md' ! -name 'documentation.md' | while read -r file; do \ docroot="$$(realpath --relative-to="$$(dirname "$$file")" $$tmp/manual/src)"; \ sed -i "s,@docroot@,$$docroot,g" "$$file"; \ done; \ diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index b43622ed6a4..d04f32b49bf 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -1,7 +1,9 @@ -// redirect rules for anchors ensure backwards compatibility of URLs. -// this must be done on the client side, as web servers do not see the anchor part of the URL. +// redirect rules for URL fragments (client-side) to prevent link rot. +// this must be done on the client side, as web servers do not see the fragment part of the URL. +// it will only work with JavaScript enabled in the browser, but this is the best we can do here. +// see ./_redirects for path redirects (client-side) -// redirections are declared as follows: +// redirects are declared as follows: // each entry has as its key a path matching the requested URL path, relative to the mdBook document root. // // IMPORTANT: it must specify the full path with file name and suffix @@ -19,6 +21,7 @@ const redirects = { "chap-distributed-builds": "advanced-topics/distributed-builds.html", "chap-post-build-hook": "advanced-topics/post-build-hook.html", "chap-post-build-hook-caveats": "advanced-topics/post-build-hook.html#implementation-caveats", + "chap-writing-nix-expressions": "language/index.html", "part-command-ref": "command-ref/command-ref.html", "conf-allow-import-from-derivation": "command-ref/conf-file.html#conf-allow-import-from-derivation", "conf-allow-new-privileges": "command-ref/conf-file.html#conf-allow-new-privileges", @@ -336,14 +339,13 @@ const redirects = { "simple-values": "#primitives", "lists": "#list", "strings": "#string", - "lists": "#list", "attribute-sets": "#attribute-set", }, "installation/installing-binary.html": { "linux": "uninstall.html#linux", "macos": "uninstall.html#macos", "uninstalling": "uninstall.html", - } + }, "contributing/hacking.html": { "nix-with-flakes": "#building-nix-with-flakes", "classic-nix": "#building-nix", @@ -355,6 +357,7 @@ const redirects = { "installer-tests": "testing.html#installer-tests", "one-time-setup": "testing.html#one-time-setup", "using-the-ci-generated-installer-for-manual-testing": "testing.html#using-the-ci-generated-installer-for-manual-testing", + "characterization-testing": "#characterisation-testing-unit", } }; diff --git a/doc/manual/rl-next/allowed-uris-can-now-match-whole-schemes.md b/doc/manual/rl-next/allowed-uris-can-now-match-whole-schemes.md new file mode 100644 index 00000000000..3cf75a61253 --- /dev/null +++ b/doc/manual/rl-next/allowed-uris-can-now-match-whole-schemes.md @@ -0,0 +1,7 @@ +--- +synopsis: Option `allowed-uris` can now match whole schemes in URIs without slashes +prs: 9547 +--- + +If a scheme, such as `github:` is specified in the `allowed-uris` option, all URIs starting with `github:` are allowed. +Previously this only worked for schemes whose URIs used the `://` syntax. diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index c2318651143..f4d4c1f4150 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -16,26 +16,31 @@ - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) - [Uninstalling Nix](installation/uninstall.md) -- [Package Management](package-management/package-management.md) - - [Basic Package Management](package-management/basic-package-mgmt.md) - - [Profiles](package-management/profiles.md) - - [Garbage Collection](package-management/garbage-collection.md) - - [Garbage Collector Roots](package-management/garbage-collector-roots.md) - - [Sharing Packages Between Machines](package-management/sharing-packages.md) - - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) - - [Copying Closures via SSH](package-management/copy-closure.md) - - [Serving a Nix store via SSH](package-management/ssh-substituter.md) - - [Serving a Nix store via S3](package-management/s3-substituter.md) +- [Nix Store](store/index.md) + - [File System Object](store/file-system-object.md) + - [Store Object](store/store-object.md) + - [Store Path](store/store-path.md) - [Nix Language](language/index.md) - [Data Types](language/values.md) - [Language Constructs](language/constructs.md) - [String interpolation](language/string-interpolation.md) + - [Lookup path](language/constructs/lookup-path.md) - [Operators](language/operators.md) - [Derivations](language/derivations.md) - [Advanced Attributes](language/advanced-attributes.md) + - [Import From Derivation](language/import-from-derivation.md) - [Built-in Constants](language/builtin-constants.md) - [Built-in Functions](language/builtins.md) +- [Package Management](package-management/package-management.md) + - [Profiles](package-management/profiles.md) + - [Garbage Collection](package-management/garbage-collection.md) + - [Garbage Collector Roots](package-management/garbage-collector-roots.md) - [Advanced Topics](advanced-topics/advanced-topics.md) + - [Sharing Packages Between Machines](package-management/sharing-packages.md) + - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) + - [Copying Closures via SSH](package-management/copy-closure.md) + - [Serving a Nix store via SSH](package-management/ssh-substituter.md) + - [Serving a Nix store via S3](package-management/s3-substituter.md) - [Remote Builds](advanced-topics/distributed-builds.md) - [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md) - [Verifying Build Reproducibility](advanced-topics/diff-hook.md) @@ -97,7 +102,6 @@ - [Channels](command-ref/files/channels.md) - [Default Nix expression](command-ref/files/default-nix-expression.md) - [Architecture and Design](architecture/architecture.md) - - [File System Object](architecture/file-system-object.md) - [Protocols](protocols/protocols.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md) @@ -105,11 +109,12 @@ - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) - [Testing](contributing/testing.md) + - [Documentation](contributing/documentation.md) - [Experimental Features](contributing/experimental-features.md) - [CLI guideline](contributing/cli-guideline.md) - [C++ style guide](contributing/cxx.md) - [Release Notes](release-notes/release-notes.md) - - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md) - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md) - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md) - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md) diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md index 73a113d352b..507c5ecb769 100644 --- a/doc/manual/src/advanced-topics/distributed-builds.md +++ b/doc/manual/src/advanced-topics/distributed-builds.md @@ -12,14 +12,14 @@ machine is accessible via SSH and that it has Nix installed. You can test whether connecting to the remote Nix instance works, e.g. ```console -$ nix store ping --store ssh://mac +$ nix store info --store ssh://mac ``` will try to connect to the machine named `mac`. It is possible to specify an SSH identity file as part of the remote store URI, e.g. ```console -$ nix store ping --store ssh://mac?ssh-key=/home/alice/my-key +$ nix store info --store ssh://mac?ssh-key=/home/alice/my-key ``` Since builds should be non-interactive, the key should not have a diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md index a251dec4886..3c1cc9b369a 100644 --- a/doc/manual/src/advanced-topics/post-build-hook.md +++ b/doc/manual/src/advanced-topics/post-build-hook.md @@ -17,9 +17,8 @@ the build loop. # Prerequisites -This tutorial assumes you have [configured an S3-compatible binary -cache](../package-management/s3-substituter.md), and that the `root` -user's default AWS profile can upload to the bucket. +This tutorial assumes you have configured an [S3-compatible binary cache](@docroot@/command-ref/new-cli/nix3-help-stores.md#s3-binary-cache-store) as a [substituter](../command-ref/conf-file.md#conf-substituters), +and that the `root` user's default AWS profile can upload to the bucket. # Set up a Signing Key @@ -69,6 +68,8 @@ exec nix copy --to "s3://example-nix-cache" $OUT_PATHS > store sign`. Nix guarantees the paths will not contain any spaces, > however a store path might contain glob characters. The `set -f` > disables globbing in the shell. +> If you want to upload the `.drv` file too, the `$DRV_PATH` variable +> is also defined for the script and works just like `$OUT_PATHS`. Then make sure the hook program is executable by the `root` user: diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 9e969972e7f..79429508f64 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -59,10 +59,11 @@ The [Nix language](../language/index.md) evaluator transforms Nix expressions in The command line interface and Nix expressions are what users deal with most. > **Note** +> > The Nix language itself does not have a notion of *packages* or *configurations*. > As far as we are concerned here, the inputs and results of a build plan are just data. -Underlying the command line interface and the Nix language evaluator is the [Nix store](../glossary.md#gloss-store), a mechanism to keep track of build plans, data, and references between them. +Underlying the command line interface and the Nix language evaluator is the [Nix store](../store/index.md), a mechanism to keep track of build plans, data, and references between them. It can also execute build plans to produce new data, which are made available to the operating system as files. A build plan itself is a series of *build tasks*, together with their build inputs. diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index b4a9bb2a975..34e0dbfbd91 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -2,109 +2,124 @@ Most Nix commands interpret the following environment variables: - - [`IN_NIX_SHELL`](#env-IN_NIX_SHELL)\ - Indicator that tells if the current environment was set up by - `nix-shell`. It can have the values `pure` or `impure`. - - - [`NIX_PATH`](#env-NIX_PATH)\ - A colon-separated list of directories used to look up the location of Nix - expressions using [paths](@docroot@/language/values.md#type-path) - enclosed in angle brackets (i.e., ``), - e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the - [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). - - If `NIX_PATH` is not set at all, Nix will fall back to the following list in [impure](@docroot@/command-ref/conf-file.md#conf-pure-eval) and [unrestricted](@docroot@/command-ref/conf-file.md#conf-restrict-eval) evaluation mode: - - 1. `$HOME/.nix-defexpr/channels` - 2. `nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixpkgs` - 3. `/nix/var/nix/profiles/per-user/root/channels` - - If `NIX_PATH` is set to an empty string, resolving search paths will always fail. - For example, attempting to use `` will produce: - - error: file 'nixpkgs' was not found in the Nix search path - - - [`NIX_IGNORE_SYMLINK_STORE`](#env-NIX_IGNORE_SYMLINK_STORE)\ - Normally, the Nix store directory (typically `/nix/store`) is not - allowed to contain any symlink components. This is to prevent - “impure” builds. Builders sometimes “canonicalise” paths by - resolving all symlink components. Thus, builds on different machines - (with `/nix/store` resolving to different locations) could yield - different results. This is generally not a problem, except when - builds are deployed to machines where `/nix/store` resolves - differently. If you are sure that you’re not going to do that, you - can set `NIX_IGNORE_SYMLINK_STORE` to `1`. - - Note that if you’re symlinking the Nix store so that you can put it - on another file system than the root file system, on Linux you’re - better off using `bind` mount points, e.g., - - ```console - $ mkdir /nix - $ mount -o bind /mnt/otherdisk/nix /nix - ``` - - Consult the mount 8 manual page for details. - - - [`NIX_STORE_DIR`](#env-NIX_STORE_DIR)\ - Overrides the location of the Nix store (default `prefix/store`). - - - [`NIX_DATA_DIR`](#env-NIX_DATA_DIR)\ - Overrides the location of the Nix static data directory (default - `prefix/share`). - - - [`NIX_LOG_DIR`](#env-NIX_LOG_DIR)\ - Overrides the location of the Nix log directory (default - `prefix/var/log/nix`). - - - [`NIX_STATE_DIR`](#env-NIX_STATE_DIR)\ - Overrides the location of the Nix state directory (default - `prefix/var/nix`). - - - [`NIX_CONF_DIR`](#env-NIX_CONF_DIR)\ - Overrides the location of the system Nix configuration directory - (default `prefix/etc/nix`). - - - [`NIX_CONFIG`](#env-NIX_CONFIG)\ - Applies settings from Nix configuration from the environment. - The content is treated as if it was read from a Nix configuration file. - Settings are separated by the newline character. - - - [`NIX_USER_CONF_FILES`](#env-NIX_USER_CONF_FILES)\ - Overrides the location of the Nix user configuration files to load from. - - The default are the locations according to the [XDG Base Directory Specification]. - See the [XDG Base Directories](#xdg-base-directories) sub-section for details. - - The variable is treated as a list separated by the `:` token. - - - [`TMPDIR`](#env-TMPDIR)\ - Use the specified directory to store temporary files. In particular, - this includes temporary build directories; these can take up - substantial amounts of disk space. The default is `/tmp`. - - - [`NIX_REMOTE`](#env-NIX_REMOTE)\ - This variable should be set to `daemon` if you want to use the Nix - daemon to execute Nix operations. This is necessary in [multi-user - Nix installations](@docroot@/installation/multi-user.md). If the Nix - daemon's Unix socket is at some non-standard path, this variable - should be set to `unix://path/to/socket`. Otherwise, it should be - left unset. - - - [`NIX_SHOW_STATS`](#env-NIX_SHOW_STATS)\ - If set to `1`, Nix will print some evaluation statistics, such as - the number of values allocated. - - - [`NIX_COUNT_CALLS`](#env-NIX_COUNT_CALLS)\ - If set to `1`, Nix will print how often functions were called during - Nix expression evaluation. This is useful for profiling your Nix - expressions. - - - [`GC_INITIAL_HEAP_SIZE`](#env-GC_INITIAL_HEAP_SIZE)\ - If Nix has been configured to use the Boehm garbage collector, this - variable sets the initial size of the heap in bytes. It defaults to - 384 MiB. Setting it to a low value reduces memory consumption, but - will increase runtime due to the overhead of garbage collection. +- [`IN_NIX_SHELL`](#env-IN_NIX_SHELL) + + Indicator that tells if the current environment was set up by + `nix-shell`. It can have the values `pure` or `impure`. + +- [`NIX_PATH`](#env-NIX_PATH) + + A colon-separated list of directories used to look up the location of Nix + expressions using [paths](@docroot@/language/values.md#type-path) + enclosed in angle brackets (i.e., ``), + e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the + [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). + + If `NIX_PATH` is not set at all, Nix will fall back to the following list in [impure](@docroot@/command-ref/conf-file.md#conf-pure-eval) and [unrestricted](@docroot@/command-ref/conf-file.md#conf-restrict-eval) evaluation mode: + + 1. `$HOME/.nix-defexpr/channels` + 2. `nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixpkgs` + 3. `/nix/var/nix/profiles/per-user/root/channels` + + If `NIX_PATH` is set to an empty string, resolving search paths will always fail. + For example, attempting to use `` will produce: + + error: file 'nixpkgs' was not found in the Nix search path + +- [`NIX_IGNORE_SYMLINK_STORE`](#env-NIX_IGNORE_SYMLINK_STORE) + + Normally, the Nix store directory (typically `/nix/store`) is not + allowed to contain any symlink components. This is to prevent + “impure” builds. Builders sometimes “canonicalise” paths by + resolving all symlink components. Thus, builds on different machines + (with `/nix/store` resolving to different locations) could yield + different results. This is generally not a problem, except when + builds are deployed to machines where `/nix/store` resolves + differently. If you are sure that you’re not going to do that, you + can set `NIX_IGNORE_SYMLINK_STORE` to `1`. + + Note that if you’re symlinking the Nix store so that you can put it + on another file system than the root file system, on Linux you’re + better off using `bind` mount points, e.g., + + ```console + $ mkdir /nix + $ mount -o bind /mnt/otherdisk/nix /nix + ``` + + Consult the mount 8 manual page for details. + +- [`NIX_STORE_DIR`](#env-NIX_STORE_DIR) + + Overrides the location of the Nix store (default `prefix/store`). + +- [`NIX_DATA_DIR`](#env-NIX_DATA_DIR) + + Overrides the location of the Nix static data directory (default + `prefix/share`). + +- [`NIX_LOG_DIR`](#env-NIX_LOG_DIR) + + Overrides the location of the Nix log directory (default + `prefix/var/log/nix`). + +- [`NIX_STATE_DIR`](#env-NIX_STATE_DIR) + + Overrides the location of the Nix state directory (default + `prefix/var/nix`). + +- [`NIX_CONF_DIR`](#env-NIX_CONF_DIR) + + Overrides the location of the system Nix configuration directory + (default `prefix/etc/nix`). + +- [`NIX_CONFIG`](#env-NIX_CONFIG) + + Applies settings from Nix configuration from the environment. + The content is treated as if it was read from a Nix configuration file. + Settings are separated by the newline character. + +- [`NIX_USER_CONF_FILES`](#env-NIX_USER_CONF_FILES) + + Overrides the location of the Nix user configuration files to load from. + + The default are the locations according to the [XDG Base Directory Specification]. + See the [XDG Base Directories](#xdg-base-directories) sub-section for details. + + The variable is treated as a list separated by the `:` token. + +- [`TMPDIR`](#env-TMPDIR) + + Use the specified directory to store temporary files. In particular, + this includes temporary build directories; these can take up + substantial amounts of disk space. The default is `/tmp`. + +- [`NIX_REMOTE`](#env-NIX_REMOTE) + + This variable should be set to `daemon` if you want to use the Nix + daemon to execute Nix operations. This is necessary in [multi-user + Nix installations](@docroot@/installation/multi-user.md). If the Nix + daemon's Unix socket is at some non-standard path, this variable + should be set to `unix://path/to/socket`. Otherwise, it should be + left unset. + +- [`NIX_SHOW_STATS`](#env-NIX_SHOW_STATS) + + If set to `1`, Nix will print some evaluation statistics, such as + the number of values allocated. + +- [`NIX_COUNT_CALLS`](#env-NIX_COUNT_CALLS) + + If set to `1`, Nix will print how often functions were called during + Nix expression evaluation. This is useful for profiling your Nix + expressions. + +- [`GC_INITIAL_HEAP_SIZE`](#env-GC_INITIAL_HEAP_SIZE) + + If Nix has been configured to use the Boehm garbage collector, this + variable sets the initial size of the heap in bytes. It defaults to + 384 MiB. Setting it to a low value reduces memory consumption, but + will increase runtime due to the overhead of garbage collection. ## XDG Base Directories diff --git a/doc/manual/src/command-ref/nix-env/install.md b/doc/manual/src/command-ref/nix-env/install.md index 5dc04c38572..c1fff50e80f 100644 --- a/doc/manual/src/command-ref/nix-env/install.md +++ b/doc/manual/src/command-ref/nix-env/install.md @@ -14,16 +14,21 @@ # Description -The install operation creates a new user environment, based on the -current generation of the active profile, to which a set of store paths -described by *args* is added. The arguments *args* map to store paths in -a number of possible ways: +The install operation creates a new user environment. +It is based on the current generation of the active [profile](@docroot@/command-ref/files/profiles.md), to which a set of [store paths] described by *args* is added. - - By default, *args* is a set of derivation names denoting derivations - in the active Nix expression. These are realised, and the resulting - output paths are installed. Currently installed derivations with a - name equal to the name of a derivation being added are removed - unless the option `--preserve-installed` is specified. +[store paths]: @docroot@/glossary.md#gloss-store-path + +The arguments *args* map to store paths in a number of possible ways: + + + - By default, *args* is a set of [derivation] names denoting derivations in the [default Nix expression]. + These are [realised], and the resulting output paths are installed. + Currently installed derivations with a name equal to the name of a derivation being added are removed unless the option `--preserve-installed` is specified. + + [derivation]: @docroot@/glossary.md#gloss-derivation + [default Nix expression]: @docroot@/command-ref/files/default-nix-expression.md + [realised]: @docroot@/glossary.md#gloss-realise If there are multiple derivations matching a name in *args* that have the same name (e.g., `gcc-3.3.6` and `gcc-4.1.1`), then the @@ -40,44 +45,90 @@ a number of possible ways: gcc-3.3.6 gcc-4.1.1` will install both version of GCC (and will probably cause a user environment conflict\!). - - If `--attr` (`-A`) is specified, the arguments are *attribute - paths* that select attributes from the top-level Nix - expression. This is faster than using derivation names and - unambiguous. To find out the attribute paths of available - packages, use `nix-env --query --available --attr-path `. + - If [`--attr`](#opt-attr) / `-A` is specified, the arguments are *attribute paths* that select attributes from the [default Nix expression]. + This is faster than using derivation names and unambiguous. + Show the attribute paths of available packages with [`nix-env --query`](./query.md): + + ```console + nix-env --query --available --attr-path` + ``` - If `--from-profile` *path* is given, *args* is a set of names - denoting installed store paths in the profile *path*. This is an + denoting installed [store paths] in the profile *path*. This is an easy way to copy user environment elements from one profile to another. - - If `--from-expression` is given, *args* are Nix - [functions](@docroot@/language/constructs.md#functions) - that are called with the active Nix expression as their single - argument. The derivations returned by those function calls are - installed. This allows derivations to be specified in an - unambiguous way, which is necessary if there are multiple - derivations with the same name. - - - If *args* are [store derivations](@docroot@/glossary.md#gloss-store-derivation), then these are - [realised](@docroot@/command-ref/nix-store/realise.md), and the resulting output paths - are installed. + - If `--from-expression` is given, *args* are [Nix language functions](@docroot@/language/constructs.md#functions) that are called with the [default Nix expression] as their single argument. + The derivations returned by those function calls are installed. + This allows derivations to be specified in an unambiguous way, which is necessary if there are multiple derivations with the same name. + + - If *args* are [store derivations](@docroot@/glossary.md#gloss-store-derivation), then these are [realised], and the resulting output paths are installed. + + - If *args* are [store paths] that are not store derivations, then these are [realised] and installed. + + - By default all [outputs](@docroot@/language/derivations.md#attr-outputs) are installed for each [derivation]. + This can be overridden by adding a `meta.outputsToInstall` attribute on the derivation listing a subset of the output names. + + Example: + + The file `example.nix` defines a derivation with two outputs `foo` and `bar`, each containing a file. + + ```nix + # example.nix + let + pkgs = import {}; + command = '' + ${pkgs.coreutils}/bin/mkdir -p $foo $bar + echo foo > $foo/foo-file + echo bar > $bar/bar-file + ''; + in + derivation { + name = "example"; + builder = "${pkgs.bash}/bin/bash"; + args = [ "-c" command ]; + outputs = [ "foo" "bar" ]; + system = builtins.currentSystem; + } + ``` + + Installing from this Nix expression will make files from both outputs appear in the current profile. + + ```console + $ nix-env --install --file example.nix + installing 'example' + $ ls ~/.nix-profile + foo-file + bar-file + manifest.nix + ``` + + Adding `meta.outputsToInstall` to that derivation will make `nix-env` only install files from the specified outputs. + + ```nix + # example-outputs.nix + import ./example.nix // { meta.outputsToInstall = [ "bar" ]; } + ``` + + ```console + $ nix-env --install --file example-outputs.nix + installing 'example' + $ ls ~/.nix-profile + bar-file + manifest.nix + ``` + +# Options + + - `--prebuilt-only` / `-b` - - If *args* are store paths that are not store derivations, then these - are [realised](@docroot@/command-ref/nix-store/realise.md) and installed. - - - By default all outputs are installed for each derivation. That can - be reduced by setting `meta.outputsToInstall`. - -# Flags - - - `--prebuilt-only` / `-b`\ Use only derivations for which a substitute is registered, i.e., there is a pre-built binary available that can be downloaded in lieu of building the derivation. Thus, no packages will be built from source. - - `--preserve-installed` / `-P`\ + - `--preserve-installed` / `-P` + Do not remove derivations with a name matching one of the derivations being installed. Usually, trying to have two versions of the same package installed in the same generation of a profile will @@ -85,7 +136,8 @@ a number of possible ways: clashes between the two versions. However, this is not the case for all packages. - - `--remove-all` / `-r`\ + - `--remove-all` / `-r` + Remove all previously installed packages first. This is equivalent to running `nix-env --uninstall '.*'` first, except that everything happens in a single transaction. diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 195b72be50d..1eaf3c36aa7 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -235,14 +235,14 @@ package like Terraform: ```bash #! /usr/bin/env nix-shell -#! nix-shell -i bash --packages "terraform.withPlugins (plugins: [ plugins.openstack ])" +#! nix-shell -i bash --packages 'terraform.withPlugins (plugins: [ plugins.openstack ])' terraform apply ``` > **Note** > -> You must use double quotes (`"`) when passing a simple Nix expression +> You must use single or double quotes (`'`, `"`) when passing a simple Nix expression > in a nix-shell shebang. Finally, using the merging of multiple nix-shell shebangs the following @@ -251,7 +251,7 @@ branch): ```haskell #! /usr/bin/env nix-shell -#! nix-shell -i runghc --packages "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])" +#! nix-shell -i runghc --packages 'haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])' #! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-20.03.tar.gz import Network.Curl.Download diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index a421a422032..5428d57fa50 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -15,8 +15,12 @@ Each of *paths* is processed as follows: 1. If it is not [valid], substitute the store derivation file itself. 2. Realise its [output paths]: - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure]. - - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying content-addressed realisation entries in the [Nix database]. - - For any store paths that cannot be substituted, produce the required store objects. This involves first realising all outputs of the derivation's dependencies and then running the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable. + - With [content-addressed derivations] (experimental): + Determine the output paths to realise by querying content-addressed realisation entries in the [Nix database]. + - For any store paths that cannot be substituted, produce the required store objects: + 1. Realise all outputs of the derivation's dependencies + 2. Run the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable + - Otherwise, and if the path is not already valid: Try to fetch the associated [store objects] in the path's [closure] from [substituters]. If no substitutes are available and no store derivation is given, realisation fails. diff --git a/doc/manual/src/command-ref/opt-common-syn.md b/doc/manual/src/command-ref/opt-common-syn.md deleted file mode 100644 index b66d318c24f..00000000000 --- a/doc/manual/src/command-ref/opt-common-syn.md +++ /dev/null @@ -1,57 +0,0 @@ -\--help - -\--version - -\--verbose - -\-v - -\--quiet - -\--log-format - -format - -\--no-build-output - -\-Q - -\--max-jobs - -\-j - -number - -\--cores - -number - -\--max-silent-time - -number - -\--timeout - -number - -\--keep-going - -\-k - -\--keep-failed - -\-K - -\--fallback - -\--readonly-mode - -\-I - -path - -\--option - -name - -value diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 0647228c23a..114b292f930 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -203,3 +203,7 @@ Most Nix commands accept the following command-line options: Fix corrupted or missing store paths by redownloading or rebuilding them. Note that this is slow because it requires computing a cryptographic hash of the contents of every path in the closure of the build. Also note the warning under `nix-store --repair-path`. + +> **Note** +> +> See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. diff --git a/doc/manual/src/command-ref/opt-inst-syn.md b/doc/manual/src/command-ref/opt-inst-syn.md deleted file mode 100644 index 1703c40e39c..00000000000 --- a/doc/manual/src/command-ref/opt-inst-syn.md +++ /dev/null @@ -1,15 +0,0 @@ -\--prebuilt-only - -\-b - -\--attr - -\-A - -\--from-expression - -\-E - -\--from-profile - -path diff --git a/doc/manual/src/contributing/contributing.md b/doc/manual/src/contributing/contributing.md index 854139a319a..4d55c17a46f 100644 --- a/doc/manual/src/contributing/contributing.md +++ b/doc/manual/src/contributing/contributing.md @@ -1 +1,8 @@ -# Contributing +# Development + +Nix is developed on GitHub. +Check the [contributing guide](https://github.com/NixOS/nix/blob/master/CONTRIBUTING.md) if you want to get involved. + +This chapter is a collection of guides for making changes to the code and documentation. + +If you're not sure where to start, try to [compile Nix from source](./hacking.md) and consider [making improvements to documentation](./documentation.md). diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/contributing/documentation.md new file mode 100644 index 00000000000..75226cd1a0c --- /dev/null +++ b/doc/manual/src/contributing/documentation.md @@ -0,0 +1,210 @@ +# Contributing documentation + +Improvements to documentation are very much appreciated, and a good way to start out with contributing to Nix. + +This is how you can help: +- Address [open issues with documentation](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+label%3Adocumentation) +- Review [pull requests concerning documentation](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+label%3Adocumentation) + +Incremental refactorings of the documentation build setup to make it faster or easier to understand and maintain are also welcome. + +## Building the manual + +Build the manual from scratch: + +```console +nix-build $(nix-instantiate)'!doc' +``` + +or + +```console +nix build .#^doc +``` + +and open `./result-doc/share/doc/nix/manual/index.html`. + +To build the manual incrementally, [enter the development shell](./hacking.md) and run: + +```console +make manual-html -j $NIX_BUILD_CORES +``` + +and open `./outputs/out/share/doc/nix/manual/language/index.html`. + +In order to reflect changes to the [Makefile for the manual], clear all generated files before re-building: + +[Makefile for the manual]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk + +```console +rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/command-ref/new-cli && make manual-html -j $NIX_BUILD_CORES +``` + +## Style guide + +The goal of this style guide is to make it such that +- The manual is easy to search and skim for relevant information +- Documentation sources are easy to edit +- Changes to documentation are easy to review + +You will notice that this is not implemented consistently yet. +Please follow the guide when making additions or changes to existing documentation. +Do not make sweeping changes, unless they are programmatic and can be validated easily. + +### Language + +This manual is [reference documentation](https://diataxis.fr/reference/). +The typical usage pattern is to look up isolated pieces of information. +It should therefore aim to be correct, consistent, complete, and easy to navigate at a glance. + +- Aim for clarity and brevity. + + Please take the time to read the [plain language guidelines](https://www.plainlanguage.gov/guidelines/) for details. + +- Describe the subject factually. + + In particular, do not make value judgements or recommendations. + Check the code or add tests if in doubt. + +- Provide complete, minimal examples, and explain them. + + Readers should be able to try examples verbatim and get the same results as shown in the manual. + Always describe in words what a given example does. + + Non-trivial examples may need additional explanation, especially if they use concepts from outside the given context. + +- Always explain code examples in the text. + + Use comments in code samples very sparingly, for instance to highlight a particular aspect. + Readers tend to glance over large amounts of code when scanning for information. + + Especially beginners will likely find reading more complex-looking code strenuous and may therefore avoid it altogether. + + If a code sample appears to require a lot of inline explanation, consider replacing it with a simpler one. + If that's not possible, break the example down into multiple parts, explain them separately, and then show the combined result at the end. + This should be a last resort, as that would amount to writing a [tutorial](https://diataxis.fr/tutorials/) on the given subject. + +- Use British English. + + This is a somewhat arbitrary choice to force consistency, and accounts for the fact that a majority of Nix users and developers are from Europe. + +### Links and anchors + +Reference documentation must be readable in arbitrary order. +Readers cannot be expected to have any particular prerequisite knowledge about Nix. +While the table of contents can provide guidance and full-text search can help, they are most likely to find what they need by following sensible cross-references. + +- Link to technical terms + + When mentioning Nix-specific concepts, commands, options, settings, etc., link to appropriate documentation. + Also link to external tools or concepts, especially if their meaning may be ambiguous. + You may also want to link to definitions of less common technical terms. + + Then readers won't have to actively search for definitions and are more likely to discover relevant information on their own. + + > **Note** + > + > `man` and `--help` pages don't display links. + > Use appropriate link texts such that readers of terminal output can infer search terms. + +- Do not break existing URLs between releases. + + There are countless links in the wild pointing to old versions of the manual. + We want people to find up-to-date documentation when following popular advice. + + - When moving files, update [redirects on nixos.org](https://github.com/NixOS/nixos-homepage/blob/master/netlify.toml). + + This is especially important when moving information out of the Nix manual to other resources. + + - When changing anchors, update [client-side redirects](https://github.com/NixOS/nix/blob/master/doc/manual/redirects.js) + + The current setup is cumbersome, and help making better automation is appreciated. + +The build checks for broken internal links with. +This happens late in the process, so [building the whole manual](#building-the-manual) is not suitable for iterating quickly. +[`mdbook-linkcheck`] does not implement checking [URI fragments] yet. + +[`mdbook-linkcheck`]: https://github.com/Michael-F-Bryan/mdbook-linkcheck +[URI fragments]: https://en.wikipedia.org/wiki/URI_fragment + +### Markdown conventions + +The manual is written in markdown, and rendered with [mdBook](https://github.com/rust-lang/mdBook) for the web and with [lowdown](https://github.com/kristapsdz/lowdown) for `man` pages and `--help` output. + +For supported markdown features, refer to: +- [mdBook documentation](https://rust-lang.github.io/mdBook/format/markdown.html) +- [lowdown documentation](https://kristaps.bsd.lv/lowdown/) + +Please observe these guidelines to ease reviews: + +- Write one sentence per line. + + This makes long sentences immediately visible, and makes it easier to review changes and make direct suggestions. + +- Use reference links – sparingly – to ease source readability. + Put definitions close to their first use. + + Example: + + ``` + A [store object] contains a [file system object] and [references] to other store objects. + + [store object]: @docroot@/glossary.md#gloss-store-object + [file system object]: @docroot@/architecture/file-system-object.md + [references]: @docroot@/glossary.md#gloss-reference + ``` + +- Use admonitions of the following form: + + ``` + > **Note** + > + > This is a note. + ``` + + Highlight examples as such: + + ```` + > **Example** + > + > ```console + > $ nix --version + > ``` + ```` + + Highlight syntax definiions as such, using [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) notation: + + ```` + > **Syntax** + > + > *attribute-set* = `{` [ *attribute-name* `=` *expression* `;` ... ] `}` + ```` + +### The `@docroot@` variable + +`@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own. + +If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory. + +If the `@docroot@` literal appears in an error message from the [`mdbook-linkcheck`] tool, the `@docroot@` replacement needs to be applied to the generated source file that mentions it. +See existing `@docroot@` logic in the [Makefile for the manual]. +Regular markdown files used for the manual have a base path of their own and they can use relative paths instead of `@docroot@`. + +## API documentation + +[Doxygen API documentation] is available online. +You can also build and view it yourself: + +[Doxygen API documentation]: https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs + +```console +# nix build .#hydraJobs.internal-api-docs +# xdg-open ./result/share/doc/nix/internal-api/html/index.html +``` + +or inside `nix-shell` or `nix develop`: + +``` +# make internal-api-html +# xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html +``` diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 4b0a3a3e5ad..fe08ceb94e9 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -42,8 +42,8 @@ $ nix develop .#native-clang11StdenvPackages To build Nix itself in this shell: ```console -[nix-shell]$ ./bootstrap.sh -[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out +[nix-shell]$ autoreconfPhase +[nix-shell]$ configurePhase [nix-shell]$ make -j $NIX_BUILD_CORES ``` @@ -86,7 +86,7 @@ $ nix-shell --attr devShells.x86_64-linux.native-clang11StdenvPackages To build Nix itself in this shell: ```console -[nix-shell]$ ./bootstrap.sh +[nix-shell]$ autoreconfPhase [nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out [nix-shell]$ make -j $NIX_BUILD_CORES ``` @@ -210,7 +210,7 @@ See [supported compilation environments](#compilation-environments) and instruct To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running: ```console -make clean && bear -- make -j$NIX_BUILD_CORES install +make clean && bear -- make -j$NIX_BUILD_CORES default check install ``` Configure your editor to use the `clangd` from the shell, either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). @@ -220,68 +220,3 @@ Configure your editor to use the `clangd` from the shell, either by running it i > For some editors (e.g. Visual Studio Code), you may need to install a [special extension](https://open-vsx.org/extension/llvm-vs-code-extensions/vscode-clangd) for the editor to interact with `clangd`. > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim). > Editor-specific setup is typically opinionated, so we will not cover it here in more detail. - -### Checking links in the manual - -The build checks for broken internal links. -This happens late in the process, so `nix build` is not suitable for iterating. -To build the manual incrementally, run: - -```console -make html -j $NIX_BUILD_CORES -``` - -In order to reflect changes to the [Makefile], clear all generated files before re-building: - -[Makefile]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk - -```console -rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/command-ref/new-cli && make html -j $NIX_BUILD_CORES -``` - -[`mdbook-linkcheck`] does not implement checking [URI fragments] yet. - -[`mdbook-linkcheck`]: https://github.com/Michael-F-Bryan/mdbook-linkcheck -[URI fragments]: https://en.wikipedia.org/wiki/URI_fragment - -#### `@docroot@` variable - -`@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own. - -If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory. - -If the `@docroot@` literal appears in an error message from the `mdbook-linkcheck` tool, the `@docroot@` replacement needs to be applied to the generated source file that mentions it. -See existing `@docroot@` logic in the [Makefile]. -Regular markdown files used for the manual have a base path of their own and they can use relative paths instead of `@docroot@`. - -## API documentation - -Doxygen API documentation is [available -online](https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs). You -can also build and view it yourself: - -```console -# nix build .#hydraJobs.internal-api-docs -# xdg-open ./result/share/doc/nix/internal-api/html/index.html -``` - -or inside a `nix develop` shell by running: - -``` -# make internal-api-html -# xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html -``` - -## Coverage analysis - -A coverage analysis report is [available -online](https://hydra.nixos.org/job/nix/master/coverage/latest/download-by-type/report/coverage). You -can build it yourself: - -``` -# nix build .#hydraJobs.coverage -# xdg-open ./result/coverage/index.html -``` - -Metrics about the change in line/function coverage over time are also -[available](https://hydra.nixos.org/job/nix/master/coverage#tabs-charts). diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index cd94d5cfbaf..d8d16237981 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -1,18 +1,117 @@ # Running tests +## Coverage analysis + +A [coverage analysis report] is available online +You can build it yourself: + +[coverage analysis report]: https://hydra.nixos.org/job/nix/master/coverage/latest/download-by-type/report/coverage + +``` +# nix build .#hydraJobs.coverage +# xdg-open ./result/coverage/index.html +``` + +[Extensive records of build metrics](https://hydra.nixos.org/job/nix/master/coverage#tabs-charts), such as test coverage over time, are also available online. + ## Unit-tests -The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined -under `src/{library_name}/tests` using the -[googletest](https://google.github.io/googletest/) and -[rapidcheck](https://github.com/emil-e/rapidcheck) frameworks. +The unit tests are defined using the [googletest] and [rapidcheck] frameworks. + +[googletest]: https://google.github.io/googletest/ +[rapidcheck]: https://github.com/emil-e/rapidcheck +[property testing]: https://en.wikipedia.org/wiki/Property_testing + +### Source and header layout + +> An example of some files, demonstrating much of what is described below +> +> ``` +> src +> ├── libexpr +> │ ├── local.mk +> │ ├── value/context.hh +> │ ├── value/context.cc +> │ … +> │ +> ├── tests +> │ │ +> │ … +> │ └── unit +> │ ├── libutil +> │ │ ├── local.mk +> │ │ … +> │ │ └── data +> │ │ ├── git/tree.txt +> │ │ … +> │ │ +> │ ├── libexpr-support +> │ │ ├── local.mk +> │ │ └── tests +> │ │ ├── value/context.hh +> │ │ ├── value/context.cc +> │ │ … +> │ │ +> │ ├── libexpr +> │ … ├── local.mk +> │ ├── value/context.cc +> │ … +> … +> ``` + +The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `tests/unit/${library_name_without-nix}`. +Given a interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `tests/unit/libexpr/tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `tests/unit/libexpr-support/tests/value/context.{hh,cc}`. + +Data for unit tests is stored in a `data` subdir of the directory for each unit test executable. +For example, `libnixstore` code is in `src/libstore`, and its test data is in `tests/unit/libstore/data`. +The path to the `tests/unit/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. +Note that each executable only gets the data for its tests. + +The unit test libraries are in `tests/unit/${library_name_without-nix}-lib`. +All headers are in a `tests` subdirectory so they are included with `#include "tests/"`. + +The use of all these separate directories for the unit tests might seem inconvenient, as for example the tests are not "right next to" the part of the code they are testing. +But organizing the tests this way has one big benefit: +there is no risk of any build-system wildcards for the library accidentally picking up test code that should not built and installed as part of the library. + +### Running tests You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable. +### Characterisation testing { #characaterisation-testing-unit } + +See [functional characterisation testing](#characterisation-testing-functional) for a broader discussion of characterisation testing. + +Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used. +For example: +```shell-session +$ _NIX_TEST_ACCEPT=1 make libstore-tests_RUN +... +[ SKIPPED ] WorkerProtoTest.string_read +[ SKIPPED ] WorkerProtoTest.string_write +[ SKIPPED ] WorkerProtoTest.storePath_read +[ SKIPPED ] WorkerProtoTest.storePath_write +... +``` +will regenerate the "golden master" expected result for the `libnixstore` characterisation tests. +The characterisation tests will mark themselves "skipped" since they regenerated the expected result instead of actually testing anything. + +### Unit test support libraries + +There are headers and code which are not just used to test the library in question, but also downstream libraries. +For example, we do [property testing] with the [rapidcheck] library. +This requires writing `Arbitrary` "instances", which are used to describe how to generate values of a given type for the sake of running property tests. +Because types contain other types, `Arbitrary` "instances" for some type are not just useful for testing that type, but also any other type that contains it. +Downstream types frequently contain upstream types, so it is very important that we share arbitrary instances so that downstream libraries' property tests can also use them. + +It is important that these testing libraries don't contain any actual tests themselves. +On some platforms they would be run as part of every test executable that uses them, which is redundant. +On other platforms they wouldn't be run at all. + ## Functional tests -The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. +The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`. Each test is a bash script. ### Running the whole test suite @@ -21,8 +120,8 @@ The whole test suite can be run with: ```shell-session $ make install && make installcheck -ran test tests/foo.sh... [PASS] -ran test tests/bar.sh... [PASS] +ran test tests/functional/foo.sh... [PASS] +ran test tests/functional/bar.sh... [PASS] ... ``` @@ -30,14 +129,14 @@ ran test tests/bar.sh... [PASS] Sometimes it is useful to group related tests so they can be easily run together without running the entire test suite. Each test group is in a subdirectory of `tests`. -For example, `tests/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs. +For example, `tests/functional/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs. That test group can be run like this: ```shell-session $ make ca.test-group -j50 -ran test tests/ca/nix-run.sh... [PASS] -ran test tests/ca/import-derivation.sh... [PASS] +ran test tests/functional/ca/nix-run.sh... [PASS] +ran test tests/functional/ca/import-derivation.sh... [PASS] ... ``` @@ -56,24 +155,24 @@ install-tests-groups += $(test-group-name) Individual tests can be run with `make`: ```shell-session -$ make tests/${testName}.sh.test -ran test tests/${testName}.sh... [PASS] +$ make tests/functional/${testName}.sh.test +ran test tests/functional/${testName}.sh... [PASS] ``` or without `make`: ```shell-session -$ ./mk/run-test.sh tests/${testName}.sh -ran test tests/${testName}.sh... [PASS] +$ ./mk/run-test.sh tests/functional/${testName}.sh tests/functional/init.sh +ran test tests/functional/${testName}.sh... [PASS] ``` To see the complete output, one can also run: ```shell-session -$ ./mk/debug-test.sh tests/${testName}.sh -+ foo +$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh ++(${testName}.sh:1) foo output from foo -+ bar ++(${testName}.sh:2) bar output from bar ... ``` @@ -105,7 +204,7 @@ edit it like so: Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: ```shell-session -$ ./mk/debug-test.sh tests/${testName}.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh ... + gdb blash blub GNU gdb (GDB) 12.1 @@ -116,17 +215,29 @@ GNU gdb (GDB) 12.1 One can debug the Nix invocation in all the usual ways. For example, enter `run` to start the Nix invocation. -### Characterization testing +### Troubleshooting + +Sometimes running tests in the development shell may leave artefacts in the local repository. +To remove any traces of that: -Occasionally, Nix utilizes a technique called [Characterization Testing](https://en.wikipedia.org/wiki/Characterization_test) as part of the functional tests. +```console +git clean -x --force tests +``` + +### Characterisation testing { #characterisation-testing-functional } + +Occasionally, Nix utilizes a technique called [Characterisation Testing](https://en.wikipedia.org/wiki/Characterization_test) as part of the functional tests. This technique is to include the exact output/behavior of a former version of Nix in a test in order to check that Nix continues to produce the same behavior going forward. For example, this technique is used for the language tests, to check both the printed final value if evaluation was successful, and any errors and warnings encountered. It is frequently useful to regenerate the expected output. -To do that, rerun the failed test with `_NIX_TEST_ACCEPT=1`. -(At least, this is the convention we've used for `tests/lang.sh`. -If we add more characterization testing we should always strive to be consistent.) +To do that, rerun the failed test(s) with `_NIX_TEST_ACCEPT=1`. +For example: +```bash +_NIX_TEST_ACCEPT=1 make tests/functional/lang.sh.test +``` +This convention is shared with the [characterisation unit tests](#characterisation-testing-unit) too. An interesting situation to document is the case when these tests are "overfitted". The language tests are, again, an example of this. @@ -139,7 +250,7 @@ Diagnostic outputs are indeed not a stable interface, but they still are importa By recording the expected output, the test suite guards against accidental changes, and ensure the *result* (not just the code that implements it) of the diagnostic code paths are under code review. Regressions are caught, and improvements always show up in code review. -To ensure that characterization testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. +To ensure that characterisation testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. ## Integration tests @@ -153,7 +264,7 @@ You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix- After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch. -Creating a Cachix cache for your installer tests and adding its authorization token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91): +Creating a Cachix cache for your installer tests and adding its authorisation token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91): - The `installer` job generates installers for the platforms below and uploads them to your Cachix cache: - `x86_64-linux` diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index b7c340d36d6..07891175a4e 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -33,11 +33,15 @@ Ensure a [store path] is [valid][validity]. - This means either running the [`builder`](@docroot@/language/derivations.md#attr-builder) executable as specified in the corresponding [derivation], or fetching a pre-built [store object] from a [substituter], or delegating to a [remote builder](@docroot@/advanced-topics/distributed-builds.html) and retrieving the outputs. + This can be achieved by: + - Fetching a pre-built [store object] from a [substituter] + - Running the [`builder`](@docroot@/language/derivations.md#attr-builder) executable as specified in the corresponding [derivation] + - Delegating to a [remote builder](@docroot@/advanced-topics/distributed-builds.html) and retrieving the outputs + - See [`nix-build`](./command-ref/nix-build.md) and [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md). + See [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md) for a detailed description of the algorithm. - See [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). + See also [`nix-build`](./command-ref/nix-build.md) and [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). [realise]: #gloss-realise @@ -54,22 +58,16 @@ - [store]{#gloss-store} - The location in the file system where store objects live. Typically - `/nix/store`. + A collection of store objects, with operations to manipulate that collection. + See [Nix store](./store/index.md) for details. - From the perspective of the location where Nix is - invoked, the Nix store can be referred to - as a "_local_" or a "_remote_" one: + There are many types of stores. + See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) for a complete list. - + A [local store]{#gloss-local-store} exists on the filesystem of - the machine where Nix is invoked. You can use other - local stores by passing the `--store` flag to the - `nix` command. Local stores can be used for building derivations. - - + A *remote store* exists anywhere other than the - local filesystem. One example is the `/nix/store` - directory on another machine, accessed via `ssh` or - served by the `nix-serve` Perl script. + From the perspective of the location where Nix is invoked, the Nix store can be referred to _local_ or _remote_. + Only a [local store]{#gloss-local-store} exposes a location in the file system of the machine where Nix is invoked that allows access to store objects, typically `/nix/store`. + Local stores can be used for building [derivations](#derivation). + See [Local Store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) for details. [store]: #gloss-store [local store]: #gloss-local-store @@ -88,10 +86,13 @@ - [store path]{#gloss-store-path} - The location of a [store object] in the file system, i.e., an - immediate child of the Nix store directory. + The location of a [store object](@docroot@/store/index.md#store-object) in the file system, i.e., an immediate child of the Nix store directory. + + > **Example** + > + > `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` - Example: `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` + See [Store Path](@docroot@/store/store-path.md) for details. [store path]: #gloss-store-path @@ -99,18 +100,25 @@ The Nix data model for representing simplified file system data. - See [File System Object](@docroot@/architecture/file-system-object.md) for details. + See [File System Object](@docroot@/store/file-system-object.md) for details. [file system object]: #gloss-file-system-object - [store object]{#gloss-store-object} + Part of the contents of a [store]. - A store object consists of a [file system object], [reference]s to other store objects, and other metadata. + A store object consists of a [file system object], [references][reference] to other store objects, and other metadata. It can be referred to by a [store path]. + See [Store Object](@docroot@/store/index.md#store-object) for details. + [store object]: #gloss-store-object +- [IFD]{#gloss-ifd} + + [Import From Derivation](./language/import-from-derivation.md) + - [input-addressed store object]{#gloss-input-addressed-store-object} A store object produced by building a @@ -200,6 +208,7 @@ - [output]{#gloss-output} A [store object] produced by a [derivation]. + See [the `outputs` argument to the `derivation` function](@docroot@/language/derivations.md#attr-outputs) for details. [output]: #gloss-output diff --git a/doc/manual/src/installation/building-source.md b/doc/manual/src/installation/building-source.md index ed1efffd8eb..7dad9805a23 100644 --- a/doc/manual/src/installation/building-source.md +++ b/doc/manual/src/installation/building-source.md @@ -3,7 +3,7 @@ After cloning Nix's Git repository, issue the following commands: ```console -$ ./bootstrap.sh +$ autoreconf -vfi $ ./configure options... $ make $ make install diff --git a/doc/manual/src/installation/installing-docker.md b/doc/manual/src/installation/installing-docker.md index 9d6d8f2d945..6f77d6a5708 100644 --- a/doc/manual/src/installation/installing-docker.md +++ b/doc/manual/src/installation/installing-docker.md @@ -3,14 +3,14 @@ To run the latest stable release of Nix with Docker run the following command: ```console -$ docker run -ti nixos/nix -Unable to find image 'nixos/nix:latest' locally -latest: Pulling from nixos/nix +$ docker run -ti ghcr.io/nixos/nix +Unable to find image 'ghcr.io/nixos/nix:latest' locally +latest: Pulling from ghcr.io/nixos/nix 5843afab3874: Pull complete b52bf13f109c: Pull complete 1e2415612aa3: Pull complete Digest: sha256:27f6e7f60227e959ee7ece361f75d4844a40e1cc6878b6868fe30140420031ff -Status: Downloaded newer image for nixos/nix:latest +Status: Downloaded newer image for ghcr.io/nixos/nix:latest 35ca4ada6e96:/# nix --version nix (Nix) 2.3.12 35ca4ada6e96:/# exit diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 5e8aaeba05b..282b75af2cf 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -112,6 +112,13 @@ Derivations can declare some infrequently used optional attributes. > environmental variables come from the environment of the > `nix-build`. + If the [`configurable-impure-env` experimental + feature](@docroot@/contributing/experimental-features.md#xp-feature-configurable-impure-env) + is enabled, these environment variables can also be controlled + through the + [`impure-env`](@docroot@/command-ref/conf-file.md#conf-impure-env) + configuration setting. + - [`outputHash`]{#adv-attr-outputHash}; [`outputHashAlgo`]{#adv-attr-outputHashAlgo}; [`outputHashMode`]{#adv-attr-outputHashMode}\ These attributes declare that the derivation is a so-called *fixed-output derivation*, which means that a cryptographic hash of @@ -229,6 +236,8 @@ Derivations can declare some infrequently used optional attributes. [`outputHashAlgo`](#adv-attr-outputHashAlgo) like for *fixed-output derivations* (see above). + It also implicitly requires that the machine to build the derivation must have the `ca-derivations` [system feature](@docroot@/command-ref/conf-file.md#conf-system-features). + - [`passAsFile`]{#adv-attr-passAsFile}\ A list of names of attributes that should be passed via files rather than environment variables. For example, if you have @@ -261,6 +270,9 @@ Derivations can declare some infrequently used optional attributes. useful for very trivial derivations (such as `writeText` in Nixpkgs) that are cheaper to build than to substitute from a binary cache. + You may disable the effects of this attibute by enabling the + `always-allow-substitutes` configuration option in Nix. + > **Note** > > You need to have a builder configured which satisfies the @@ -271,18 +283,21 @@ Derivations can declare some infrequently used optional attributes. - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\ If the special attribute `__structuredAttrs` is set to `true`, the other derivation - attributes are serialised in JSON format and made available to the - builder via the file `.attrs.json` in the builder’s temporary - directory. This obviates the need for [`passAsFile`](#adv-attr-passAsFile) since JSON files - have no size restrictions, unlike process environments. + attributes are serialised into a file in JSON format. The environment variable + `NIX_ATTRS_JSON_FILE` points to the exact location of that file both in a build + and a [`nix-shell`](../command-ref/nix-shell.md). This obviates the need for + [`passAsFile`](#adv-attr-passAsFile) since JSON files have no size restrictions, + unlike process environments. It also makes it possible to tweak derivation settings in a structured way; see [`outputChecks`](#adv-attr-outputChecks) for example. As a convenience to Bash builders, - Nix writes a script named `.attrs.sh` to the builder’s directory - that initialises shell variables corresponding to all attributes - that are representable in Bash. This includes non-nested + Nix writes a script that initialises shell variables + corresponding to all attributes that are representable in Bash. The + environment variable `NIX_ATTRS_SH_FILE` points to the exact + location of the script, both in a build and a + [`nix-shell`](../command-ref/nix-shell.md). This includes non-nested (associative) arrays. For example, the attribute `hardening.format = true` ends up as the Bash associative array element `${hardening[format]}`. @@ -335,3 +350,15 @@ Derivations can declare some infrequently used optional attributes. This is useful, for example, when generating self-contained filesystem images with their own embedded Nix store: hashes found inside such an image refer to the embedded store and not to the host's Nix store. + +- [`requiredSystemFeatures`]{#adv-attr-requiredSystemFeatures}\ + + If a derivation has the `requiredSystemFeatures` attribute, then Nix will only build it on a machine that has the corresponding features set in its [`system-features` configuration](@docroot@/command-ref/conf-file.md#conf-system-features). + + For example, setting + + ```nix + requiredSystemFeatures = [ "kvm" ]; + ``` + + ensures that the derivation can only be built on a machine with the `kvm` feature. diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md index a3590f55df9..a82ec5960a8 100644 --- a/doc/manual/src/language/constructs.md +++ b/doc/manual/src/language/constructs.md @@ -132,6 +132,32 @@ a = src-set.a; b = src-set.b; c = src-set.c; when used while defining local variables in a let-expression or while defining a set. +In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example + + +```nix +let + x = { a = 1; b = 2; }; + inherit (builtins) attrNames; +in +{ + names = attrNames x; +} +``` + +is equivalent to + +```nix +let + x = { a = 1; b = 2; }; +in +{ + names = builtins.attrNames x; +} +``` + +both evaluate to `{ names = [ "a" "b" ]; }`. + ## Functions Functions have the following form: @@ -146,65 +172,65 @@ three kinds of patterns: - If a pattern is a single identifier, then the function matches any argument. Example: - + ```nix let negate = x: !x; concat = x: y: x + y; in if negate true then concat "foo" "bar" else "" ``` - + Note that `concat` is a function that takes one argument and returns a function that takes another argument. This allows partial parameterisation (i.e., only filling some of the arguments of a function); e.g., - + ```nix map (concat "foo") [ "bar" "bla" "abc" ] ``` - + evaluates to `[ "foobar" "foobla" "fooabc" ]`. - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a set containing the listed attributes, and binds the values of those attributes to variables in the function body. For example, the function - + ```nix { x, y, z }: z + y + x ``` - + can only be called with a set containing exactly the attributes `x`, `y` and `z`. No other attributes are allowed. If you want to allow additional arguments, you can use an ellipsis (`...`): - + ```nix { x, y, z, ... }: z + y + x ``` - + This works on any set that contains at least the three named attributes. - + It is possible to provide *default values* for attributes, in which case they are allowed to be missing. A default value is specified by writing `name ? e`, where *e* is an arbitrary expression. For example, - + ```nix { x, y ? "foo", z ? "bar" }: z + y + x ``` - + specifies a function that only requires an attribute named `x`, but optionally accepts `y` and `z`. - An `@`-pattern provides a means of referring to the whole value being matched: - + ```nix args@{ x, y, z, ... }: z + y + x + args.a ``` - + but can also be written as: - + ```nix { x, y, z, ... } @ args: z + y + x + args.a ``` diff --git a/doc/manual/src/language/constructs/lookup-path.md b/doc/manual/src/language/constructs/lookup-path.md new file mode 100644 index 00000000000..e87d2922bd3 --- /dev/null +++ b/doc/manual/src/language/constructs/lookup-path.md @@ -0,0 +1,27 @@ +# Lookup path + +> **Syntax** +> +> *lookup-path* = `<` *identifier* [ `/` *identifier* ]... `>` + +A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/values.md#type-path) if the identifier matches a search path entry. + +The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). + +See [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile) for details on lookup path resolution. + +> **Example** +> +> ```nix +> +>``` +> +> /nix/var/nix/profiles/per-user/root/channels/nixpkgs + +> **Example** +> +> ```nix +> +>``` +> +> /nix/var/nix/profiles/per-user/root/channels/nixpkgs/nixos diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 79a09122a3e..2aded5527b9 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -1,161 +1,315 @@ # Derivations -The most important built-in function is `derivation`, which is used to -describe a single derivation (a build task). It takes as input a set, -the attributes of which specify the inputs of the build. - - - There must be an attribute named [`system`]{#attr-system} whose value must be a - string specifying a Nix system type, such as `"i686-linux"` or - `"x86_64-darwin"`. (To figure out your system type, run `nix -vv - --version`.) The build can only be performed on a machine and - operating system matching the system type. (Nix can automatically - [forward builds for other - platforms](../advanced-topics/distributed-builds.md) by forwarding - them to other machines.) - - - There must be an attribute named `name` whose value must be a - string. This is used as a symbolic name for the package by - `nix-env`, and it is appended to the output paths of the derivation. - - - There must be an attribute named [`builder`]{#attr-builder} that identifies the - program that is executed to perform the build. It can be either a - derivation or a source (a local file reference, e.g., - `./builder.sh`). - - - Every attribute is passed as an environment variable to the builder. - Attribute values are translated to environment variables as follows: - - - Strings and numbers are just passed verbatim. - - - A *path* (e.g., `../foo/sources.tar`) causes the referenced file - to be copied to the store; its location in the store is put in - the environment variable. The idea is that all sources should - reside in the Nix store, since all inputs to a derivation should - reside in the Nix store. - - - A *derivation* causes that derivation to be built prior to the - present derivation; its default output path is put in the - environment variable. - - - Lists of the previous types are also allowed. They are simply - concatenated, separated by spaces. - - - `true` is passed as the string `1`, `false` and `null` are - passed as an empty string. - - - The optional attribute `args` specifies command-line arguments to be - passed to the builder. It should be a list. - - - The optional attribute `outputs` specifies a list of symbolic - outputs of the derivation. By default, a derivation produces a - single output path, denoted as `out`. However, derivations can - produce multiple output paths. This is useful because it allows - outputs to be downloaded or garbage-collected separately. For - instance, imagine a library package that provides a dynamic library, - header files, and documentation. A program that links against the - library doesn’t need the header files and documentation at runtime, - and it doesn’t need the documentation at build time. Thus, the - library package could specify: - - ```nix - outputs = [ "lib" "headers" "doc" ]; - ``` - - This will cause Nix to pass environment variables `lib`, `headers` - and `doc` to the builder containing the intended store paths of each - output. The builder would typically do something like - - ```bash - ./configure \ - --libdir=$lib/lib \ - --includedir=$headers/include \ - --docdir=$doc/share/doc - ``` - - for an Autoconf-style package. You can refer to each output of a - derivation by selecting it as an attribute, e.g. - - ```nix - buildInputs = [ pkg.lib pkg.headers ]; - ``` - - The first element of `outputs` determines the *default output*. - Thus, you could also write - - ```nix - buildInputs = [ pkg pkg.headers ]; - ``` - - since `pkg` is equivalent to `pkg.lib`. - -The function `mkDerivation` in the Nixpkgs standard environment is a -wrapper around `derivation` that adds a default value for `system` and -always uses Bash as the builder, to which the supplied builder is passed -as a command-line argument. See the Nixpkgs manual for details. - -The builder is executed as follows: - - - A temporary directory is created under the directory specified by - `TMPDIR` (default `/tmp`) where the build will take place. The - current directory is changed to this directory. - - - The environment is cleared and set to the derivation attributes, as - specified above. - - - In addition, the following variables are set: - - - `NIX_BUILD_TOP` contains the path of the temporary directory for - this build. - - - Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the - temporary directory. This is to prevent the builder from - accidentally writing temporary files anywhere else. Doing so - might cause interference by other processes. - - - `PATH` is set to `/path-not-set` to prevent shells from - initialising it to their built-in default value. - - - `HOME` is set to `/homeless-shelter` to prevent programs from - using `/etc/passwd` or the like to find the user's home - directory, which could cause impurity. Usually, when `HOME` is - set, it is used as the location of the home directory, even if - it points to a non-existent path. - - - `NIX_STORE` is set to the path of the top-level Nix store - directory (typically, `/nix/store`). - - - For each output declared in `outputs`, the corresponding - environment variable is set to point to the intended path in the - Nix store for that output. Each output path is a concatenation - of the cryptographic hash of all build inputs, the `name` - attribute and the output name. (The output name is omitted if - it’s `out`.) - - - If an output path already exists, it is removed. Also, locks are - acquired to prevent multiple Nix instances from performing the same - build at the same time. - - - A log of the combined standard output and error is written to - `/nix/var/log/nix`. - - - The builder is executed with the arguments specified by the - attribute `args`. If it exits with exit code 0, it is considered to - have succeeded. - - - The temporary directory is removed (unless the `-K` option was - specified). - - - If the build was successful, Nix scans each output path for - references to input paths by looking for the hash parts of the input - paths. Since these are potential runtime dependencies, Nix registers - them as dependencies of the output paths. - - - After the build, Nix sets the last-modified timestamp on all files - in the build result to 1 (00:00:01 1/1/1970 UTC), sets the group to - the default group, and sets the mode of the file to 0444 or 0555 - (i.e., read-only, with execute permission enabled if the file was - originally executable). Note that possible `setuid` and `setgid` - bits are cleared. Setuid and setgid programs are not currently - supported by Nix. This is because the Nix archives used in - deployment have no concept of ownership information, and because it - makes the build result dependent on the user performing the build. +The most important built-in function is `derivation`, which is used to describe a single derivation: +a specification for running an executable on precisely defined input files to repeatably produce output files at uniquely determined file system paths. + +It takes as input an attribute set, the attributes of which specify the inputs to the process. +It outputs an attribute set, and produces a [store derivation] as a side effect of evaluation. + +[store derivation]: @docroot@/glossary.md#gloss-store-derivation + +## Input attributes + +### Required + +- [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string)) + + A symbolic name for the derivation. + It is added to the [store path] of the corresponding [store derivation] as well as to its [output paths](@docroot@/glossary.md#gloss-output-path). + + [store path]: @docroot@/glossary.md#gloss-store-path + + > **Example** + > + > ```nix + > derivation { + > name = "hello"; + > # ... + > } + > ``` + > + > The store derivation's path will be `/nix/store/-hello.drv`. + > The [output](#attr-outputs) paths will be of the form `/nix/store/-hello[-]` + +- [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) + + The system type on which the [`builder`](#attr-builder) executable is meant to be run. + + A necessary condition for Nix to build derivations locally is that the `system` attribute matches the current [`system` configuration option]. + It can automatically [build on other platforms](../advanced-topics/distributed-builds.md) by forwarding build requests to other machines. + + [`system` configuration option]: @docroot@/command-ref/conf-file.md#conf-system + + > **Example** + > + > Declare a derivation to be built on a specific system type: + > + > ```nix + > derivation { + > # ... + > system = "x86_64-linux"; + > # ... + > } + > ``` + + > **Example** + > + > Declare a derivation to be built on the system type that evaluates the expression: + > + > ```nix + > derivation { + > # ... + > system = builtins.currentSystem; + > # ... + > } + > ``` + > + > [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. + +- [`builder`]{#attr-builder} ([Path](@docroot@/language/values.md#type-path) | [String](@docroot@/language/values.md#type-string)) + + Path to an executable that will perform the build. + + > **Example** + > + > Use the file located at `/bin/bash` as the builder executable: + > + > ```nix + > derivation { + > # ... + > builder = "/bin/bash"; + > # ... + > }; + > ``` + + + + > **Example** + > + > Copy a local file to the Nix store for use as the builder executable: + > + > ```nix + > derivation { + > # ... + > builder = ./builder.sh; + > # ... + > }; + > ``` + + + + > **Example** + > + > Use a file from another derivation as the builder executable: + > + > ```nix + > let pkgs = import {}; in + > derivation { + > # ... + > builder = "${pkgs.python}/bin/python"; + > # ... + > }; + > ``` + +### Optional + +- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) + + Default: `[ ]` + + Command-line arguments to be passed to the [`builder`](#attr-builder) executable. + + > **Example** + > + > Pass arguments to Bash to interpret a shell command: + > + > ```nix + > derivation { + > # ... + > builder = "/bin/bash"; + > args = [ "-c" "echo hello world > $out" ]; + > # ... + > }; + > ``` + +- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) + + Default: `[ "out" ]` + + Symbolic outputs of the derivation. + Each output name is passed to the [`builder`](#attr-builder) executable as an environment variable with its value set to the corresponding [store path]. + + By default, a derivation produces a single output called `out`. + However, derivations can produce multiple outputs. + This allows the associated [store objects](@docroot@/glossary.md#gloss-store-object) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. + + > **Example** + > + > Imagine a library package that provides a dynamic library, header files, and documentation. + > A program that links against such a library doesn’t need the header files and documentation at runtime, and it doesn’t need the documentation at build time. + > Thus, the library package could specify: + > + > ```nix + > derivation { + > # ... + > outputs = [ "lib" "dev" "doc" ]; + > # ... + > } + > ``` + > + > This will cause Nix to pass environment variables `lib`, `dev`, and `doc` to the builder containing the intended store paths of each output. + > The builder would typically do something like + > + > ```bash + > ./configure \ + > --libdir=$lib/lib \ + > --includedir=$dev/include \ + > --docdir=$doc/share/doc + > ``` + > + > for an Autoconf-style package. + + The name of an output is combined with the name of the derivation to create the name part of the output's store path, unless it is `out`, in which case just the name of the derivation is used. + + > **Example** + > + > + > ```nix + > derivation { + > name = "example"; + > outputs = [ "lib" "dev" "doc" "out" ]; + > # ... + > } + > ``` + > + > The store derivation path will be `/nix/store/-example.drv`. + > The output paths will be + > - `/nix/store/-example-lib` + > - `/nix/store/-example-dev` + > - `/nix/store/-example-doc` + > - `/nix/store/-example` + + You can refer to each output of a derivation by selecting it as an attribute. + The first element of `outputs` determines the *default output* and ends up at the top-level. + + > **Example** + > + > Select an output by attribute name: + > + > ```nix + > let + > myPackage = derivation { + > name = "example"; + > outputs = [ "lib" "dev" "doc" "out" ]; + > # ... + > }; + > in myPackage.dev + > ``` + > + > Since `lib` is the first output, `myPackage` is equivalent to `myPackage.lib`. + + + +- See [Advanced Attributes](./advanced-attributes.md) for more, infrequently used, optional attributes. + + + +- Every other attribute is passed as an environment variable to the builder. + Attribute values are translated to environment variables as follows: + + - Strings are passed unchanged. + + - Integral numbers are converted to decimal notation. + + - Floating point numbers are converted to simple decimal or scientific notation with a preset precision. + + - A *path* (e.g., `../foo/sources.tar`) causes the referenced file + to be copied to the store; its location in the store is put in + the environment variable. The idea is that all sources should + reside in the Nix store, since all inputs to a derivation should + reside in the Nix store. + + - A *derivation* causes that derivation to be built prior to the + present derivation. The environment variable is set to the [store path] of the derivation's default [output](#attr-outputs). + + - Lists of the previous types are also allowed. They are simply + concatenated, separated by spaces. + + - `true` is passed as the string `1`, `false` and `null` are + passed as an empty string. + + + +## Builder execution + +The [`builder`](#attr-builder) is executed as follows: + +- A temporary directory is created under the directory specified by + `TMPDIR` (default `/tmp`) where the build will take place. The + current directory is changed to this directory. + +- The environment is cleared and set to the derivation attributes, as + specified above. + +- In addition, the following variables are set: + + - `NIX_BUILD_TOP` contains the path of the temporary directory for + this build. + + - Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the + temporary directory. This is to prevent the builder from + accidentally writing temporary files anywhere else. Doing so + might cause interference by other processes. + + - `PATH` is set to `/path-not-set` to prevent shells from + initialising it to their built-in default value. + + - `HOME` is set to `/homeless-shelter` to prevent programs from + using `/etc/passwd` or the like to find the user's home + directory, which could cause impurity. Usually, when `HOME` is + set, it is used as the location of the home directory, even if + it points to a non-existent path. + + - `NIX_STORE` is set to the path of the top-level Nix store + directory (typically, `/nix/store`). + + - `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs` + is set to `true` for the dervation. A detailed explanation of this + behavior can be found in the + [section about structured attrs](./advanced-attributes.md#adv-attr-structuredAttrs). + + - For each output declared in `outputs`, the corresponding + environment variable is set to point to the intended path in the + Nix store for that output. Each output path is a concatenation + of the cryptographic hash of all build inputs, the `name` + attribute and the output name. (The output name is omitted if + it’s `out`.) + +- If an output path already exists, it is removed. Also, locks are + acquired to prevent multiple Nix instances from performing the same + build at the same time. + +- A log of the combined standard output and error is written to + `/nix/var/log/nix`. + +- The builder is executed with the arguments specified by the + attribute `args`. If it exits with exit code 0, it is considered to + have succeeded. + +- The temporary directory is removed (unless the `-K` option was + specified). + +- If the build was successful, Nix scans each output path for + references to input paths by looking for the hash parts of the input + paths. Since these are potential runtime dependencies, Nix registers + them as dependencies of the output paths. + +- After the build, Nix sets the last-modified timestamp on all files + in the build result to 1 (00:00:01 1/1/1970 UTC), sets the group to + the default group, and sets the mode of the file to 0444 or 0555 + (i.e., read-only, with execute permission enabled if the file was + originally executable). Note that possible `setuid` and `setgid` + bits are cleared. Setuid and setgid programs are not currently + supported by Nix. This is because the Nix archives used in + deployment have no concept of ownership information, and because it + makes the build result dependent on the user performing the build. diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md new file mode 100644 index 00000000000..03b3f9d91b1 --- /dev/null +++ b/doc/manual/src/language/import-from-derivation.md @@ -0,0 +1,139 @@ +# Import From Derivation + +The value of a Nix expression can depend on the contents of a [store object](@docroot@/glossary.md#gloss-store-object). + +Passing an expression `expr` that evaluates to a [store path](@docroot@/glossary.md#gloss-store-path) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): + +- [`import`](./builtins.md#builtins-import)` expr` +- [`builtins.readFile`](./builtins.md#builtins-readFile)` expr` +- [`builtins.readFileType`](./builtins.md#builtins-readFileType)` expr` +- [`builtins.readDir`](./builtins.md#builtins-readDir)` expr` +- [`builtins.pathExists`](./builtins.md#builtins-pathExists)` expr` +- [`builtins.filterSource`](./builtins.md#builtins-filterSource)` f expr` +- [`builtins.path`](./builtins.md#builtins-path)` { path = expr; }` +- [`builtins.hashFile`](./builtins.md#builtins-hashFile)` t expr` +- `builtins.scopedImport x drv` + +When the store path needs to be accessed, evaluation will be paused, the corresponding store object [realised], and then evaluation resumed. + +[realised]: @docroot@/glossary.md#gloss-realise + +This has performance implications: +Evaluation can only finish when all required store objects are realised. +Since the Nix language evaluator is sequential, it only finds store paths to read from one at a time. +While realisation is always parallel, in this case it cannot be done for all required store paths at once, and is therefore much slower than otherwise. + +Realising store objects during evaluation can be disabled by setting [`allow-import-from-derivation`](../command-ref/conf-file.md#conf-allow-import-from-derivation) to `false`. +Without IFD it is ensured that evaluation is complete and Nix can produce a build plan before starting any realisation. + +## Example + +In the following Nix expression, the inner derivation `drv` produces a file with contents `hello`. + +```nix +# IFD.nix +let + drv = derivation { + name = "hello"; + builder = "/bin/sh"; + args = [ "-c" "echo -n hello > $out" ]; + system = builtins.currentSystem; + }; +in "${builtins.readFile drv} world" +``` + +```shellSession +nix-instantiate IFD.nix --eval --read-write-mode +``` + +``` +building '/nix/store/348q1cal6sdgfxs8zqi9v8llrsn4kqkq-hello.drv'... +"hello world" +``` + +The contents of the derivation's output have to be [realised] before they can be read with [`readFile`](./builtins.md#builtins-readFile). +Only then evaluation can continue to produce the final result. + +## Illustration + +As a first approximation, the following data flow graph shows how evaluation and building are interleaved, if the value of a Nix expression depends on realising a [store object]. +Boxes are data structures, arrow labels are transformations. + +``` ++----------------------+ +------------------------+ +| Nix evaluator | | Nix store | +| .----------------. | | | +| | Nix expression | | | | +| '----------------' | | | +| | | | | +| evaluate | | | +| | | | | +| V | | | +| .------------. | | .------------------. | +| | derivation |----|-instantiate-|->| store derivation | | +| '------------' | | '------------------' | +| | | | | +| | | realise | +| | | | | +| | | V | +| .----------------. | | .--------------. | +| | Nix expression |<-|----read-----|----| store object | | +| '----------------' | | '--------------' | +| | | | | +| evaluate | | | +| | | | | +| V | | | +| .------------. | | | +| | value | | | | +| '------------' | | | ++----------------------+ +------------------------+ +``` + +In more detail, the following sequence diagram shows how the expression is evaluated step by step, and where evaluation is blocked to wait for the build output to appear. + +``` +.-------. .-------------. .---------. +|Nix CLI| |Nix evaluator| |Nix store| +'-------' '-------------' '---------' + | | | + |evaluate IFD.nix| | + |--------------->| | + | | | + | evaluate `"${readFile drv} world"` | + | | | + | evaluate `readFile drv` | + | | | + | evaluate `drv` as string | + | | | + | |instantiate /nix/store/...-hello.drv| + | |----------------------------------->| + | : | + | : realise /nix/store/...-hello.drv | + | :----------------------------------->| + | : | + | |--------. + | : | | + | (evaluation blocked) | echo hello > $out + | : | | + | |<-------' + | : /nix/store/...-hello | + | |<-----------------------------------| + | | | + | resume `readFile /nix/store/...-hello` | + | | | + | | readFile /nix/store/...-hello | + | |----------------------------------->| + | | | + | | hello | + | |<-----------------------------------| + | | | + | resume `"${"hello"} world"` | + | | | + | resume `"hello world"` | + | | | + | "hello world" | | + |<---------------| | +.-------. .-------------. .---------. +|Nix CLI| |Nix evaluator| |Nix store| +'-------' '-------------' '---------' +``` diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md index 29950a52d38..a26e43a05da 100644 --- a/doc/manual/src/language/index.md +++ b/doc/manual/src/language/index.md @@ -83,7 +83,8 @@ This is an incomplete overview of language features, by example. - A multi-line string. Strips common prefixed whitespace. Evaluates to `"multi\n line\n string"`. + + A multi-line string. Strips common prefixed whitespace. Evaluates to `"multi\n line\n  string"`. diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index f8382ae196e..e9cbb5c9242 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -25,7 +25,7 @@ | Inequality | *expr* `!=` *expr* | none | 11 | | Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | | Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | -| [Logical implication] | *bool* `->` *bool* | none | 14 | +| [Logical implication] | *bool* `->` *bool* | right | 14 | [string]: ./values.md#type-string [path]: ./values.md#type-path @@ -35,6 +35,8 @@ ## Attribute selection +> **Syntax** +> > *attrset* `.` *attrpath* \[ `or` *expr* \] Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*. @@ -42,21 +44,29 @@ If the attribute doesn’t exist, return the *expr* after `or` if provided, othe An attribute path is a dot-separated list of [attribute names](./values.md#attribute-set). +> **Syntax** +> > *attrpath* = *name* [ `.` *name* ]... [Attribute selection]: #attribute-selection ## Has attribute +> **Syntax** +> > *attrset* `?` *attrpath* Test whether [attribute set] *attrset* contains the attribute denoted by *attrpath*. The result is a [Boolean] value. +See also: [`builtins.hasAttr`](@docroot@/language/builtins.md#builtins-hasAttr) + [Boolean]: ./values.md#type-boolean [Has attribute]: #has-attribute +After evaluating *attrset* and *attrpath*, the computational complexity is O(log(*n*)) for *n* attributes in the *attrset* + ## Arithmetic Numbers are type-compatible: @@ -70,6 +80,8 @@ The `+` operator is overloaded to also work on strings and paths. ## String concatenation +> **Syntax** +> > *string* `+` *string* Concatenate two [string]s and merge their string contexts. @@ -78,6 +90,8 @@ Concatenate two [string]s and merge their string contexts. ## Path concatenation +> **Syntax** +> > *path* `+` *path* Concatenate two [path]s. @@ -87,6 +101,8 @@ The result is a path. ## Path and string concatenation +> **Syntax** +> > *path* + *string* Concatenate *[path]* with *[string]*. @@ -100,6 +116,8 @@ The result is a path. ## String and path concatenation +> **Syntax** +> > *string* + *path* Concatenate *[string]* with *[path]*. @@ -117,6 +135,8 @@ The result is a string. ## Update +> **Syntax** +> > *attrset1* // *attrset2* Update [attribute set] *attrset1* with names and values from *attrset2*. diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index ddc6b823071..e999b287b9c 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -1,19 +1,12 @@ # String interpolation -String interpolation is a language feature where a [string], [path], or [attribute name] can contain expressions enclosed in `${ }` (dollar-sign with curly brackets). +String interpolation is a language feature where a [string], [path], or [attribute name][attribute set] can contain expressions enclosed in `${ }` (dollar-sign with curly brackets). -Such a string is an *interpolated string*, and an expression inside is an *interpolated expression*. - -Interpolated expressions must evaluate to one of the following: - -- a [string] -- a [path] -- a [derivation] +Such a construct is called *interpolated string*, and the expression inside is an [interpolated expression](#interpolated-expression). [string]: ./values.md#type-string [path]: ./values.md#type-path -[attribute name]: ./values.md#attribute-set -[derivation]: ../glossary.md#gloss-derivation +[attribute set]: ./values.md#attribute-set ## Examples @@ -70,13 +63,136 @@ you can instead write ### Attribute name -Attribute names can be created dynamically with string interpolation: + -```nix -let name = "foo"; in -{ - ${name} = "bar"; -} -``` +Attribute names can be interpolated strings. + +> **Example** +> +> ```nix +> let name = "foo"; in +> { ${name} = 123; } +> ``` +> +> { foo = 123; } - { foo = "bar"; } +Attributes can be selected with interpolated strings. + +> **Example** +> +> ```nix +> let name = "foo"; in +> { foo = 123; }.${name} +> ``` +> +> 123 + +# Interpolated expression + +An expression that is interpolated must evaluate to one of the following: + +- a [string] +- a [path] +- an [attribute set] that has a `__toString` attribute or an `outPath` attribute + + - `__toString` must be a function that takes the attribute set itself and returns a string + - `outPath` must be a string + + This includes [derivations](./derivations.md) or [flake inputs](@docroot@/command-ref/new-cli/nix3-flake.md#flake-inputs) (experimental). + +A string interpolates to itself. + +A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](../glossary.md#gloss-store-object). + +[store path]: ../glossary.md#gloss-store-path + +> **Example** +> +> ```console +> $ mkdir foo +> ``` +> +> Reference the empty directory in an interpolated expression: +> +> ```nix +> "${./foo}" +> ``` +> +> "/nix/store/2hhl2nz5v0khbn06ys82nrk99aa1xxdw-foo" + +A derivation interpolates to the [store path] of its first [output](./derivations.md#attr-outputs). + +> **Example** +> +> ```nix +> let +> pkgs = import {}; +> in +> "${pkgs.hello}" +> ``` +> +> "/nix/store/4xpfqf29z4m8vbhrqcz064wfmb46w5r7-hello-2.12.1" + +An attribute set interpolates to the return value of the function in the `__toString` applied to the attribute set itself. + +> **Example** +> +> ```nix +> let +> a = { +> value = 1; +> __toString = self: toString (self.value + 1); +> }; +> in +> "${a}" +> ``` +> +> "2" + +An attribute set also interpolates to the value of its `outPath` attribute. + +> **Example** +> +> ```nix +> let +> a = { outPath = "foo"; }; +> in +> "${a}" +> ``` +> +> "foo" + +If both `__toString` and `outPath` are present in an attribute set, `__toString` takes precedence. + +> **Example** +> +> ```nix +> let +> a = { __toString = _: "yes"; outPath = throw "no"; }; +> in +> "${a}" +> ``` +> +> "yes" + +If neither is present, an error is thrown. + +> **Example** +> +> ```nix +> let +> a = {}; +> in +> "${a}" +> ``` +> +> error: cannot coerce a set to a string +> +> at «string»:4:2: +> +> 3| in +> 4| "${a}" +> | ^ diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md index 2ae3e143a16..0bb6567463b 100644 --- a/doc/manual/src/language/values.md +++ b/doc/manual/src/language/values.md @@ -107,29 +107,24 @@ e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user whose home directory is `/home/edolstra`. - Paths can also be specified between angle brackets, e.g. - ``. This means that the directories listed in the - environment variable `NIX_PATH` will be searched for the given file - or directory name. - - When an [interpolated string][string interpolation] evaluates to a path, the path is first copied into the Nix store and the resulting string is the [store path] of the newly created [store object]. - - [store path]: ../glossary.md#gloss-store-path - [store object]: ../glossary.md#gloss-store-object - For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` in the current directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. Note that the Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. For example, assume you used a file path in an interpolated string during a `nix repl` session. - Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new store path, since Nix might not re-read the file contents. + Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. - Paths themselves, except those in angle brackets (`< >`), support [string interpolation]. + [store path]: ../glossary.md#gloss-store-path + + Paths can include [string interpolation] and can themselves be [interpolated in other expressions]. + [interpolated in other expressions]: ./string-interpolation.md#interpolated-expressions At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division operation. `./a.${foo}/b.${bar}` is a path. + [Lookup paths](./constructs/lookup-path.md) such as `` resolve to path values. + - Boolean *Booleans* with values `true` and `false`. @@ -167,13 +162,17 @@ An attribute set is a collection of name-value-pairs (called *attributes*) enclo An attribute name can be an identifier or a [string](#string). An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`). +> **Syntax** +> > *name* = *identifier* | *string* \ > *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` Names and values are separated by an equal sign (`=`). Each value is an arbitrary expression terminated by a semicolon (`;`). -> *attrset* = `{` [ *name* `=` *expr* `;` `]`... `}` +> **Syntax** +> +> *attrset* = `{` [ *name* `=` *expr* `;` ]... `}` Attributes can appear in any order. An attribute name may only occur once. diff --git a/doc/manual/src/package-management/basic-package-mgmt.md b/doc/manual/src/package-management/basic-package-mgmt.md deleted file mode 100644 index 07b92fb7646..00000000000 --- a/doc/manual/src/package-management/basic-package-mgmt.md +++ /dev/null @@ -1,179 +0,0 @@ -# Basic Package Management - -The main command for package management is -[`nix-env`](../command-ref/nix-env.md). You can use it to install, -upgrade, and erase packages, and to query what packages are installed -or are available for installation. - -In Nix, different users can have different “views” on the set of -installed applications. That is, there might be lots of applications -present on the system (possibly in many different versions), but users -can have a specific selection of those active — where “active” just -means that it appears in a directory in the user’s `PATH`. Such a view -on the set of installed applications is called a *user environment*, -which is just a directory tree consisting of symlinks to the files of -the active applications. - -Components are installed from a set of *Nix expressions* that tell Nix -how to build those packages, including, if necessary, their -dependencies. There is a collection of Nix expressions called the -Nixpkgs package collection that contains packages ranging from basic -development stuff such as GCC and Glibc, to end-user applications like -Mozilla Firefox. (Nix is however not tied to the Nixpkgs package -collection; you could write your own Nix expressions based on Nixpkgs, -or completely new ones.) - -You can manually download the latest version of Nixpkgs from -. However, it’s much more -convenient to use the Nixpkgs [*channel*](../command-ref/nix-channel.md), since it makes -it easy to stay up to date with new versions of Nixpkgs. Nixpkgs is -automatically added to your list of “subscribed” channels when you -install Nix. If this is not the case for some reason, you can add it -as follows: - -```console -$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable -$ nix-channel --update -``` - -> **Note** -> -> On NixOS, you’re automatically subscribed to a NixOS channel -> corresponding to your NixOS major release (e.g. -> ). A NixOS channel is identical -> to the Nixpkgs channel, except that it contains only Linux binaries -> and is updated only if a set of regression tests succeed. - -You can view the set of available packages in Nixpkgs: - -```console -$ nix-env --query --available --attr-path -nixpkgs.aterm aterm-2.2 -nixpkgs.bash bash-3.0 -nixpkgs.binutils binutils-2.15 -nixpkgs.bison bison-1.875d -nixpkgs.blackdown blackdown-1.4.2 -nixpkgs.bzip2 bzip2-1.0.2 -… -``` - -The flag `-q` specifies a query operation, `-a` means that you want -to show the “available” (i.e., installable) packages, as opposed to the -installed packages, and `-P` prints the attribute paths that can be used -to unambiguously select a package for installation (listed in the first column). -If you downloaded Nixpkgs yourself, or if you checked it out from GitHub, -then you need to pass the path to your Nixpkgs tree using the `-f` flag: - -```console -$ nix-env --query --available --attr-path --file /path/to/nixpkgs -aterm aterm-2.2 -bash bash-3.0 -… -``` - -where */path/to/nixpkgs* is where you’ve unpacked or checked out -Nixpkgs. - -You can filter the packages by name: - -```console -$ nix-env --query --available --attr-path firefox -nixpkgs.firefox-esr firefox-91.3.0esr -nixpkgs.firefox firefox-94.0.1 -``` - -and using regular expressions: - -```console -$ nix-env --query --available --attr-path 'firefox.*' -``` - -It is also possible to see the *status* of available packages, i.e., -whether they are installed into the user environment and/or present in -the system: - -```console -$ nix-env --query --available --attr-path --status -… --PS nixpkgs.bash bash-3.0 ---S nixpkgs.binutils binutils-2.15 -IPS nixpkgs.bison bison-1.875d -… -``` - -The first character (`I`) indicates whether the package is installed in -your current user environment. The second (`P`) indicates whether it is -present on your system (in which case installing it into your user -environment would be a very quick operation). The last one (`S`) -indicates whether there is a so-called *substitute* for the package, -which is Nix’s mechanism for doing binary deployment. It just means that -Nix knows that it can fetch a pre-built package from somewhere -(typically a network server) instead of building it locally. - -You can install a package using `nix-env --install --attr `. For instance, - -```console -$ nix-env --install --attr nixpkgs.subversion -``` - -will install the package called `subversion` from `nixpkgs` channel (which is, of course, the -[Subversion version management system](http://subversion.tigris.org/)). - -> **Note** -> -> When you ask Nix to install a package, it will first try to get it in -> pre-compiled form from a *binary cache*. By default, Nix will use the -> binary cache ; it contains binaries for most -> packages in Nixpkgs. Only if no binary is available in the binary -> cache, Nix will build the package from source. So if `nix-env -> -iA nixpkgs.subversion` results in Nix building stuff from source, then either -> the package is not built for your platform by the Nixpkgs build -> servers, or your version of Nixpkgs is too old or too new. For -> instance, if you have a very recent checkout of Nixpkgs, then the -> Nixpkgs build servers may not have had a chance to build everything -> and upload the resulting binaries to . The -> Nixpkgs channel is only updated after all binaries have been uploaded -> to the cache, so if you stick to the Nixpkgs channel (rather than -> using a Git checkout of the Nixpkgs tree), you will get binaries for -> most packages. - -Naturally, packages can also be uninstalled. Unlike when installing, you will -need to use the derivation name (though the version part can be omitted), -instead of the attribute path, as `nix-env` does not record which attribute -was used for installing: - -```console -$ nix-env --uninstall subversion -``` - -Upgrading to a new version is just as easy. If you have a new release of -Nix Packages, you can do: - -```console -$ nix-env --upgrade --attr nixpkgs.subversion -``` - -This will *only* upgrade Subversion if there is a “newer” version in the -new set of Nix expressions, as defined by some pretty arbitrary rules -regarding ordering of version numbers (which generally do what you’d -expect of them). To just unconditionally replace Subversion with -whatever version is in the Nix expressions, use `-i` instead of `-u`; -`-i` will remove whatever version is already installed. - -You can also upgrade all packages for which there are newer versions: - -```console -$ nix-env --upgrade -``` - -Sometimes it’s useful to be able to ask what `nix-env` would do, without -actually doing it. For instance, to find out what packages would be -upgraded by `nix-env --upgrade `, you can do - -```console -$ nix-env --upgrade --dry-run -(dry run; not doing anything) -upgrading `libxslt-1.1.0' to `libxslt-1.1.10' -upgrading `graphviz-1.10' to `graphviz-1.12' -upgrading `coreutils-5.0' to `coreutils-5.2.1' -``` diff --git a/doc/manual/src/package-management/s3-substituter.md b/doc/manual/src/package-management/s3-substituter.md deleted file mode 100644 index d8a1d9105b4..00000000000 --- a/doc/manual/src/package-management/s3-substituter.md +++ /dev/null @@ -1,115 +0,0 @@ -# Serving a Nix store via S3 - -Nix has [built-in support](@docroot@/command-ref/new-cli/nix3-help-stores.md#s3-binary-cache-store) -for storing and fetching store paths from -Amazon S3 and S3-compatible services. This uses the same *binary* -cache mechanism that Nix usually uses to fetch prebuilt binaries from -[cache.nixos.org](https://cache.nixos.org/). - -In this example we will use the bucket named `example-nix-cache`. - -## Anonymous Reads to your S3-compatible binary cache - -If your binary cache is publicly accessible and does not require -authentication, the simplest and easiest way to use Nix with your S3 -compatible binary cache is to use the HTTP URL for that cache. - -For AWS S3 the binary cache URL for example bucket will be exactly - or -. For S3 compatible binary caches, consult that -cache's documentation. - -Your bucket will need the following bucket policy: - -```json -{ - "Id": "DirectReads", - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "AllowDirectReads", - "Action": [ - "s3:GetObject", - "s3:GetBucketLocation" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:s3:::example-nix-cache", - "arn:aws:s3:::example-nix-cache/*" - ], - "Principal": "*" - } - ] -} -``` - -## Authenticated Reads to your S3 binary cache - -For AWS S3 the binary cache URL for example bucket will be exactly -. - -Nix will use the [default credential provider -chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html) -for authenticating requests to Amazon S3. - -Nix supports authenticated reads from Amazon S3 and S3 compatible binary -caches. - -Your bucket will need a bucket policy allowing the desired users to -perform the `s3:GetObject` and `s3:GetBucketLocation` action on all -objects in the bucket. The [anonymous policy given -above](#anonymous-reads-to-your-s3-compatible-binary-cache) can be -updated to have a restricted `Principal` to support this. - -## Authenticated Writes to your S3-compatible binary cache - -Nix support fully supports writing to Amazon S3 and S3 compatible -buckets. The binary cache URL for our example bucket will be -. - -Nix will use the [default credential provider -chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html) -for authenticating requests to Amazon S3. - -Your account will need the following IAM policy to upload to the cache: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "UploadToCache", - "Effect": "Allow", - "Action": [ - "s3:AbortMultipartUpload", - "s3:GetBucketLocation", - "s3:GetObject", - "s3:ListBucket", - "s3:ListBucketMultipartUploads", - "s3:ListMultipartUploadParts", - "s3:PutObject" - ], - "Resource": [ - "arn:aws:s3:::example-nix-cache", - "arn:aws:s3:::example-nix-cache/*" - ] - } - ] -} -``` - -## Examples - -To upload with a specific credential profile for Amazon S3: - -```console -$ nix copy nixpkgs.hello \ - --to 's3://example-nix-cache?profile=cache-upload®ion=eu-west-2' -``` - -To upload to an S3-compatible binary cache: - -```console -$ nix copy nixpkgs.hello --to \ - 's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com' -``` diff --git a/doc/manual/src/release-notes/release-notes.md b/doc/manual/src/release-notes/release-notes.md index b05d5ee0a2d..cc805e63116 100644 --- a/doc/manual/src/release-notes/release-notes.md +++ b/doc/manual/src/release-notes/release-notes.md @@ -1 +1,12 @@ # Nix Release Notes + +Nix has a release cycle of roughly 6 weeks. +Notable changes and additions are announced in the release notes for each version. + +Bugfixes can be backported on request to previous Nix releases. +We typically backport only as far back as the Nix version used in the latest NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). + +Backports never skip releases. +If a feature is backported to version `x.y`, it must also be available in version `x.(y+1)`. +This ensures that upgrading from an older version with backports is still safe and no backported functionality will go missing. + diff --git a/doc/manual/src/release-notes/rl-2.12.md b/doc/manual/src/release-notes/rl-2.12.md index e2045d7bf5b..e1e3efe1a8a 100644 --- a/doc/manual/src/release-notes/rl-2.12.md +++ b/doc/manual/src/release-notes/rl-2.12.md @@ -2,7 +2,6 @@ * On Linux, Nix can now run builds in a user namespace where they run as root (UID 0) and have 65,536 UIDs available. - This is primarily useful for running containers such as `systemd-nspawn` inside a Nix build. For an example, see [`tests/systemd-nspawn/nix`][nspawn]. diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md new file mode 100644 index 00000000000..ba6eb9c64a4 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.19.md @@ -0,0 +1,77 @@ +# Release 2.19 (2023-11-17) + +- The experimental `nix` command can now act as a [shebang interpreter](@docroot@/command-ref/new-cli/nix.md#shebang-interpreter) + by appending the contents of any `#! nix` lines and the script's location into a single call. + +- [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters. + +- [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`). + +- The experimental feature `repl-flake` is no longer needed, as its functionality is now part of the `flakes` experimental feature. To get the previous behavior, use the `--file/--expr` flags accordingly. + +- There is a new flake installable syntax `flakeref#.attrPath` where the "." prefix specifies that `attrPath` is interpreted from the root of the flake outputs, with no searching of default attribute prefixes like `packages.` or `legacyPackages.`. + +- Nix adds `apple-virt` to the default system features on macOS systems that support virtualization. This is similar to what's done for the `kvm` system feature on Linux hosts. + +- Add a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash). + +- `nix-shell` shebang lines now support single-quoted arguments. + +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). + This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). + +- The interface for creating and updating lock files has been overhauled: + + - [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) only creates lock files and adds missing inputs now. + It will *never* update existing inputs. + + - [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) does the same, but *will* update inputs. + - Passing no arguments will update all inputs of the current flake, just like it already did. + - Passing input names as arguments will ensure only those are updated. This replaces the functionality of `nix flake lock --update-input` + - To operate on a flake outside the current directory, you must now pass `--flake path/to/flake`. + + - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. + They are superceded by `nix flake update`. + +- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + +- [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md) + (experimental) now returns a JSON map rather than JSON list. + The `path` field of each object has instead become the key in the outer map, since it is unique. + The `valid` field also goes away because we just use `null` instead. + + - Old way: + + ```json5 + [ + { + "path": "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15", + "valid": true, + // ... + }, + { + "path": "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path", + "valid": false + } + ] + ``` + + - New way + + ```json5 + { + "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15": { + // ... + }, + "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path": null, + } + ``` + + This makes it match `nix derivation show`, which also maps store paths to information. + +- When Nix is installed using the [binary installer](@docroot@/installation/installing-binary.md), in supported shells (Bash, Zsh, Fish) + [`XDG_DATA_DIRS`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#variables) is now populated with the path to the `/share` subdirectory of the current profile. + This means that command completion scripts, `.desktop` files, and similar artifacts installed via [`nix-env`](@docroot@/command-ref/nix-env.md) or [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) + (experimental) can be found by any program that follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). + +- A new command `nix store add` has been added. It replaces `nix store add-file` and `nix store add-path` which are now deprecated. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index c869b5e2fab..78ae99f4bb7 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1 +1,2 @@ # Release X.Y (202?-??-??) + diff --git a/doc/manual/src/architecture/file-system-object.md b/doc/manual/src/store/file-system-object.md similarity index 100% rename from doc/manual/src/architecture/file-system-object.md rename to doc/manual/src/store/file-system-object.md diff --git a/doc/manual/src/store/index.md b/doc/manual/src/store/index.md new file mode 100644 index 00000000000..8a53050629a --- /dev/null +++ b/doc/manual/src/store/index.md @@ -0,0 +1,5 @@ +# Nix Store + +The *Nix store* is an abstraction to store immutable file system data (such as software packages) that can have dependencies on other such data. + +There are multiple implementations of Nix stores with different capabilities, such as the actual filesystem (`/nix/store`) or binary caches. diff --git a/doc/manual/src/store/store-object.md b/doc/manual/src/store/store-object.md new file mode 100644 index 00000000000..caf5657d1f0 --- /dev/null +++ b/doc/manual/src/store/store-object.md @@ -0,0 +1,10 @@ +## Store Object + +A Nix store is a collection of *store objects* with *references* between them. +A store object consists of + + - A [file system object](./file-system-object.md) as data + - A set of [store paths](./store-path.md) as references to other store objects + +Store objects are [immutable](https://en.wikipedia.org/wiki/Immutable_object): +Once created, they do not change until they are deleted. diff --git a/doc/manual/src/store/store-path.md b/doc/manual/src/store/store-path.md new file mode 100644 index 00000000000..b5ad0c654f4 --- /dev/null +++ b/doc/manual/src/store/store-path.md @@ -0,0 +1,69 @@ +# Store Path + +Nix implements references to [store objects](./index.md#store-object) as *store paths*. + +Think of a store path as an [opaque], [unique identifier]: +The only way to obtain store path is by adding or building store objects. +A store path will always reference exactly one store object. + +[opaque]: https://en.m.wikipedia.org/wiki/Opaque_data_type +[unique identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier + +Store paths are pairs of + +- A 20-byte digest for identification +- A symbolic name for people to read + +> **Example** +> +> - Digest: `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z` +> - Name: `firefox-33.1` + +To make store objects accessible to operating system processes, stores have to expose store objects through the file system. + +A store path is rendered to a file system path as the concatenation of + +- [Store directory](#store-directory) (typically `/nix/store`) +- Path separator (`/`) +- Digest rendered in a custom variant of [Base32](https://en.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters) +- Hyphen (`-`) +- Name + +> **Example** +> +> ``` +> /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1 +> |--------| |------------------------------| |----------| +> store directory digest name +> ``` + +## Store Directory + +Every [Nix store](./index.md) has a store directory. + +Not every store can be accessed through the file system. +But if the store has a file system representation, the store directory contains the store’s [file system objects], which can be addressed by [store paths](#store-path). + +[file system objects]: ./file-system-object.md + +This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in. + +> **Note** +> +> The store directory defaults to `/nix/store`, but is in principle arbitrary. + +It is important which store a given store object belongs to: +Files in the store object can contain store paths, and processes may read these paths. +Nix can only guarantee referential integrity if store paths do not cross store boundaries. + +Therefore one can only copy store objects to a different store if + +- The source and target stores' directories match + + or + +- The store object in question has no references, that is, contains no store paths + +One cannot copy a store object to a store with a different store directory. +Instead, it has to be rebuilt, together with all its dependencies. +It is in general not enough to replace the store directory string in file contents, as this may render executables unusable by invalidating their internal offsets or checksums. diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix index 9043dd8cdba..849832b2c6a 100644 --- a/doc/manual/utils.nix +++ b/doc/manual/utils.nix @@ -44,63 +44,6 @@ rec { optionalString = cond: string: if cond then string else ""; - showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }: - let - result = squash '' - - ${if useAnchors - then ''[`${name}`](#conf-${name})'' - else ''`${name}`''} - - ${indent " " body} - ''; - - experimentalFeatureNote = optionalString (experimentalFeature != null) '' - > **Warning** - > This setting is part of an - > [experimental feature](@docroot@/contributing/experimental-features.md). - - To change this setting, you need to make sure the corresponding experimental feature, - [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), - is enabled. - For example, include the following in [`nix.conf`](#): - - ``` - extra-experimental-features = ${experimentalFeature} - ${name} = ... - ``` - ''; - - # separate body to cleanly handle indentation - body = '' - ${description} - - ${experimentalFeatureNote} - - **Default:** ${showDefault documentDefault defaultValue} - - ${showAliases aliases} - ''; - - showDefault = documentDefault: defaultValue: - if documentDefault then - # a StringMap value type is specified as a string, but - # this shows the value type. The empty stringmap is `null` in - # JSON, but that converts to `{ }` here. - if defaultValue == "" || defaultValue == [] || isAttrs defaultValue - then "*empty*" - else if isBool defaultValue then - if defaultValue then "`true`" else "`false`" - else "`${toString defaultValue}`" - else "*machine-specific*"; - - showAliases = aliases: - optionalString (aliases != []) - "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}"; - - in result; - indent = prefix: s: concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s)); - - showSettings = args: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting args) settingsInfo)); } diff --git a/flake.lock b/flake.lock index 75b6ae6a70c..d11b468c54f 100644 --- a/flake.lock +++ b/flake.lock @@ -34,16 +34,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1695124524, - "narHash": "sha256-trXDytVCqf3KryQQQrHOZKUabu1/lB8/ndOAuZKQrOE=", - "owner": "edolstra", + "lastModified": 1700748986, + "narHash": "sha256-/nqLrNU297h3PCw4QyDpZKZEUHmialJdZW2ceYFobds=", + "owner": "NixOS", "repo": "nixpkgs", - "rev": "a3d30b525535e3158221abc1a957ce798ab159fe", + "rev": "9ba29e2346bc542e9909d1021e8fd7d4b3f64db0", "type": "github" }, "original": { - "owner": "edolstra", - "ref": "fix-aws-sdk-cpp", + "owner": "NixOS", + "ref": "nixos-23.05-small", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index dfb2dc1f8c7..5cf2a2419f7 100644 --- a/flake.nix +++ b/flake.nix @@ -1,8 +1,7 @@ { description = "The purely functional package manager"; - #inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; - inputs.nixpkgs.url = "github:edolstra/nixpkgs/fix-aws-sdk-cpp"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; @@ -12,7 +11,7 @@ let inherit (nixpkgs) lib; - officialRelease = false; + officialRelease = true; version = lib.fileContents ./.version + versionSuffix; versionSuffix = @@ -25,8 +24,11 @@ linuxSystems = linux32BitSystems ++ linux64BitSystems; darwinSystems = [ "x86_64-darwin" "aarch64-darwin" ]; systems = linuxSystems ++ darwinSystems; - - crossSystems = [ "armv6l-linux" "armv7l-linux" ]; + + crossSystems = [ + "armv6l-linux" "armv7l-linux" + "x86_64-freebsd13" "x86_64-netbsd" + ]; stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ]; @@ -57,44 +59,55 @@ # that would interfere with repo semantics. fileset.fileFilter (f: f.name != ".gitignore") ./.; + configureFiles = fileset.unions [ + ./.version + ./configure.ac + ./m4 + # TODO: do we really need README.md? It doesn't seem used in the build. + ./README.md + ]; + + topLevelBuildFiles = fileset.unions [ + ./local.mk + ./Makefile + ./Makefile.config.in + ./mk + ]; + + functionalTestFiles = fileset.unions [ + ./tests/functional + (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) + ]; + nixSrc = fileset.toSource { root = ./.; - fileset = fileset.intersect baseFiles ( - fileset.difference - (fileset.unions [ - ./.version - ./boehmgc-coroutine-sp-fallback.diff - ./bootstrap.sh - ./configure.ac - ./doc - ./local.mk - ./m4 - ./Makefile - ./Makefile.config.in - ./misc - ./mk - ./precompiled-headers.h - ./src - ./tests - ./COPYING - ./scripts/local.mk - (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) - # TODO: do we really need README.md? It doesn't seem used in the build. - ./README.md - ]) - (fileset.unions [ - # Removed file sets - ./tests/nixos - ./tests/installer - ]) - ); + fileset = fileset.intersect baseFiles (fileset.unions [ + configureFiles + topLevelBuildFiles + ./boehmgc-coroutine-sp-fallback.diff + ./doc + ./misc + ./precompiled-headers.h + ./src + ./tests/unit + ./COPYING + ./scripts/local.mk + functionalTestFiles + ]); }; # Memoize nixpkgs for different platforms for efficiency. nixpkgsFor = forAllSystems (system: let make-pkgs = crossSystem: stdenv: import nixpkgs { - inherit system crossSystem; + localSystem = { + inherit system; + }; + crossSystem = if crossSystem == null then null else { + system = crossSystem; + } // lib.optionalAttrs (crossSystem == "x86_64-freebsd13") { + useLLVM = true; + }; overlays = [ (overlayFor (p: p.${stdenv})) ]; @@ -149,6 +162,10 @@ testConfigureFlags = [ "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" + ] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ + "--enable-install-unit-tests" + "--with-check-bin-dir=${builtins.placeholder "check"}/bin" + "--with-check-lib-dir=${builtins.placeholder "check"}/lib" ]; internalApiDocsConfigureFlags = [ @@ -170,6 +187,7 @@ buildPackages.git buildPackages.mercurial # FIXME: remove? only needed for tests buildPackages.jq # Also for custom mdBook preprocessor. + buildPackages.openssh # only needed for tests (ssh-keygen) ] ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]; @@ -180,9 +198,9 @@ libarchive boost lowdown-nix + libsodium ] ++ lib.optionals stdenv.isLinux [libseccomp] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid; checkDeps = [ @@ -258,7 +276,14 @@ "-${client.version}-against-${daemon.version}"; inherit version; - src = nixSrc; + src = fileset.toSource { + root = ./.; + fileset = fileset.intersect baseFiles (fileset.unions [ + configureFiles + topLevelBuildFiles + functionalTestFiles + ]); + }; VERSION_SUFFIX = versionSuffix; @@ -268,7 +293,9 @@ enableParallelBuilding = true; - configureFlags = testConfigureFlags; # otherwise configure fails + configureFlags = + testConfigureFlags # otherwise configure fails + ++ [ "--disable-build" ]; dontBuild = true; doInstallCheck = true; @@ -276,7 +303,10 @@ mkdir -p $out ''; - installCheckPhase = "make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES"; + installCheckPhase = '' + mkdir -p src/nix-channel + make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES + ''; }; binaryTarball = nix: pkgs: @@ -376,7 +406,8 @@ src = nixSrc; VERSION_SUFFIX = versionSuffix; - outputs = [ "out" "dev" "doc" ]; + outputs = [ "out" "dev" "doc" ] + ++ lib.optional (currentStdenv.hostPlatform != currentStdenv.buildPlatform) "check"; nativeBuildInputs = nativeBuildDeps; buildInputs = buildDeps @@ -449,39 +480,13 @@ hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - passthru.perl-bindings = with final; perl.pkgs.toPerlModule (currentStdenv.mkDerivation { - name = "nix-perl-${version}"; - - src = self; - - nativeBuildInputs = - [ buildPackages.autoconf-archive - buildPackages.autoreconfHook - buildPackages.pkg-config - ]; - - buildInputs = - [ nix - curl - bzip2 - xz - pkgs.perl - boost - ] - ++ lib.optional (currentStdenv.isLinux || currentStdenv.isDarwin) libsodium - ++ lib.optional currentStdenv.isDarwin darwin.apple_sdk.frameworks.Security; - - configureFlags = [ - "--with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}" - "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}" - ]; - - enableParallelBuilding = true; - - postUnpack = "sourceRoot=$sourceRoot/perl"; - }); + passthru.perl-bindings = final.callPackage ./perl { + inherit fileset; + stdenv = currentStdenv; + }; meta.platforms = lib.platforms.unix; + meta.mainProgram = "nix"; }); lowdown-nix = with final; currentStdenv.mkDerivation rec { @@ -502,18 +507,6 @@ }; }; - nixos-lib = import (nixpkgs + "/nixos/lib") { }; - - # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests - runNixOSTestFor = system: test: nixos-lib.runTest { - imports = [ test ]; - hostPkgs = nixpkgsFor.${system}.native; - defaults = { - nixpkgs.pkgs = nixpkgsFor.${system}.native; - }; - _module.args.nixpkgs = nixpkgs; - }; - in { # A Nixpkgs overlay that overrides the 'nix' and # 'nix.perl-bindings' packages. @@ -620,49 +613,29 @@ }; # System tests. - tests.authorization = runNixOSTestFor "x86_64-linux" ./tests/nixos/authorization.nix; - - tests.remoteBuilds = runNixOSTestFor "x86_64-linux" ./tests/nixos/remote-builds.nix; - - tests.nix-copy-closure = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy-closure.nix; - - tests.nix-copy = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy.nix; + tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { - tests.nssPreload = runNixOSTestFor "x86_64-linux" ./tests/nixos/nss-preload.nix; - - tests.githubFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/github-flakes.nix; - - tests.sourcehutFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/sourcehut-flakes.nix; - - tests.tarballFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/tarball-flakes.nix; - - tests.containers = runNixOSTestFor "x86_64-linux" ./tests/nixos/containers/containers.nix; - - tests.setuid = lib.genAttrs - ["i686-linux" "x86_64-linux"] - (system: runNixOSTestFor system ./tests/nixos/setuid.nix); - - - # Make sure that nix-env still produces the exact same result - # on a particular version of Nixpkgs. - tests.evalNixpkgs = - with nixpkgsFor.x86_64-linux.native; - runCommand "eval-nixos" { buildInputs = [ nix ]; } - '' - type -p nix-env - # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. - time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages - [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]] - mkdir $out - ''; + # Make sure that nix-env still produces the exact same result + # on a particular version of Nixpkgs. + evalNixpkgs = + with nixpkgsFor.x86_64-linux.native; + runCommand "eval-nixos" { buildInputs = [ nix ]; } + '' + type -p nix-env + # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. + time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages + [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]] + mkdir $out + ''; - tests.nixpkgsLibTests = - forAllSystems (system: - import (nixpkgs + "/lib/tests/release.nix") - { pkgs = nixpkgsFor.${system}.native; - nixVersions = [ self.packages.${system}.nix ]; - } - ); + nixpkgsLibTests = + forAllSystems (system: + import (nixpkgs + "/lib/tests/release.nix") + { pkgs = nixpkgsFor.${system}.native; + nixVersions = [ self.packages.${system}.nix ]; + } + ); + }; metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { pkgs = nixpkgsFor.x86_64-linux.native; @@ -733,20 +706,29 @@ devShells = let makeShell = pkgs: stdenv: + let + canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform; + in with commonDeps { inherit pkgs; }; stdenv.mkDerivation { name = "nix"; - outputs = [ "out" "dev" "doc" ]; + outputs = [ "out" "dev" "doc" ] + ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "check"; nativeBuildInputs = nativeBuildDeps - ++ (lib.optionals stdenv.cc.isClang [ pkgs.bear pkgs.clang-tools ]); + ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear + ++ lib.optional + (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) + pkgs.buildPackages.clang-tools + ; buildInputs = buildDeps ++ propagatedDeps ++ awsDeps ++ checkDeps ++ internalApiDocsDeps; configureFlags = configureFlags - ++ testConfigureFlags ++ internalApiDocsConfigureFlags; + ++ testConfigureFlags ++ internalApiDocsConfigureFlags + ++ lib.optional (!canRunInstalled) "--disable-doc-gen"; enableParallelBuilding = true; diff --git a/local.mk b/local.mk index 6951c179e9d..3f3abb9f0d6 100644 --- a/local.mk +++ b/local.mk @@ -1,5 +1,3 @@ -clean-files += Makefile.config - GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch # Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers. ERROR_SWITCH_ENUM = -Werror=switch-enum diff --git a/m4/gcc_bug_80431.m4 b/m4/gcc_bug_80431.m4 new file mode 100644 index 00000000000..e42f0195614 --- /dev/null +++ b/m4/gcc_bug_80431.m4 @@ -0,0 +1,64 @@ +# Ensure that this bug is not present in the C++ toolchain we are using. +# +# URL for bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431 +# +# The test program is from that issue, with only a slight modification +# to set an exit status instead of printing strings. +AC_DEFUN([ENSURE_NO_GCC_BUG_80431], +[ + AC_MSG_CHECKING([that GCC bug 80431 is fixed]) + AC_LANG_PUSH(C++) + AC_RUN_IFELSE( + [AC_LANG_PROGRAM( + [[ + #include + + static bool a = true; + static bool b = true; + + struct Options { }; + + struct Option + { + Option(Options * options) + { + a = false; + } + + ~Option() + { + b = false; + } + }; + + struct MyOptions : Options { }; + + struct MyOptions2 : virtual MyOptions + { + Option foo{this}; + }; + ]], + [[ + { + MyOptions2 opts; + } + return (a << 1) | b; + ]])], + [status_80431=0], + [status_80431=$?], + [ + # Assume we're bug-free when cross-compiling + ]) + AC_LANG_POP(C++) + AS_CASE([$status_80431], + [0],[ + AC_MSG_RESULT(yes) + ], + [2],[ + AC_MSG_RESULT(no) + AC_MSG_ERROR(Cannot build Nix with C++ compiler with this bug) + ], + [ + AC_MSG_RESULT(unexpected result $status_80431: not expected failure with bug, ignoring) + ]) +]) diff --git a/maintainers/README.md b/maintainers/README.md index 0d520cb0cc5..ee97c1195d6 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -2,7 +2,7 @@ ## Motivation -The team's main responsibility is to set a direction for the development of Nix and ensure that the code is in good shape. +The team's main responsibility is to guide and direct the development of Nix and ensure that the code is in good shape. We aim to achieve this by improving the contributor experience and attracting more maintainers – that is, by helping other people contributing to Nix and eventually taking responsibility – in order to scale the development process to match users' needs. @@ -50,7 +50,9 @@ The team meets twice a week: 1. Code review on pull requests from [In review](#in-review). 2. Other chores and tasks. -Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw), and published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). +Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw). +Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are easy to find from both places. +[All meeting notes](https://discourse.nixos.org/search?expanded=true&q=Nix%20team%20meeting%20minutes%20%23%20%23dev%3Anix%20in%3Atitle%20order%3Alatest_topic) are published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). ## Project board protocol @@ -96,8 +98,10 @@ What constitutes a trivial pull request is up to maintainers' judgement. Pull requests and issues that are deemed important and controversial are discussed by the team during discussion meetings. This may be where the merit of the change itself or the implementation strategy is contested by a team member. +Whenever the discussion opens up questions about the process or this team's goals, this may indicate that the change is too large in scope. +In that case it is taken off the board to be reconsidered by the author or broken down into smaller pieces that are less far-reaching and can be reviewed independently. -As a general guideline, the order of items is determined as follows: +As a general guideline, the order of items to discuss is determined as follows: - Prioritise pull requests over issues diff --git a/mk/common-test.sh b/mk/common-test.sh index 0a2e4c1c2e2..2783d293bc4 100644 --- a/mk/common-test.sh +++ b/mk/common-test.sh @@ -1,11 +1,27 @@ -TESTS_ENVIRONMENT=("TEST_NAME=${test%.*}" 'NIX_REMOTE=') +# Remove overall test dir (at most one of the two should match) and +# remove file extension. +test_name=$(echo -n "$test" | sed \ + -e "s|^tests/unit/[^/]*/data/||" \ + -e "s|^tests/functional/||" \ + -e "s|\.sh$||" \ + ) + +TESTS_ENVIRONMENT=( + "TEST_NAME=$test_name" + 'NIX_REMOTE=' + 'PS4=+(${BASH_SOURCE[0]-$0}:$LINENO) ' +) : ${BASH:=/usr/bin/env bash} +run () { + cd "$(dirname $1)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -x -e -u -o pipefail $(basename $1) +} + init_test () { - cd tests && env "${TESTS_ENVIRONMENT[@]}" $BASH -e init.sh 2>/dev/null > /dev/null + run "$init" 2>/dev/null > /dev/null } run_test_proper () { - cd $(dirname $test) && env "${TESTS_ENVIRONMENT[@]}" $BASH -e $(basename $test) + run "$test" } diff --git a/mk/debug-test.sh b/mk/debug-test.sh index b5b628ecd68..52482c01e47 100755 --- a/mk/debug-test.sh +++ b/mk/debug-test.sh @@ -3,9 +3,12 @@ set -eu -o pipefail test=$1 +init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" -(init_test) +if [ -n "$init" ]; then + (init_test) +fi run_test_proper diff --git a/mk/lib.mk b/mk/lib.mk index e86a7f1a448..49abe986264 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -122,14 +122,15 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b $(foreach script, $(bin-scripts), $(eval programs-list += $(script))) $(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template)))) +install_test_init=tests/functional/init.sh $(foreach test, $(install-tests), \ - $(eval $(call run-install-test,$(test))) \ + $(eval $(call run-test,$(test),$(install_test_init))) \ $(eval installcheck: $(test).test)) $(foreach test-group, $(install-tests-groups), \ - $(eval $(call run-install-test-group,$(test-group))) \ + $(eval $(call run-test-group,$(test-group),$(install_test_init))) \ $(eval installcheck: $(test-group).test-group) \ $(foreach test, $($(test-group)-tests), \ - $(eval $(call run-install-test,$(test))) \ + $(eval $(call run-test,$(test),$(install_test_init))) \ $(eval $(test-group).test-group: $(test).test))) $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file)))))) diff --git a/mk/programs.mk b/mk/programs.mk index 1ee1d3fa5de..6235311e9e3 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -87,6 +87,6 @@ define build-program # Phony target to run this program (typically as a dependency of 'check'). .PHONY: $(1)_RUN $(1)_RUN: $$($(1)_PATH) - $(trace-test) $$($(1)_PATH) + $(trace-test) $$($(1)_ENV) $$($(1)_PATH) endef diff --git a/mk/run-test.sh b/mk/run-test.sh index 1a1d659304e..da9c5a473b4 100755 --- a/mk/run-test.sh +++ b/mk/run-test.sh @@ -8,6 +8,7 @@ yellow="" normal="" test=$1 +init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" @@ -21,7 +22,9 @@ if [ -t 1 ]; then fi run_test () { - (init_test 2>/dev/null > /dev/null) + if [ -n "$init" ]; then + (init_test 2>/dev/null > /dev/null) + fi log="$(run_test_proper 2>&1)" && status=0 || status=$? } diff --git a/mk/tests.mk b/mk/tests.mk index ec8128bdf87..bac9b704ad1 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -2,19 +2,22 @@ test-deps = -define run-install-test +define run-bash - .PHONY: $1.test - $1.test: $1 $(test-deps) - @env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null + .PHONY: $1 + $1: $2 + @env BASH=$(bash) $(bash) $3 < /dev/null - .PHONY: $1.test-debug - $1.test-debug: $1 $(test-deps) - @env BASH=$(bash) $(bash) mk/debug-test.sh $1 < /dev/null +endef + +define run-test + + $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1 $2)) + $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1 $2)) endef -define run-install-test-group +define run-test-group .PHONY: $1.test-group diff --git a/perl/Makefile b/perl/Makefile index c2c95f25503..832668dd155 100644 --- a/perl/Makefile +++ b/perl/Makefile @@ -1,6 +1,12 @@ makefiles = local.mk -GLOBAL_CXXFLAGS += -g -Wall -std=c++2a -I ../src +GLOBAL_CXXFLAGS += -g -Wall -std=c++2a + +# A convenience for concurrent development of Nix and its Perl bindings. +# Not needed in a standalone build of the Perl bindings. +ifneq ("$(wildcard ../src)", "") + GLOBAL_CXXFLAGS += -I ../src +endif -include Makefile.config diff --git a/perl/default.nix b/perl/default.nix new file mode 100644 index 00000000000..4687976a168 --- /dev/null +++ b/perl/default.nix @@ -0,0 +1,51 @@ +{ lib, fileset +, stdenv +, perl, perlPackages +, autoconf-archive, autoreconfHook, pkg-config +, nix, curl, bzip2, xz, boost, libsodium, darwin +}: + +perl.pkgs.toPerlModule (stdenv.mkDerivation { + name = "nix-perl-${nix.version}"; + + src = fileset.toSource { + root = ../.; + fileset = fileset.unions [ + ../.version + ../m4 + ../mk + ./MANIFEST + ./Makefile + ./Makefile.config.in + ./configure.ac + ./lib + ./local.mk + ]; + }; + + nativeBuildInputs = + [ autoconf-archive + autoreconfHook + pkg-config + ]; + + buildInputs = + [ nix + curl + bzip2 + xz + perl + boost + ] + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium + ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; + + configureFlags = [ + "--with-dbi=${perlPackages.DBI}/${perl.libPrefix}" + "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}" + ]; + + enableParallelBuilding = true; + + postUnpack = "sourceRoot=$sourceRoot/perl"; +}) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index e96885e4cca..f89ac4077d0 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -11,7 +11,6 @@ #include "derivations.hh" #include "globals.hh" #include "store-api.hh" -#include "util.hh" #include "crypto.hh" #include @@ -78,7 +77,7 @@ SV * queryReferences(char * path) SV * queryPathHash(char * path) PPCODE: try { - auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true); + auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(HashFormat::Base32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -104,7 +103,7 @@ SV * queryPathInfo(char * path, int base32) XPUSHs(&PL_sv_undef); else XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); - auto s = info->narHash.to_string(base32 ? Base32 : Base16, true); + auto s = info->narHash.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); @@ -206,7 +205,7 @@ SV * hashPath(char * algo, int base32, char * path) PPCODE: try { Hash h = hashPath(parseHashType(algo), path).first; - auto s = h.to_string(base32 ? Base32 : Base16, false); + auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -217,7 +216,7 @@ SV * hashFile(char * algo, int base32, char * path) PPCODE: try { Hash h = hashFile(parseHashType(algo), path); - auto s = h.to_string(base32 ? Base32 : Base16, false); + auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -228,7 +227,7 @@ SV * hashString(char * algo, int base32, char * s) PPCODE: try { Hash h = hashString(parseHashType(algo), s); - auto s = h.to_string(base32 ? Base32 : Base16, false); + auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -239,7 +238,7 @@ SV * convertHash(char * algo, char * s, int toBase32) PPCODE: try { auto h = Hash::parseAny(s, parseHashType(algo)); - auto s = h.to_string(toBase32 ? Base32 : Base16, false); + auto s = h.to_string(toBase32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 656769d84d4..a08f62333dc 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -452,6 +452,14 @@ EOF # a row for different files. if [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then # this backup process first released in Nix 2.1 + + if diff -q "$profile_target$PROFILE_BACKUP_SUFFIX" "$profile_target" > /dev/null; then + # a backup file for the rc-file exist, but they are identical, + # so we can safely ignore it and overwrite it with the same + # content later + continue + fi + failure <printStats(); + evalState->maybePrintStats(); } ref EvalCommand::getEvalStore() @@ -175,7 +175,7 @@ void BuiltPathsCommand::run(ref store, Installables && installables) throw UsageError("'--all' does not expect arguments"); // XXX: Only uses opaque paths, ignores all the realisations for (auto & p : store->queryAllValidPaths()) - paths.push_back(BuiltPath::Opaque{p}); + paths.emplace_back(BuiltPath::Opaque{p}); } else { paths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables); if (recursive) { @@ -188,7 +188,7 @@ void BuiltPathsCommand::run(ref store, Installables && installables) } store->computeFSClosure(pathsRoots, pathsClosure); for (auto & path : pathsClosure) - paths.push_back(BuiltPath::Opaque{path}); + paths.emplace_back(BuiltPath::Opaque{path}); } } diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 96236b987b0..120c832ac23 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -34,21 +34,28 @@ struct NixMultiCommand : virtual MultiCommand, virtual Command // For the overloaded run methods #pragma GCC diagnostic ignored "-Woverloaded-virtual" -/* A command that requires a Nix store. */ +/** + * A command that requires a \ref Store "Nix store". + */ struct StoreCommand : virtual Command { StoreCommand(); void run() override; ref getStore(); virtual ref createStore(); + /** + * Main entry point, with a `Store` provided + */ virtual void run(ref) = 0; private: std::shared_ptr _store; }; -/* A command that copies something between `--from` and `--to` - stores. */ +/** + * A command that copies something between `--from` and `--to` \ref + * Store stores. + */ struct CopyCommand : virtual StoreCommand { std::string srcUri, dstUri; @@ -60,6 +67,9 @@ struct CopyCommand : virtual StoreCommand ref getDstStore(); }; +/** + * A command that needs to evaluate Nix language expressions. + */ struct EvalCommand : virtual StoreCommand, MixEvalArgs { bool startReplOnEvalErrors = false; @@ -79,20 +89,26 @@ private: std::shared_ptr evalState; }; +/** + * A mixin class for commands that process flakes, adding a few standard + * flake-related options/flags. + */ struct MixFlakeOptions : virtual Args, EvalCommand { flake::LockFlags lockFlags; - std::optional needsFlakeInputCompletion = {}; - MixFlakeOptions(); - virtual std::vector getFlakesForCompletion() + /** + * The completion for some of these flags depends on the flake(s) in + * question. + * + * This method should be implemented to gather all flakerefs the + * command is operating with (presumably specified via some other + * arguments) so that the completions for these flags can use them. + */ + virtual std::vector getFlakeRefsForCompletion() { return {}; } - - void completeFlakeInput(std::string_view prefix); - - void completionHook() override; }; struct SourceExprCommand : virtual Args, MixFlakeOptions @@ -112,15 +128,35 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions virtual Strings getDefaultFlakeAttrPathPrefixes(); - void completeInstallable(std::string_view prefix); + /** + * Complete an installable from the given prefix. + */ + void completeInstallable(AddCompletions & completions, std::string_view prefix); + + /** + * Convenience wrapper around the underlying function to make setting the + * callback easier. + */ + CompleterClosure getCompleteInstallable(); }; +/** + * A mixin class for commands that need a read-only flag. + * + * What exactly is "read-only" is unspecified, but it will usually be + * the \ref Store "Nix store". + */ struct MixReadOnlyOption : virtual Args { MixReadOnlyOption(); }; -/* Like InstallablesCommand but the installables are not loaded */ +/** + * Like InstallablesCommand but the installables are not loaded. + * + * This is needed by `CmdRepl` which wants to load (and reload) the + * installables itself. + */ struct RawInstallablesCommand : virtual Args, SourceExprCommand { RawInstallablesCommand(); @@ -129,19 +165,22 @@ struct RawInstallablesCommand : virtual Args, SourceExprCommand void run(ref store) override; - // FIXME make const after CmdRepl's override is fixed up + // FIXME make const after `CmdRepl`'s override is fixed up virtual void applyDefaultInstallables(std::vector & rawInstallables); bool readFromStdIn = false; - std::vector getFlakesForCompletion() override; + std::vector getFlakeRefsForCompletion() override; private: std::vector rawInstallables; }; -/* A command that operates on a list of "installables", which can be - store paths, attribute paths, Nix expressions, etc. */ + +/** + * A command that operates on a list of "installables", which can be + * store paths, attribute paths, Nix expressions, etc. + */ struct InstallablesCommand : RawInstallablesCommand { virtual void run(ref store, Installables && installables) = 0; @@ -149,7 +188,9 @@ struct InstallablesCommand : RawInstallablesCommand void run(ref store, std::vector && rawInstallables) override; }; -/* A command that operates on exactly one "installable" */ +/** + * A command that operates on exactly one "installable". + */ struct InstallableCommand : virtual Args, SourceExprCommand { InstallableCommand(); @@ -158,10 +199,7 @@ struct InstallableCommand : virtual Args, SourceExprCommand void run(ref store) override; - std::vector getFlakesForCompletion() override - { - return {_installable}; - } + std::vector getFlakeRefsForCompletion() override; private: @@ -175,7 +213,12 @@ struct MixOperateOnOptions : virtual Args MixOperateOnOptions(); }; -/* A command that operates on zero or more store paths. */ +/** + * A command that operates on zero or more extant store paths. + * + * If the argument the user passes is a some sort of recipe for a path + * not yet built, it must be built first. + */ struct BuiltPathsCommand : InstallablesCommand, virtual MixOperateOnOptions { private: @@ -207,7 +250,9 @@ struct StorePathsCommand : public BuiltPathsCommand void run(ref store, BuiltPaths && paths) override; }; -/* A command that operates on exactly one store path. */ +/** + * A command that operates on exactly one store path. + */ struct StorePathCommand : public StorePathsCommand { virtual void run(ref store, const StorePath & storePath) = 0; @@ -215,7 +260,9 @@ struct StorePathCommand : public StorePathsCommand void run(ref store, StorePaths && storePaths) override; }; -/* A helper class for registering commands globally. */ +/** + * A helper class for registering \ref Command commands globally. + */ struct RegisterCommand { typedef std::map, std::function()>> Commands; @@ -271,13 +318,24 @@ struct MixEnvironment : virtual Args { MixEnvironment(); - /* Modify global environ based on ignoreEnvironment, keep, and unset. It's expected that exec will be called before this class goes out of scope, otherwise environ will become invalid. */ + /*** + * Modify global environ based on `ignoreEnvironment`, `keep`, and + * `unset`. It's expected that exec will be called before this class + * goes out of scope, otherwise `environ` will become invalid. + */ void setEnviron(); }; -void completeFlakeRef(ref store, std::string_view prefix); +void completeFlakeInputPath( + AddCompletions & completions, + ref evalState, + const std::vector & flakeRefs, + std::string_view prefix); + +void completeFlakeRef(AddCompletions & completions, ref store, std::string_view prefix); void completeFlakeRefWithFragment( + AddCompletions & completions, ref evalState, flake::LockFlags lockFlags, Strings attrPathPrefixes, diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index e36bda52fd9..401acc38e96 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -2,13 +2,13 @@ #include "common-eval-args.hh" #include "shared.hh" #include "filetransfer.hh" -#include "util.hh" #include "eval.hh" #include "fetchers.hh" #include "registry.hh" #include "flake/flakeref.hh" #include "store-api.hh" #include "command.hh" +#include "tarball.hh" namespace nix { @@ -132,8 +132,8 @@ MixEvalArgs::MixEvalArgs() if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); }}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(openStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, openStore(), prefix); }} }); @@ -164,18 +164,18 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) return res.finish(); } -SourcePath lookupFileArg(EvalState & state, std::string_view s) +SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir) { if (EvalSettings::isPseudoUrl(s)) { auto storePath = fetchers::downloadTarball( - state.store, EvalSettings::resolvePseudoUrl(s), "source", false).tree.storePath; + state.store, EvalSettings::resolvePseudoUrl(s), "source", false).storePath; return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } else if (hasPrefix(s, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); - auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first.storePath; + auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first; return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } @@ -185,7 +185,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s) } else - return state.rootPath(CanonPath::fromCwd(s)); + return state.rootPath(CanonPath(s, baseDir)); } } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 6359b2579ca..4b403d93674 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -2,6 +2,7 @@ ///@file #include "args.hh" +#include "canon-path.hh" #include "common-args.hh" #include "search-path.hh" @@ -28,6 +29,6 @@ private: std::map autoArgs; }; -SourcePath lookupFileArg(EvalState & state, std::string_view s); +SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir = CanonPath::fromCwd()); } diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc index a17c6f12a7f..619d3673f39 100644 --- a/src/libcmd/editor-for.cc +++ b/src/libcmd/editor-for.cc @@ -1,5 +1,5 @@ -#include "util.hh" #include "editor-for.hh" +#include "environment-variables.hh" namespace nix { diff --git a/src/libcmd/installable-attr-path.hh b/src/libcmd/installable-attr-path.hh index e9f0c33da81..86c2f82192c 100644 --- a/src/libcmd/installable-attr-path.hh +++ b/src/libcmd/installable-attr-path.hh @@ -4,7 +4,6 @@ #include "globals.hh" #include "installable-value.hh" #include "outputs-spec.hh" -#include "util.hh" #include "command.hh" #include "attr-path.hh" #include "common-eval-args.hh" diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 4074da06d72..2f428cb7e4b 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -28,6 +28,11 @@ namespace nix { std::vector InstallableFlake::getActualAttrPaths() { std::vector res; + if (attrPaths.size() == 1 && attrPaths.front().starts_with(".")){ + attrPaths.front().erase(0,1); + res.push_back(attrPaths.front()); + return res; + } for (auto & prefix : prefixes) res.push_back(prefix + *attrPaths.begin()); diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index eb19030847d..6e670efeac0 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -4,6 +4,7 @@ #include "installable-attr-path.hh" #include "installable-flake.hh" #include "outputs-spec.hh" +#include "users.hh" #include "util.hh" #include "command.hh" #include "attr-path.hh" @@ -28,15 +29,38 @@ namespace nix { +void completeFlakeInputPath( + AddCompletions & completions, + ref evalState, + const std::vector & flakeRefs, + std::string_view prefix) +{ + for (auto & flakeRef : flakeRefs) { + auto flake = flake::getFlake(*evalState, flakeRef, true); + for (auto & input : flake.inputs) + if (hasPrefix(input.first, prefix)) + completions.add(input.first); + } +} + MixFlakeOptions::MixFlakeOptions() { auto category = "Common flake-related options"; addFlag({ .longName = "recreate-lock-file", - .description = "Recreate the flake's lock file from scratch.", + .description = R"( + Recreate the flake's lock file from scratch. + + > **DEPRECATED** + > + > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead. + )", .category = category, - .handler = {&lockFlags.recreateLockFile, true} + .handler = {[&]() { + lockFlags.recreateLockFile = true; + warn("'--recreate-lock-file' is deprecated and will be removed in a future version; use 'nix flake update' instead."); + }} }); addFlag({ @@ -55,8 +79,13 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "no-registries", - .description = - "Don't allow lookups in the flake registries. This option is deprecated; use `--no-use-registries`.", + .description = R"( + Don't allow lookups in the flake registries. + + > **DEPRECATED** + > + > Use [`--no-use-registries`](#opt-no-use-registries) instead. + )", .category = category, .handler = {[&]() { lockFlags.useRegistries = false; @@ -73,14 +102,21 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "update-input", - .description = "Update a specific flake input (ignoring its previous entry in the lock file).", + .description = R"( + Update a specific flake input (ignoring its previous entry in the lock file). + + > **DEPRECATED** + > + > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead. + )", .category = category, .labels = {"input-path"}, .handler = {[&](std::string s) { + warn("'--update-input' is a deprecated alias for 'flake update' and will be removed in a future version."); lockFlags.inputUpdates.insert(flake::parseInputPath(s)); }}, - .completer = {[&](size_t, std::string_view prefix) { - needsFlakeInputCompletion = {std::string(prefix)}; + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); }} }); @@ -93,13 +129,14 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.writeLockFile = false; lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), - parseFlakeRef(flakeRef, absPath("."), true)); + parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true)); }}, - .completer = {[&](size_t n, std::string_view prefix) { - if (n == 0) - needsFlakeInputCompletion = {std::string(prefix)}; - else if (n == 1) - completeFlakeRef(getEvalState()->store, prefix); + .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) { + if (n == 0) { + completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); + } else if (n == 1) { + completeFlakeRef(completions, getEvalState()->store, prefix); + } }} }); @@ -134,7 +171,7 @@ MixFlakeOptions::MixFlakeOptions() auto evalState = getEvalState(); auto flake = flake::lockFlake( *evalState, - parseFlakeRef(flakeRef, absPath(".")), + parseFlakeRef(flakeRef, absPath(getCommandBaseDir())), { .writeLockFile = false }); for (auto & [inputName, input] : flake.lockFile.root->inputs) { auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes @@ -146,30 +183,12 @@ MixFlakeOptions::MixFlakeOptions() } } }}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getEvalState()->store, prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getEvalState()->store, prefix); }} }); } -void MixFlakeOptions::completeFlakeInput(std::string_view prefix) -{ - auto evalState = getEvalState(); - for (auto & flakeRefS : getFlakesForCompletion()) { - auto flakeRef = parseFlakeRefWithFragment(expandTilde(flakeRefS), absPath(".")).first; - auto flake = flake::getFlake(*evalState, flakeRef, true); - for (auto & input : flake.inputs) - if (hasPrefix(input.first, prefix)) - completions->add(input.first); - } -} - -void MixFlakeOptions::completionHook() -{ - if (auto & prefix = needsFlakeInputCompletion) - completeFlakeInput(*prefix); -} - SourceExprCommand::SourceExprCommand() { addFlag({ @@ -226,11 +245,18 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes() }; } -void SourceExprCommand::completeInstallable(std::string_view prefix) +Args::CompleterClosure SourceExprCommand::getCompleteInstallable() +{ + return [this](AddCompletions & completions, size_t, std::string_view prefix) { + completeInstallable(completions, prefix); + }; +} + +void SourceExprCommand::completeInstallable(AddCompletions & completions, std::string_view prefix) { try { if (file) { - completionType = ctAttrs; + completions.setType(AddCompletions::Type::Attrs); evalSettings.pureEval = false; auto state = getEvalState(); @@ -265,14 +291,15 @@ void SourceExprCommand::completeInstallable(std::string_view prefix) std::string name = state->symbols[i.name]; if (name.find(searchWord) == 0) { if (prefix_ == "") - completions->add(name); + completions.add(name); else - completions->add(prefix_ + "." + name); + completions.add(prefix_ + "." + name); } } } } else { completeFlakeRefWithFragment( + completions, getEvalState(), lockFlags, getDefaultFlakeAttrPathPrefixes(), @@ -285,6 +312,7 @@ void SourceExprCommand::completeInstallable(std::string_view prefix) } void completeFlakeRefWithFragment( + AddCompletions & completions, ref evalState, flake::LockFlags lockFlags, Strings attrPathPrefixes, @@ -296,12 +324,19 @@ void completeFlakeRefWithFragment( try { auto hash = prefix.find('#'); if (hash == std::string::npos) { - completeFlakeRef(evalState->store, prefix); + completeFlakeRef(completions, evalState->store, prefix); } else { - completionType = ctAttrs; + completions.setType(AddCompletions::Type::Attrs); auto fragment = prefix.substr(hash + 1); + std::string prefixRoot = ""; + if (fragment.starts_with(".")){ + fragment = fragment.substr(1); + prefixRoot = "."; + } auto flakeRefS = std::string(prefix.substr(0, hash)); + + // TODO: ideally this would use the command base directory instead of assuming ".". auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); auto evalCache = openEvalCache(*evalState, @@ -309,6 +344,9 @@ void completeFlakeRefWithFragment( auto root = evalCache->getRoot(); + if (prefixRoot == "."){ + attrPathPrefixes.clear(); + } /* Complete 'fragment' relative to all the attrpath prefixes as well as the root of the flake. */ @@ -333,7 +371,7 @@ void completeFlakeRefWithFragment( auto attrPath2 = (*attr)->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); - completions->add(flakeRefS + "#" + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); + completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); } } } @@ -344,7 +382,7 @@ void completeFlakeRefWithFragment( for (auto & attrPath : defaultFlakeAttrPaths) { auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath)); if (!attr) continue; - completions->add(flakeRefS + "#"); + completions.add(flakeRefS + "#" + prefixRoot); } } } @@ -353,15 +391,15 @@ void completeFlakeRefWithFragment( } } -void completeFlakeRef(ref store, std::string_view prefix) +void completeFlakeRef(AddCompletions & completions, ref store, std::string_view prefix) { if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) return; if (prefix == "") - completions->add("."); + completions.add("."); - completeDir(0, prefix); + Args::completeDir(completions, 0, prefix); /* Look for registry entries that match the prefix. */ for (auto & registry : fetchers::getRegistries(store)) { @@ -370,10 +408,10 @@ void completeFlakeRef(ref store, std::string_view prefix) if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { std::string from2(from, 6); if (hasPrefix(from2, prefix)) - completions->add(from2); + completions.add(from2); } else { if (hasPrefix(from, prefix)) - completions->add(from); + completions.add(from); } } } @@ -447,10 +485,12 @@ Installables SourceExprCommand::parseInstallables( auto e = state->parseStdin(); state->eval(e, *vFile); } - else if (file) - state->evalFile(lookupFileArg(*state, *file), *vFile); + else if (file) { + state->evalFile(lookupFileArg(*state, *file, CanonPath::fromCwd(getCommandBaseDir())), *vFile); + } else { - auto e = state->parseExprFromString(*expr, state->rootPath(CanonPath::fromCwd())); + CanonPath dir(CanonPath::fromCwd(getCommandBaseDir())); + auto e = state->parseExprFromString(*expr, state->rootPath(dir)); state->eval(e, *vFile); } @@ -485,7 +525,7 @@ Installables SourceExprCommand::parseInstallables( } try { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(".")); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir())); result.push_back(make_ref( this, getEvalState(), @@ -669,7 +709,7 @@ BuiltPaths Installable::toBuiltPaths( BuiltPaths res; for (auto & drvPath : Installable::toDerivations(store, installables, true)) - res.push_back(BuiltPath::Opaque{drvPath}); + res.emplace_back(BuiltPath::Opaque{drvPath}); return res; } } @@ -739,9 +779,7 @@ RawInstallablesCommand::RawInstallablesCommand() expectArgs({ .label = "installables", .handler = {&rawInstallables}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .completer = getCompleteInstallable(), }); } @@ -754,6 +792,17 @@ void RawInstallablesCommand::applyDefaultInstallables(std::vector & } } +std::vector RawInstallablesCommand::getFlakeRefsForCompletion() +{ + applyDefaultInstallables(rawInstallables); + std::vector res; + for (auto i : rawInstallables) + res.push_back(parseFlakeRefWithFragment( + expandTilde(i), + absPath(getCommandBaseDir())).first); + return res; +} + void RawInstallablesCommand::run(ref store) { if (readFromStdIn && !isatty(STDIN_FILENO)) { @@ -767,10 +816,13 @@ void RawInstallablesCommand::run(ref store) run(store, std::move(rawInstallables)); } -std::vector RawInstallablesCommand::getFlakesForCompletion() +std::vector InstallableCommand::getFlakeRefsForCompletion() { - applyDefaultInstallables(rawInstallables); - return rawInstallables; + return { + parseFlakeRefWithFragment( + expandTilde(_installable), + absPath(getCommandBaseDir())).first + }; } void InstallablesCommand::run(ref store, std::vector && rawInstallables) @@ -786,9 +838,7 @@ InstallableCommand::InstallableCommand() .label = "installable", .optional = true, .handler = {&_installable}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .completer = getCompleteInstallable(), }); } diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index b0dc0dc02c5..e087f935c85 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "util.hh" #include "path.hh" #include "outputs-spec.hh" #include "derived-path.hh" diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 668a077632c..8b3bbc1b5e8 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -1,6 +1,7 @@ #include "markdown.hh" #include "util.hh" #include "finally.hh" +#include "terminal.hh" #include #include diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 944609f222a..bf5643a5c3c 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -22,6 +22,7 @@ extern "C" { #include "repl.hh" #include "ansicolor.hh" +#include "signals.hh" #include "shared.hh" #include "eval.hh" #include "eval-cache.hh" @@ -36,6 +37,8 @@ extern "C" { #include "globals.hh" #include "flake/flake.hh" #include "flake/lockfile.hh" +#include "users.hh" +#include "terminal.hh" #include "editor-for.hh" #include "finally.hh" #include "markdown.hh" @@ -922,7 +925,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m case nString: str << ANSI_WARNING; - printLiteralString(str, v.string.s); + printLiteralString(str, v.string_view()); str << ANSI_NORMAL; break; diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index ab654c1b0f2..7481a2232da 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -1,6 +1,5 @@ #include "attr-path.hh" #include "eval-inline.hh" -#include "util.hh" namespace nix { @@ -132,7 +131,7 @@ std::pair findPackageFilename(EvalState & state, Value & v if (colon == std::string::npos) fail(); std::string filename(fn, 0, colon); auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos)); - return {CanonPath(fn.substr(0, colon)), lineno}; + return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno}; } catch (std::invalid_argument & e) { fail(); abort(); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 2c9aa55327f..6c0e33709f7 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -1,3 +1,4 @@ +#include "users.hh" #include "eval-cache.hh" #include "sqlite.hh" #include "eval.hh" @@ -50,7 +51,7 @@ struct AttrDb Path cacheDir = getCacheDir() + "/nix/eval-cache-v5"; createDirs(cacheDir); - Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; + Path dbPath = cacheDir + "/" + fingerprint.to_string(HashFormat::Base16, false) + ".sqlite"; state->db = SQLite(dbPath); state->db.isCache(); @@ -440,8 +441,8 @@ Value & AttrCursor::forceValue() if (root->db && (!cachedValue || std::get_if(&cachedValue->second))) { if (v.type() == nString) - cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), - string_t{v.string.s, {}}}; + cachedValue = {root->db->setString(getKey(), v.c_str(), v.context()), + string_t{v.c_str(), {}}}; else if (v.type() == nPath) { auto path = v.path().path; cachedValue = {root->db->setString(getKey(), path.abs()), string_t{path.abs(), {}}}; @@ -582,7 +583,7 @@ std::string AttrCursor::getString() if (v.type() != nString && v.type() != nPath) root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow(); - return v.type() == nString ? v.string.s : v.path().to_string(); + return v.type() == nString ? v.c_str() : v.path().to_string(); } string_t AttrCursor::getStringWithContext() @@ -624,7 +625,7 @@ string_t AttrCursor::getStringWithContext() if (v.type() == nString) { NixStringContext context; copyContext(v, context); - return {v.string.s, std::move(context)}; + return {v.c_str(), std::move(context)}; } else if (v.type() == nPath) return {v.path().to_string(), {}}; diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 93b4a5289dc..444a7d7d6dc 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -1,3 +1,4 @@ +#include "users.hh" #include "globals.hh" #include "profiles.hh" #include "eval.hh" diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 19122bc314d..3009a462c42 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -1,4 +1,6 @@ #pragma once +///@file + #include "config.hh" namespace nix { @@ -29,10 +31,12 @@ struct EvalSettings : Config this, false, "restrict-eval", R"( If set to `true`, the Nix evaluator will not allow access to any - files outside of the Nix search path (as set via the `NIX_PATH` - environment variable or the `-I` option), or to URIs outside of - [`allowed-uris`](../command-ref/conf-file.md#conf-allowed-uris). - The default is `false`. + files outside of + [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath), + or to URIs outside of + [`allowed-uris`](@docroot@/command-ref/conf-file.md#conf-allowed-uris). + + Also the default value for [`nix-path`](#conf-nix-path) is ignored, such that only explicitly set search path entries are taken into account. )"}; Setting pureEval{this, false, "pure-eval", @@ -40,18 +44,22 @@ struct EvalSettings : Config Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state: - Restrict file system and network access to files specified by cryptographic hash - - Disable [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime) + - Disable impure constants: + - [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) + - [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime) + - [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath) )" }; Setting enableImportFromDerivation{ this, true, "allow-import-from-derivation", R"( - By default, Nix allows you to `import` from a derivation, allowing - building at evaluation time. With this option set to false, Nix will - throw an error when evaluating an expression that uses this feature, - allowing users to ensure their evaluation will not require any - builds to take place. + By default, Nix allows [Import from Derivation](@docroot@/language/import-from-derivation.md). + + With this option set to `false`, Nix will throw an error when evaluating an expression that uses this feature, + even when the required store object is readily available. + This ensures that evaluation will not require any builds to take place, + regardless of the state of the store. )"}; Setting allowedUris{this, {}, "allowed-uris", @@ -60,6 +68,11 @@ struct EvalSettings : Config evaluation mode. For example, when set to `https://github.com/NixOS`, builtin functions such as `fetchGit` are allowed to access `https://github.com/NixOS/patchelf.git`. + + Access is granted when + - the URI is equal to the prefix, + - or the URI is a subpath of the prefix, + - or the prefix is a URI scheme ended by a colon `:` and the URI has the same scheme. )"}; Setting traceFunctionCalls{this, false, "trace-function-calls", diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 6d445fd968b..e1fbddd453f 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1,6 +1,7 @@ #include "eval.hh" #include "eval-settings.hh" #include "hash.hh" +#include "primops.hh" #include "types.hh" #include "util.hh" #include "store-api.hh" @@ -12,6 +13,10 @@ #include "function-trace.hh" #include "profiles.hh" #include "print.hh" +#include "fs-input-accessor.hh" +#include "memory-input-accessor.hh" +#include "signals.hh" +#include "url.hh" #include #include @@ -114,7 +119,7 @@ void Value::print(const SymbolTable &symbols, std::ostream &str, printLiteralBool(str, boolean); break; case tString: - printLiteralString(str, string.s); + printLiteralString(str, string_view()); break; case tPath: str << path().to_string(); // !!! escaping? @@ -339,7 +344,7 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) Value nameValue; name.expr->eval(state, env, nameValue); state.forceStringNoCtx(nameValue, noPos, "while evaluating an attribute name"); - return state.symbols.create(nameValue.string.s); + return state.symbols.create(nameValue.string_view()); } } @@ -503,7 +508,17 @@ EvalState::EvalState( , sOutputSpecified(symbols.create("outputSpecified")) , repair(NoRepair) , emptyBindings(0) - , derivationInternal(rootPath(CanonPath("/builtin/derivation.nix"))) + , rootFS(makeFSInputAccessor(CanonPath::root)) + , corepkgsFS(makeMemoryInputAccessor()) + , internalFS(makeMemoryInputAccessor()) + , derivationInternal{corepkgsFS->addFile( + CanonPath("derivation-internal.nix"), + #include "primops/derivation.nix.gen.hh" + )} + , callFlakeInternal{internalFS->addFile( + CanonPath("call-flake.nix"), + #include "flake/call-flake.nix.gen.hh" + )} , store(store) , buildStore(buildStore ? buildStore : store) , debugRepl(nullptr) @@ -539,7 +554,7 @@ EvalState::EvalState( auto r = resolveSearchPathPath(i.path); if (!r) continue; - auto path = *std::move(r); + auto path = std::move(*r); if (store->isInStore(path)) { try { @@ -555,6 +570,11 @@ EvalState::EvalState( } } + corepkgsFS->addFile( + CanonPath("fetchurl.nix"), + #include "fetchurl.nix.gen.hh" + ); + createBaseEnv(); } @@ -583,8 +603,20 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & mkStorePathString(storePath, v); } +inline static bool isJustSchemePrefix(std::string_view prefix) +{ + return + !prefix.empty() + && prefix[prefix.size() - 1] == ':' + && isValidSchemeName(prefix.substr(0, prefix.size() - 1)); +} + + SourcePath EvalState::checkSourcePath(const SourcePath & path_) { + // Don't check non-rootFS accessors, they're in a different namespace. + if (path_.accessor != ref(rootFS)) return path_; + if (!allowedPaths) return path_; auto i = resolvedPaths.find(path_.path.abs()); @@ -599,8 +631,6 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_) */ Path abspath = canonPath(path_.path.abs()); - if (hasPrefix(abspath, corepkgsPrefix)) return CanonPath(abspath); - for (auto & i : *allowedPaths) { if (isDirOrInDir(abspath, i)) { found = true; @@ -617,7 +647,7 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_) /* Resolve symlinks. */ debug("checking access to '%s'", abspath); - SourcePath path = CanonPath(canonPath(abspath, true)); + SourcePath path = rootPath(CanonPath(canonPath(abspath, true))); for (auto & i : *allowedPaths) { if (isDirOrInDir(path.path.abs(), i)) { @@ -630,31 +660,47 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_) } +bool isAllowedURI(std::string_view uri, const Strings & allowedUris) +{ + /* 'uri' should be equal to a prefix, or in a subdirectory of a + prefix. Thus, the prefix https://github.co does not permit + access to https://github.com. */ + for (auto & prefix : allowedUris) { + if (uri == prefix + // Allow access to subdirectories of the prefix. + || (uri.size() > prefix.size() + && prefix.size() > 0 + && hasPrefix(uri, prefix) + && ( + // Allow access to subdirectories of the prefix. + prefix[prefix.size() - 1] == '/' + || uri[prefix.size()] == '/' + + // Allow access to whole schemes + || isJustSchemePrefix(prefix) + ) + )) + return true; + } + + return false; +} + void EvalState::checkURI(const std::string & uri) { if (!evalSettings.restrictEval) return; - /* 'uri' should be equal to a prefix, or in a subdirectory of a - prefix. Thus, the prefix https://github.co does not permit - access to https://github.com. Note: this allows 'http://' and - 'https://' as prefixes for any http/https URI. */ - for (auto & prefix : evalSettings.allowedUris.get()) - if (uri == prefix || - (uri.size() > prefix.size() - && prefix.size() > 0 - && hasPrefix(uri, prefix) - && (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/'))) - return; + if (isAllowedURI(uri, evalSettings.allowedUris.get())) return; /* If the URI is a path, then check it against allowedPaths as well. */ if (hasPrefix(uri, "/")) { - checkSourcePath(CanonPath(uri)); + checkSourcePath(rootPath(CanonPath(uri))); return; } if (hasPrefix(uri, "file://")) { - checkSourcePath(CanonPath(std::string(uri, 7))); + checkSourcePath(rootPath(CanonPath(std::string(uri, 7)))); return; } @@ -703,6 +749,23 @@ void EvalState::addConstant(const std::string & name, Value * v, Constant info) } +void PrimOp::check() +{ + if (arity > maxPrimOpArity) { + throw Error("primop arity must not exceed %1%", maxPrimOpArity); + } +} + + +void Value::mkPrimOp(PrimOp * p) +{ + p->check(); + clearValue(); + internalType = tPrimOp; + primOp = p; +} + + Value * EvalState::addPrimOp(PrimOp && primOp) { /* Hack to make constants lazy: turn them into a application of @@ -950,7 +1013,7 @@ void Value::mkStringMove(const char * s, const NixStringContext & context) void Value::mkPath(const SourcePath & path) { - mkPath(makeImmutableString(path.path.abs())); + mkPath(&*path.accessor, makeImmutableString(path.path.abs())); } @@ -1035,7 +1098,7 @@ std::string EvalState::mkOutputStringRaw( /* In practice, this is testing for the case of CA derivations, or dynamic derivations. */ return optStaticOutputPath - ? store->printStorePath(*std::move(optStaticOutputPath)) + ? store->printStorePath(std::move(*optStaticOutputPath)) /* Downstream we would substitute this for an actual path once we build the floating CA derivation */ : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b, xpSettings).render(); @@ -1165,24 +1228,6 @@ void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial if (!e) e = parseExprFromFile(checkSourcePath(resolvedPath)); - cacheFile(path, resolvedPath, e, v, mustBeTrivial); -} - - -void EvalState::resetFileCache() -{ - fileEvalCache.clear(); - fileParseCache.clear(); -} - - -void EvalState::cacheFile( - const SourcePath & path, - const SourcePath & resolvedPath, - Expr * e, - Value & v, - bool mustBeTrivial) -{ fileParseCache[resolvedPath] = e; try { @@ -1211,6 +1256,13 @@ void EvalState::cacheFile( } +void EvalState::resetFileCache() +{ + fileEvalCache.clear(); + fileParseCache.clear(); +} + + void EvalState::eval(Expr * e, Value & v) { e->eval(*this, baseEnv, v); @@ -1343,7 +1395,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) if (nameVal.type() == nNull) continue; state.forceStringNoCtx(nameVal, i.pos, "while evaluating the name of a dynamic attribute"); - auto nameSym = state.symbols.create(nameVal.string.s); + auto nameSym = state.symbols.create(nameVal.string_view()); Bindings::iterator j = v.attrs->find(nameSym); if (j != v.attrs->end()) state.error("dynamic attribute '%1%' already defined at %2%", state.symbols[nameSym], state.positions[j->pos]).atPos(i.pos).withFrame(env, *this).debugThrow(); @@ -1740,6 +1792,12 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v) Value vFun; fun->eval(state, env, vFun); + // Empirical arity of Nixpkgs lambdas by regex e.g. ([a-zA-Z]+:(\s|(/\*.*\/)|(#.*\n))*){5} + // 2: over 4000 + // 3: about 300 + // 4: about 60 + // 5: under 10 + // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total. Value * vArgs[args.size()]; for (size_t i = 0; i < args.size(); ++i) vArgs[i] = args[i]->maybeThunk(state, env); @@ -2037,7 +2095,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) else if (firstType == nPath) { if (!context.empty()) state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow(); - v.mkPath(CanonPath(canonPath(str()))); + v.mkPath(state.rootPath(CanonPath(canonPath(str())))); } else v.mkStringMove(c_str(), context); } @@ -2155,7 +2213,7 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string forceValue(v, pos); if (v.type() != nString) error("value is %1% while a string was expected", showType(v)).debugThrow(); - return v.string.s; + return v.string_view(); } catch (Error & e) { e.addTrace(positions[pos], errorCtx); throw; @@ -2182,8 +2240,8 @@ std::string_view EvalState::forceString(Value & v, NixStringContext & context, c std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx) { auto s = forceString(v, pos, errorCtx); - if (v.string.context) { - error("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string.s, v.string.context[0]).withTrace(pos, errorCtx).debugThrow(); + if (v.context()) { + error("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow(); } return s; } @@ -2196,7 +2254,7 @@ bool EvalState::isDerivation(Value & v) if (i == v.attrs->end()) return false; forceValue(*i->value, i->pos); if (i->value->type() != nString) return false; - return strcmp(i->value->string.s, "derivation") == 0; + return i->value->string_view().compare("derivation") == 0; } @@ -2228,7 +2286,7 @@ BackedStringView EvalState::coerceToString( if (v.type() == nString) { copyContext(v, context); - return std::string_view(v.string.s); + return v.string_view(); } if (v.type() == nPath) { @@ -2236,7 +2294,7 @@ BackedStringView EvalState::coerceToString( !canonicalizePath && !copyToStore ? // FIXME: hack to preserve path literals that end in a // slash, as in /foo/${x}. - v._path + v._path.path : copyToStore ? store->printStorePath(copyPathToStore(context, v.path())) : std::string(v.path().path.abs()); @@ -2290,7 +2348,7 @@ BackedStringView EvalState::coerceToString( && (!v2->isList() || v2->listSize() != 0)) result += " "; } - return std::move(result); + return result; } } @@ -2310,7 +2368,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat auto dstPath = i != srcToStore.end() ? i->second : [&]() { - auto dstPath = path.fetchToStore(store, path.baseName(), nullptr, repair); + auto dstPath = path.fetchToStore(store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair); allowPath(dstPath); srcToStore.insert_or_assign(path, dstPath); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); @@ -2326,10 +2384,34 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx) { + try { + forceValue(v, pos); + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } + + /* Handle path values directly, without coercing to a string. */ + if (v.type() == nPath) + return v.path(); + + /* Similarly, handle __toString where the result may be a path + value. */ + if (v.type() == nAttrs) { + auto i = v.attrs->find(sToString); + if (i != v.attrs->end()) { + Value v1; + callFunction(*i->value, v, v1, pos); + return coerceToPath(pos, v1, context, errorCtx); + } + } + + /* Any other value should be coercable to a string, interpreted + relative to the root filesystem. */ auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (path == "" || path[0] != '/') error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow(); - return CanonPath(path); + return rootPath(CanonPath(path)); } @@ -2426,10 +2508,13 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v return v1.boolean == v2.boolean; case nString: - return strcmp(v1.string.s, v2.string.s) == 0; + return v1.string_view().compare(v2.string_view()) == 0; case nPath: - return strcmp(v1._path, v2._path) == 0; + return + // FIXME: compare accessors by their fingerprint. + v1._path.accessor == v2._path.accessor + && strcmp(v1._path.path, v2._path.path) == 0; case nNull: return true; @@ -2477,10 +2562,37 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v } } -void EvalState::printStats() +bool EvalState::fullGC() { +#if HAVE_BOEHMGC + GC_gcollect(); + // Check that it ran. We might replace this with a version that uses more + // of the boehm API to get this reliably, at a maintenance cost. + // We use a 1K margin because technically this has a race condtion, but we + // probably won't encounter it in practice, because the CLI isn't concurrent + // like that. + return GC_get_bytes_since_gc() < 1024; +#else + return false; +#endif +} + +void EvalState::maybePrintStats() { bool showStats = getEnv("NIX_SHOW_STATS").value_or("0") != "0"; + if (showStats) { + // Make the final heap size more deterministic. +#if HAVE_BOEHMGC + if (!fullGC()) { + warn("failed to perform a full GC before reporting stats"); + } +#endif + printStatistics(); + } +} + +void EvalState::printStatistics() +{ struct rusage buf; getrusage(RUSAGE_SELF, &buf); float cpuTime = buf.ru_utime.tv_sec + ((float) buf.ru_utime.tv_usec / 1000000); @@ -2494,105 +2606,105 @@ void EvalState::printStats() GC_word heapSize, totalBytes; GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes); #endif - if (showStats) { - auto outPath = getEnv("NIX_SHOW_STATS_PATH").value_or("-"); - std::fstream fs; - if (outPath != "-") - fs.open(outPath, std::fstream::out); - json topObj = json::object(); - topObj["cpuTime"] = cpuTime; - topObj["envs"] = { - {"number", nrEnvs}, - {"elements", nrValuesInEnvs}, - {"bytes", bEnvs}, - }; - topObj["list"] = { - {"elements", nrListElems}, - {"bytes", bLists}, - {"concats", nrListConcats}, - }; - topObj["values"] = { - {"number", nrValues}, - {"bytes", bValues}, - }; - topObj["symbols"] = { - {"number", symbols.size()}, - {"bytes", symbols.totalSize()}, - }; - topObj["sets"] = { - {"number", nrAttrsets}, - {"bytes", bAttrsets}, - {"elements", nrAttrsInAttrsets}, - }; - topObj["sizes"] = { - {"Env", sizeof(Env)}, - {"Value", sizeof(Value)}, - {"Bindings", sizeof(Bindings)}, - {"Attr", sizeof(Attr)}, - }; - topObj["nrOpUpdates"] = nrOpUpdates; - topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied; - topObj["nrThunks"] = nrThunks; - topObj["nrAvoided"] = nrAvoided; - topObj["nrLookups"] = nrLookups; - topObj["nrPrimOpCalls"] = nrPrimOpCalls; - topObj["nrFunctionCalls"] = nrFunctionCalls; + + auto outPath = getEnv("NIX_SHOW_STATS_PATH").value_or("-"); + std::fstream fs; + if (outPath != "-") + fs.open(outPath, std::fstream::out); + json topObj = json::object(); + topObj["cpuTime"] = cpuTime; + topObj["envs"] = { + {"number", nrEnvs}, + {"elements", nrValuesInEnvs}, + {"bytes", bEnvs}, + }; + topObj["nrExprs"] = Expr::nrExprs; + topObj["list"] = { + {"elements", nrListElems}, + {"bytes", bLists}, + {"concats", nrListConcats}, + }; + topObj["values"] = { + {"number", nrValues}, + {"bytes", bValues}, + }; + topObj["symbols"] = { + {"number", symbols.size()}, + {"bytes", symbols.totalSize()}, + }; + topObj["sets"] = { + {"number", nrAttrsets}, + {"bytes", bAttrsets}, + {"elements", nrAttrsInAttrsets}, + }; + topObj["sizes"] = { + {"Env", sizeof(Env)}, + {"Value", sizeof(Value)}, + {"Bindings", sizeof(Bindings)}, + {"Attr", sizeof(Attr)}, + }; + topObj["nrOpUpdates"] = nrOpUpdates; + topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied; + topObj["nrThunks"] = nrThunks; + topObj["nrAvoided"] = nrAvoided; + topObj["nrLookups"] = nrLookups; + topObj["nrPrimOpCalls"] = nrPrimOpCalls; + topObj["nrFunctionCalls"] = nrFunctionCalls; #if HAVE_BOEHMGC - topObj["gc"] = { - {"heapSize", heapSize}, - {"totalBytes", totalBytes}, - }; + topObj["gc"] = { + {"heapSize", heapSize}, + {"totalBytes", totalBytes}, + }; #endif - if (countCalls) { - topObj["primops"] = primOpCalls; - { - auto& list = topObj["functions"]; - list = json::array(); - for (auto & [fun, count] : functionCalls) { - json obj = json::object(); - if (fun->name) - obj["name"] = (std::string_view) symbols[fun->name]; - else - obj["name"] = nullptr; - if (auto pos = positions[fun->pos]) { - if (auto path = std::get_if(&pos.origin)) - obj["file"] = path->to_string(); - obj["line"] = pos.line; - obj["column"] = pos.column; - } - obj["count"] = count; - list.push_back(obj); + if (countCalls) { + topObj["primops"] = primOpCalls; + { + auto& list = topObj["functions"]; + list = json::array(); + for (auto & [fun, count] : functionCalls) { + json obj = json::object(); + if (fun->name) + obj["name"] = (std::string_view) symbols[fun->name]; + else + obj["name"] = nullptr; + if (auto pos = positions[fun->pos]) { + if (auto path = std::get_if(&pos.origin)) + obj["file"] = path->to_string(); + obj["line"] = pos.line; + obj["column"] = pos.column; } + obj["count"] = count; + list.push_back(obj); } - { - auto list = topObj["attributes"]; - list = json::array(); - for (auto & i : attrSelects) { - json obj = json::object(); - if (auto pos = positions[i.first]) { - if (auto path = std::get_if(&pos.origin)) - obj["file"] = path->to_string(); - obj["line"] = pos.line; - obj["column"] = pos.column; - } - obj["count"] = i.second; - list.push_back(obj); + } + { + auto list = topObj["attributes"]; + list = json::array(); + for (auto & i : attrSelects) { + json obj = json::object(); + if (auto pos = positions[i.first]) { + if (auto path = std::get_if(&pos.origin)) + obj["file"] = path->to_string(); + obj["line"] = pos.line; + obj["column"] = pos.column; } + obj["count"] = i.second; + list.push_back(obj); } } + } - if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") { - // XXX: overrides earlier assignment - topObj["symbols"] = json::array(); - auto &list = topObj["symbols"]; - symbols.dump([&](const std::string & s) { list.emplace_back(s); }); - } - if (outPath == "-") { - std::cerr << topObj.dump(2) << std::endl; - } else { - fs << topObj.dump(2) << std::endl; - } + if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") { + // XXX: overrides earlier assignment + topObj["symbols"] = json::array(); + auto &list = topObj["symbols"]; + symbols.dump([&](const std::string & s) { list.emplace_back(s); }); + } + if (outPath == "-") { + std::cerr << topObj.dump(2) << std::endl; + } else { + fs << topObj.dump(2) << std::endl; } } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index fa8fa462b0f..23fbbf0af16 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -18,12 +18,20 @@ namespace nix { +/** + * We put a limit on primop arity because it lets us use a fixed size array on + * the stack. 8 is already an impractical number of arguments. Use an attrset + * argument for such overly complicated functions. + */ +constexpr size_t maxPrimOpArity = 8; class Store; class EvalState; class StorePath; struct SingleDerivedPath; enum RepairFlag : bool; +struct FSInputAccessor; +struct MemoryInputAccessor; /** @@ -69,6 +77,12 @@ struct PrimOp * Optional experimental for this to be gated on. */ std::optional experimentalFeature; + + /** + * Validity check to be performed by functions that introduce primops, + * such as RegisterPrimOp() and Value::mkPrimOp(). + */ + void check(); }; /** @@ -211,8 +225,26 @@ public: Bindings emptyBindings; + /** + * The accessor for the root filesystem. + */ + const ref rootFS; + + /** + * The in-memory filesystem for paths. + */ + const ref corepkgsFS; + + /** + * In-memory filesystem for internal, non-user-callable Nix + * expressions like call-flake.nix. + */ + const ref internalFS; + const SourcePath derivationInternal; + const SourcePath callFlakeInternal; + /** * Store used to materialise .drv files. */ @@ -223,7 +255,6 @@ public: */ const ref buildStore; - RootValue vCallFlake = nullptr; RootValue vImportedDrvToDerivation = nullptr; /** @@ -405,16 +436,6 @@ public: */ void evalFile(const SourcePath & path, Value & v, bool mustBeTrivial = false); - /** - * Like `evalFile`, but with an already parsed expression. - */ - void cacheFile( - const SourcePath & path, - const SourcePath & resolvedPath, - Expr * e, - Value & v, - bool mustBeTrivial = false); - void resetFileCache(); /** @@ -424,7 +445,7 @@ public: SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); /** - * Try to resolve a search path value (not the optinal key part) + * Try to resolve a search path value (not the optional key part) * * If the specified search path element is a URI, download it. * @@ -709,9 +730,25 @@ public: void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); /** - * Print statistics. + * Print statistics, if enabled. + * + * Performs a full memory GC before printing the statistics, so that the + * GC statistics are more accurate. + */ + void maybePrintStats(); + + /** + * Print statistics, unconditionally, cheaply, without performing a GC first. */ - void printStats(); + void printStatistics(); + + /** + * Perform a full memory garbage collection - not incremental. + * + * @return true if Nix was built with GC and a GC was performed, false if not. + * The return value is currently not thread safe - just the return value. + */ + bool fullGC(); /** * Realise the given context, and return a mapping from the placeholders @@ -802,7 +839,12 @@ std::string showType(const Value & v); /** * If `path` refers to a directory, then append "/default.nix". */ -SourcePath resolveExprPath(const SourcePath & path); +SourcePath resolveExprPath(SourcePath path); + +/** + * Whether a URI is allowed, assuming restrictEval is enabled + */ +bool isAllowedURI(std::string_view uri, const Strings & allowedPaths); struct InvalidPathError : EvalError { @@ -813,8 +855,6 @@ struct InvalidPathError : EvalError #endif }; -static const std::string corepkgsPrefix{"/__corepkgs__/"}; - template void ErrorBuilder::debugThrow() { diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index e890148621e..3c7ed5d8a5b 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -1,6 +1,7 @@ -#include "flake.hh" +#include "users.hh" #include "globals.hh" #include "fetch-settings.hh" +#include "flake.hh" #include diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 90427e06470..54de53e0bfd 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -1,3 +1,4 @@ +#include "terminal.hh" #include "flake.hh" #include "eval.hh" #include "eval-settings.hh" @@ -8,6 +9,7 @@ #include "fetchers.hh" #include "finally.hh" #include "fetch-settings.hh" +#include "value-to-json.hh" namespace nix { @@ -15,7 +17,7 @@ using namespace flake; namespace flake { -typedef std::pair FetchedFlake; +typedef std::pair FetchedFlake; typedef std::vector> FlakeCache; static std::optional lookupInFlakeCache( @@ -34,7 +36,7 @@ static std::optional lookupInFlakeCache( return std::nullopt; } -static std::tuple fetchOrSubstituteTree( +static std::tuple fetchOrSubstituteTree( EvalState & state, const FlakeRef & originalRef, bool allowLookup, @@ -61,16 +63,16 @@ static std::tuple fetchOrSubstituteTree( flakeCache.push_back({originalRef, *fetched}); } - auto [tree, lockedRef] = *fetched; + auto [storePath, lockedRef] = *fetched; debug("got tree '%s' from '%s'", - state.store->printStorePath(tree.storePath), lockedRef); + state.store->printStorePath(storePath), lockedRef); - state.allowPath(tree.storePath); + state.allowPath(storePath); - assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store)); + assert(!originalRef.input.getNarHash() || storePath == originalRef.input.computeStorePath(*state.store)); - return {std::move(tree), resolvedRef, lockedRef}; + return {std::move(storePath), resolvedRef, lockedRef}; } static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos) @@ -113,7 +115,7 @@ static FlakeInput parseFlakeInput(EvalState & state, try { if (attr.name == sUrl) { expectType(state, nString, *attr.value, attr.pos); - url = attr.value->string.s; + url = attr.value->string_view(); attrs.emplace("url", *url); } else if (attr.name == sFlake) { expectType(state, nBool, *attr.value, attr.pos); @@ -122,7 +124,7 @@ static FlakeInput parseFlakeInput(EvalState & state, input.overrides = parseFlakeInputs(state, attr.value, attr.pos, baseDir, lockRootPath); } else if (attr.name == sFollows) { expectType(state, nString, *attr.value, attr.pos); - auto follows(parseInputPath(attr.value->string.s)); + auto follows(parseInputPath(attr.value->c_str())); follows.insert(follows.begin(), lockRootPath.begin(), lockRootPath.end()); input.follows = follows; } else { @@ -131,7 +133,7 @@ static FlakeInput parseFlakeInput(EvalState & state, #pragma GCC diagnostic ignored "-Wswitch-enum" switch (attr.value->type()) { case nString: - attrs.emplace(state.symbols[attr.name], attr.value->string.s); + attrs.emplace(state.symbols[attr.name], attr.value->c_str()); break; case nBool: attrs.emplace(state.symbols[attr.name], Explicit { attr.value->boolean }); @@ -140,8 +142,13 @@ static FlakeInput parseFlakeInput(EvalState & state, attrs.emplace(state.symbols[attr.name], (long unsigned int)attr.value->integer); break; default: - throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", - state.symbols[attr.name], showType(*attr.value)); + if (attr.name == state.symbols.create("publicKeys")) { + experimentalFeatureSettings.require(Xp::VerifiedFetches); + NixStringContext emptyContext = {}; + attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, emptyContext).dump()); + } else + throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", + state.symbols[attr.name], showType(*attr.value)); } #pragma GCC diagnostic pop } @@ -202,34 +209,34 @@ static Flake getFlake( FlakeCache & flakeCache, InputPath lockRootPath) { - auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( + auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( state, originalRef, allowLookup, flakeCache); // Guard against symlink attacks. - auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir, true); + auto flakeDir = canonPath(state.store->toRealPath(storePath) + "/" + lockedRef.subdir, true); auto flakeFile = canonPath(flakeDir + "/flake.nix", true); - if (!isInDir(flakeFile, sourceInfo.actualPath)) + if (!isInDir(flakeFile, state.store->toRealPath(storePath))) throw Error("'flake.nix' file of flake '%s' escapes from '%s'", - lockedRef, state.store->printStorePath(sourceInfo.storePath)); + lockedRef, state.store->printStorePath(storePath)); Flake flake { .originalRef = originalRef, .resolvedRef = resolvedRef, .lockedRef = lockedRef, - .sourceInfo = std::make_shared(std::move(sourceInfo)) + .storePath = storePath, }; if (!pathExists(flakeFile)) throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); Value vInfo; - state.evalFile(CanonPath(flakeFile), vInfo, true); // FIXME: symlink attack + state.evalFile(state.rootPath(CanonPath(flakeFile)), vInfo, true); // FIXME: symlink attack - expectType(state, nAttrs, vInfo, state.positions.add({CanonPath(flakeFile)}, 1, 1)); + expectType(state, nAttrs, vInfo, state.positions.add({state.rootPath(CanonPath(flakeFile))}, 1, 1)); if (auto description = vInfo.attrs->get(state.sDescription)) { expectType(state, nString, *description->value, description->pos); - flake.description = description->value->string.s; + flake.description = description->value->c_str(); } auto sInputs = state.symbols.create("inputs"); @@ -346,7 +353,7 @@ LockedFlake lockFlake( // FIXME: symlink attack auto oldLockFile = LockFile::read( lockFlags.referenceLockFilePath.value_or( - flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock")); + state.store->toRealPath(flake.storePath) + "/" + flake.lockedRef.subdir + "/flake.lock")); debug("old lock file: %s", oldLockFile); @@ -447,8 +454,8 @@ LockedFlake lockFlake( assert(input.ref); - /* Do we have an entry in the existing lock file? And we - don't have a --update-input flag for this input? */ + /* Do we have an entry in the existing lock file? + And the input is not in updateInputs? */ std::shared_ptr oldLock; updatesUsed.insert(inputPath); @@ -472,9 +479,8 @@ LockedFlake lockFlake( node->inputs.insert_or_assign(id, childNode); - /* If we have an --update-input flag for an input - of this input, then we must fetch the flake to - update it. */ + /* If we have this input in updateInputs, then we + must fetch the flake to update it. */ auto lb = lockFlags.inputUpdates.lower_bound(inputPath); auto mustRefetch = @@ -574,7 +580,7 @@ LockedFlake lockFlake( oldLock ? std::dynamic_pointer_cast(oldLock) : LockFile::read( - inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root.get_ptr(), + state.store->toRealPath(inputFlake.storePath) + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root.get_ptr(), oldLock ? lockRootPath : inputPath, localPath, false); @@ -598,7 +604,7 @@ LockedFlake lockFlake( }; // Bring in the current ref for relative path resolution if we have it - auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir, true); + auto parentPath = canonPath(state.store->toRealPath(flake.storePath) + "/" + flake.lockedRef.subdir, true); computeLocks( flake.inputs, @@ -616,19 +622,14 @@ LockedFlake lockFlake( for (auto & i : lockFlags.inputUpdates) if (!updatesUsed.count(i)) - warn("the flag '--update-input %s' does not match any input", printInputPath(i)); + warn("'%s' does not match any input of this flake", printInputPath(i)); /* Check 'follows' inputs. */ newLockFile.check(); debug("new lock file: %s", newLockFile); - auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; auto sourcePath = topRef.input.getSourcePath(); - auto outputLockFilePath = sourcePath ? std::optional{*sourcePath + "/" + relPath} : std::nullopt; - if (lockFlags.outputLockFilePath) { - outputLockFilePath = lockFlags.outputLockFilePath; - } /* Check whether we need to / can write the new lock file. */ if (newLockFile != oldLockFile || lockFlags.outputLockFilePath) { @@ -636,7 +637,7 @@ LockedFlake lockFlake( auto diff = LockFile::diff(oldLockFile, newLockFile); if (lockFlags.writeLockFile) { - if (outputLockFilePath) { + if (sourcePath || lockFlags.outputLockFilePath) { if (auto unlockedInput = newLockFile.isUnlocked()) { if (fetchSettings.warnDirty) warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput); @@ -644,41 +645,48 @@ LockedFlake lockFlake( if (!lockFlags.updateLockFile) throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); - bool lockFileExists = pathExists(*outputLockFilePath); + auto newLockFileS = fmt("%s\n", newLockFile); + + if (lockFlags.outputLockFilePath) { + if (lockFlags.commitLockFile) + throw Error("'--commit-lock-file' and '--output-lock-file' are incompatible"); + writeFile(*lockFlags.outputLockFilePath, newLockFileS); + } else { + auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; + auto outputLockFilePath = *sourcePath + "/" + relPath; + + bool lockFileExists = pathExists(outputLockFilePath); - if (lockFileExists) { auto s = chomp(diff); - if (s.empty()) - warn("updating lock file '%s'", *outputLockFilePath); - else - warn("updating lock file '%s':\n%s", *outputLockFilePath, s); - } else - warn("creating lock file '%s'", *outputLockFilePath); + if (lockFileExists) { + if (s.empty()) + warn("updating lock file '%s'", outputLockFilePath); + else + warn("updating lock file '%s':\n%s", outputLockFilePath, s); + } else + warn("creating lock file '%s': \n%s", outputLockFilePath, s); - newLockFile.write(*outputLockFilePath); + std::optional commitMessage = std::nullopt; - std::optional commitMessage = std::nullopt; - if (lockFlags.commitLockFile) { - if (lockFlags.outputLockFilePath) { - throw Error("--commit-lock-file and --output-lock-file are currently incompatible"); - } - std::string cm; + if (lockFlags.commitLockFile) { + std::string cm; + + cm = fetchSettings.commitLockFileSummary.get(); - cm = fetchSettings.commitLockFileSummary.get(); + if (cm == "") { + cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add"); + } - if (cm == "") { - cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add"); + cm += "\n\nFlake lock file updates:\n\n"; + cm += filterANSIEscapes(diff, true); + commitMessage = cm; } - cm += "\n\nFlake lock file updates:\n\n"; - cm += filterANSIEscapes(diff, true); - commitMessage = cm; + topRef.input.putFile( + CanonPath((topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"), + newLockFileS, commitMessage); } - topRef.input.markChangedFile( - (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock", - commitMessage); - /* Rewriting the lockfile changed the top-level repo, so we should re-read it. FIXME: we could also just clear the 'rev' field... */ @@ -729,7 +737,7 @@ void callFlake(EvalState & state, emitTreeAttrs( state, - *lockedFlake.flake.sourceInfo, + lockedFlake.flake.storePath, lockedFlake.flake.lockedRef.input, *vRootSrc, false, @@ -737,14 +745,10 @@ void callFlake(EvalState & state, vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir); - if (!state.vCallFlake) { - state.vCallFlake = allocRootValue(state.allocValue()); - state.eval(state.parseExprFromString( - #include "call-flake.nix.gen.hh" - , CanonPath::root), **state.vCallFlake); - } + auto vCallFlake = state.allocValue(); + state.evalFile(state.callFlakeInternal, *vCallFlake); - state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos); + state.callFunction(*vCallFlake, *vLocks, *vTmp1, noPos); state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos); state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos); } @@ -850,7 +854,7 @@ static void prim_flakeRefToString( Explicit { attr.value->boolean }); } else if (t == nString) { attrs.emplace(state.symbols[attr.name], - std::string(attr.value->str())); + std::string(attr.value->string_view())); } else { state.error( "flake reference attribute sets may only contain integers, Booleans, " @@ -893,7 +897,7 @@ Fingerprint LockedFlake::getFingerprint() const // flake.sourceInfo.storePath for the fingerprint. return hashString(htSHA256, fmt("%s;%s;%d;%d;%s", - flake.sourceInfo->storePath.to_string(), + flake.storePath.to_string(), flake.lockedRef.subdir, flake.lockedRef.input.getRevCount().value_or(0), flake.lockedRef.input.getLastModified().value_or(0), diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index c1d1b71e509..d5ad3eaded8 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -10,8 +10,6 @@ namespace nix { class EvalState; -namespace fetchers { struct Tree; } - namespace flake { struct FlakeInput; @@ -84,7 +82,7 @@ struct Flake */ bool forceDirty = false; std::optional description; - std::shared_ptr sourceInfo; + StorePath storePath; FlakeInputs inputs; /** * 'nixConfig' attribute @@ -193,7 +191,7 @@ void callFlake( void emitTreeAttrs( EvalState & state, - const fetchers::Tree & tree, + const StorePath & storePath, const fetchers::Input & input, Value & v, bool emptyRevFallback = false, diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index e1bce90bc9e..8b0eb74601e 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -69,32 +69,130 @@ std::optional maybeParseFlakeRef( } } -std::pair parseFlakeRefWithFragment( +std::pair parsePathFlakeRefWithFragment( const std::string & url, const std::optional & baseDir, bool allowMissing, bool isFlake) { - using namespace fetchers; + std::string path = url; + std::string fragment = ""; + std::map query; + auto pathEnd = url.find_first_of("#?"); + auto fragmentStart = pathEnd; + if (pathEnd != std::string::npos && url[pathEnd] == '?') { + fragmentStart = url.find("#"); + } + if (pathEnd != std::string::npos) { + path = url.substr(0, pathEnd); + } + if (fragmentStart != std::string::npos) { + fragment = percentDecode(url.substr(fragmentStart+1)); + } + if (pathEnd != std::string::npos && fragmentStart != std::string::npos) { + query = decodeQuery(url.substr(pathEnd+1, fragmentStart-pathEnd-1)); + } - static std::string fnRegex = "[0-9a-zA-Z-._~!$&'\"()*+,;=]+"; + if (baseDir) { + /* Check if 'url' is a path (either absolute or relative + to 'baseDir'). If so, search upward to the root of the + repo (i.e. the directory containing .git). */ + + path = absPath(path, baseDir); + + if (isFlake) { + + if (!allowMissing && !pathExists(path + "/flake.nix")){ + notice("path '%s' does not contain a 'flake.nix', searching up",path); + + // Save device to detect filesystem boundary + dev_t device = lstat(path).st_dev; + bool found = false; + while (path != "/") { + if (pathExists(path + "/flake.nix")) { + found = true; + break; + } else if (pathExists(path + "/.git")) + throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", path); + else { + if (lstat(path).st_dev != device) + throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path); + } + path = dirOf(path); + } + if (!found) + throw BadURL("could not find a flake.nix file"); + } - static std::regex pathUrlRegex( - "(/?" + fnRegex + "(?:/" + fnRegex + ")*/?)" - + "(?:\\?(" + queryRegex + "))?" - + "(?:#(" + queryRegex + "))?", - std::regex::ECMAScript); + if (!S_ISDIR(lstat(path).st_mode)) + throw BadURL("path '%s' is not a flake (because it's not a directory)", path); + + if (!allowMissing && !pathExists(path + "/flake.nix")) + throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path); + + auto flakeRoot = path; + std::string subdir; + + while (flakeRoot != "/") { + if (pathExists(flakeRoot + "/.git")) { + auto base = std::string("git+file://") + flakeRoot; + + auto parsedURL = ParsedURL{ + .url = base, // FIXME + .base = base, + .scheme = "git+file", + .authority = "", + .path = flakeRoot, + .query = query, + }; + + if (subdir != "") { + if (parsedURL.query.count("dir")) + throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url); + parsedURL.query.insert_or_assign("dir", subdir); + } + + if (pathExists(flakeRoot + "/.git/shallow")) + parsedURL.query.insert_or_assign("shallow", "1"); + + return std::make_pair( + FlakeRef(fetchers::Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")), + fragment); + } + + subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); + flakeRoot = dirOf(flakeRoot); + } + } + + } else { + if (!hasPrefix(path, "/")) + throw BadURL("flake reference '%s' is not an absolute path", url); + path = canonPath(path + "/" + getOr(query, "dir", "")); + } + + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "path"); + attrs.insert_or_assign("path", path); + + return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(std::move(attrs)), ""), fragment); +}; + + +/* Check if 'url' is a flake ID. This is an abbreviated syntax for + 'flake:?ref=&rev='. */ +std::optional> parseFlakeIdRef( + const std::string & url, + bool isFlake +) +{ + std::smatch match; static std::regex flakeRegex( "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)" + "(?:#(" + queryRegex + "))?", std::regex::ECMAScript); - std::smatch match; - - /* Check if 'url' is a flake ID. This is an abbreviated syntax for - 'flake:?ref=&rev='. */ - if (std::regex_match(url, match, flakeRegex)) { auto parsedURL = ParsedURL{ .url = url, @@ -105,111 +203,53 @@ std::pair parseFlakeRefWithFragment( }; return std::make_pair( - FlakeRef(Input::fromURL(parsedURL, isFlake), ""), + FlakeRef(fetchers::Input::fromURL(parsedURL, isFlake), ""), percentDecode(match.str(6))); } - else if (std::regex_match(url, match, pathUrlRegex)) { - std::string path = match[1]; - std::string fragment = percentDecode(match.str(3)); - - if (baseDir) { - /* Check if 'url' is a path (either absolute or relative - to 'baseDir'). If so, search upward to the root of the - repo (i.e. the directory containing .git). */ - - path = absPath(path, baseDir); - - if (isFlake) { - - if (!allowMissing && !pathExists(path + "/flake.nix")){ - notice("path '%s' does not contain a 'flake.nix', searching up",path); - - // Save device to detect filesystem boundary - dev_t device = lstat(path).st_dev; - bool found = false; - while (path != "/") { - if (pathExists(path + "/flake.nix")) { - found = true; - break; - } else if (pathExists(path + "/.git")) - throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", path); - else { - if (lstat(path).st_dev != device) - throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path); - } - path = dirOf(path); - } - if (!found) - throw BadURL("could not find a flake.nix file"); - } - - if (!S_ISDIR(lstat(path).st_mode)) - throw BadURL("path '%s' is not a flake (because it's not a directory)", path); - - if (!allowMissing && !pathExists(path + "/flake.nix")) - throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path); - - auto flakeRoot = path; - std::string subdir; - - while (flakeRoot != "/") { - if (pathExists(flakeRoot + "/.git")) { - auto base = std::string("git+file://") + flakeRoot; - - auto parsedURL = ParsedURL{ - .url = base, // FIXME - .base = base, - .scheme = "git+file", - .authority = "", - .path = flakeRoot, - .query = decodeQuery(match[2]), - }; - - if (subdir != "") { - if (parsedURL.query.count("dir")) - throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url); - parsedURL.query.insert_or_assign("dir", subdir); - } - - if (pathExists(flakeRoot + "/.git/shallow")) - parsedURL.query.insert_or_assign("shallow", "1"); - - return std::make_pair( - FlakeRef(Input::fromURL(parsedURL, isFlake), getOr(parsedURL.query, "dir", "")), - fragment); - } + return {}; +} - subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); - flakeRoot = dirOf(flakeRoot); - } - } +std::optional> parseURLFlakeRef( + const std::string & url, + const std::optional & baseDir, + bool isFlake +) +{ + ParsedURL parsedURL; + try { + parsedURL = parseURL(url); + } catch (BadURL &) { + return std::nullopt; + } - } else { - if (!hasPrefix(path, "/")) - throw BadURL("flake reference '%s' is not an absolute path", url); - auto query = decodeQuery(match[2]); - path = canonPath(path + "/" + getOr(query, "dir", "")); - } + std::string fragment; + std::swap(fragment, parsedURL.fragment); - fetchers::Attrs attrs; - attrs.insert_or_assign("type", "path"); - attrs.insert_or_assign("path", path); + auto input = fetchers::Input::fromURL(parsedURL, isFlake); + input.parent = baseDir; - return std::make_pair(FlakeRef(Input::fromAttrs(std::move(attrs)), ""), fragment); - } + return std::make_pair( + FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), + fragment); +} - else { - auto parsedURL = parseURL(url); - std::string fragment; - std::swap(fragment, parsedURL.fragment); +std::pair parseFlakeRefWithFragment( + const std::string & url, + const std::optional & baseDir, + bool allowMissing, + bool isFlake) +{ + using namespace fetchers; - auto input = Input::fromURL(parsedURL, isFlake); - input.parent = baseDir; + std::smatch match; - return std::make_pair( - FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), - fragment); + if (auto res = parseFlakeIdRef(url, isFlake)) { + return *res; + } else if (auto res = parseURLFlakeRef(url, baseDir, isFlake)) { + return *res; + } else { + return parsePathFlakeRefWithFragment(url, baseDir, allowMissing, isFlake); } } @@ -232,10 +272,10 @@ FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs) fetchers::maybeGetStrAttr(attrs, "dir").value_or("")); } -std::pair FlakeRef::fetchTree(ref store) const +std::pair FlakeRef::fetchTree(ref store) const { - auto [tree, lockedInput] = input.fetch(store); - return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)}; + auto [storePath, lockedInput] = input.fetch(store); + return {std::move(storePath), FlakeRef(std::move(lockedInput), subdir)}; } std::tuple parseFlakeRefWithFragmentAndExtendedOutputsSpec( @@ -249,4 +289,6 @@ std::tuple parseFlakeRefWithFragment return {std::move(flakeRef), fragment, std::move(extendedOutputsSpec)}; } +std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript); + } diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index a7c9208c06e..5d78f49b683 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -6,6 +6,7 @@ #include "fetchers.hh" #include "outputs-spec.hh" +#include #include namespace nix { @@ -62,7 +63,7 @@ struct FlakeRef static FlakeRef fromAttrs(const fetchers::Attrs & attrs); - std::pair fetchTree(ref store) const; + std::pair fetchTree(ref store) const; }; std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); @@ -91,5 +92,7 @@ std::tuple parseFlakeRefWithFragment bool allowMissing = false, bool isFlake = true); +const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; +extern std::regex flakeIdRegex; } diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 3c202967a02..3e99fb2d417 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -2,8 +2,10 @@ #include "store-api.hh" #include "url-parts.hh" +#include #include +#include #include namespace nix::flake { @@ -45,16 +47,26 @@ StorePath LockedNode::computeStorePath(Store & store) const return lockedRef.input.computeStorePath(store); } -std::shared_ptr LockFile::findInput(const InputPath & path) -{ + +static std::shared_ptr doFind(const ref& root, const InputPath & path, std::vector& visited) { auto pos = root; + auto found = std::find(visited.cbegin(), visited.cend(), path); + + if(found != visited.end()) { + std::vector cycle; + std::transform(found, visited.cend(), std::back_inserter(cycle), printInputPath); + cycle.push_back(printInputPath(path)); + throw Error("follow cycle detected: [%s]", concatStringsSep(" -> ", cycle)); + } + visited.push_back(path); + for (auto & elem : path) { if (auto i = get(pos->inputs, elem)) { if (auto node = std::get_if<0>(&*i)) pos = *node; else if (auto follows = std::get_if<1>(&*i)) { - if (auto p = findInput(*follows)) + if (auto p = doFind(root, *follows, visited)) pos = ref(p); else return {}; @@ -66,6 +78,12 @@ std::shared_ptr LockFile::findInput(const InputPath & path) return pos; } +std::shared_ptr LockFile::findInput(const InputPath & path) +{ + std::vector visited; + return doFind(root, path, visited); +} + LockFile::LockFile(const nlohmann::json & json, const Path & path) { auto version = json.value("version", 0); @@ -196,12 +214,6 @@ std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) return stream; } -void LockFile::write(const Path & path) const -{ - createDirs(dirOf(path)); - writeFile(path, fmt("%s\n", *this)); -} - std::optional LockFile::isUnlocked() const { std::set> nodes; diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index ba4c0c8485c..5a1493404d6 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -65,8 +65,6 @@ struct LockFile static LockFile read(const Path & path); - void write(const Path & path) const; - /** * Check whether this lock file has any unlocked inputs. */ diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 506a6367763..d4e946d8104 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -1,5 +1,4 @@ #include "get-drvs.hh" -#include "util.hh" #include "eval-inline.hh" #include "derivations.hh" #include "store-api.hh" @@ -156,7 +155,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall Outputs result; for (auto elem : outTI->listItems()) { if (elem->type() != nString) throw errMsg; - auto out = outputs.find(elem->string.s); + auto out = outputs.find(elem->c_str()); if (out == outputs.end()) throw errMsg; result.insert(*out); } @@ -230,7 +229,7 @@ std::string DrvInfo::queryMetaString(const std::string & name) { Value * v = queryMeta(name); if (!v || v->type() != nString) return ""; - return v->string.s; + return v->c_str(); } @@ -242,7 +241,7 @@ NixInt DrvInfo::queryMetaInt(const std::string & name, NixInt def) if (v->type() == nString) { /* Backwards compatibility with before we had support for integer meta fields. */ - if (auto n = string2Int(v->string.s)) + if (auto n = string2Int(v->c_str())) return *n; } return def; @@ -256,7 +255,7 @@ NixFloat DrvInfo::queryMetaFloat(const std::string & name, NixFloat def) if (v->type() == nString) { /* Backwards compatibility with before we had support for float meta fields. */ - if (auto n = string2Float(v->string.s)) + if (auto n = string2Float(v->c_str())) return *n; } return def; @@ -271,8 +270,8 @@ bool DrvInfo::queryMetaBool(const std::string & name, bool def) if (v->type() == nString) { /* Backwards compatibility with before we had support for Boolean meta fields. */ - if (strcmp(v->string.s, "true") == 0) return true; - if (strcmp(v->string.s, "false") == 0) return false; + if (v->string_view() == "true") return true; + if (v->string_view() == "false") return false; } return def; } diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index d243b9cec1d..b37fe6f1d80 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -43,8 +43,8 @@ $(foreach i, $(wildcard src/libexpr/value/*.hh), \ $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) -$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh +$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh -$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh +$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/call-flake.nix.gen.hh src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 4566a13887d..22be8e68c55 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -76,12 +76,12 @@ void Expr::show(const SymbolTable & symbols, std::ostream & str) const void ExprInt::show(const SymbolTable & symbols, std::ostream & str) const { - str << n; + str << v.integer; } void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const { - str << nf; + str << v.fpoint; } void ExprString::show(const SymbolTable & symbols, std::ostream & str) const diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 5ca3d1fa6c7..10099d49e9b 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -20,7 +20,6 @@ MakeError(Abort, EvalError); MakeError(TypeError, EvalError); MakeError(UndefinedVarError, Error); MakeError(MissingArgumentError, EvalError); -MakeError(RestrictedPathError, Error); /** * Position objects. @@ -155,6 +154,10 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) struct Expr { + static unsigned long nrExprs; + Expr() { + nrExprs++; + } virtual ~Expr() { }; virtual void show(const SymbolTable & symbols, std::ostream & str) const; virtual void bindVars(EvalState & es, const std::shared_ptr & env); @@ -171,18 +174,16 @@ struct Expr struct ExprInt : Expr { - NixInt n; Value v; - ExprInt(NixInt n) : n(n) { v.mkInt(n); }; + ExprInt(NixInt n) { v.mkInt(n); }; Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; struct ExprFloat : Expr { - NixFloat nf; Value v; - ExprFloat(NixFloat nf) : nf(nf) { v.mkFloat(nf); }; + ExprFloat(NixFloat nf) { v.mkFloat(nf); }; Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; @@ -198,9 +199,13 @@ struct ExprString : Expr struct ExprPath : Expr { + ref accessor; std::string s; Value v; - ExprPath(std::string s) : s(std::move(s)) { v.mkPath(this->s.c_str()); }; + ExprPath(ref accessor, std::string s) : accessor(accessor), s(std::move(s)) + { + v.mkPath(&*accessor, this->s.c_str()); + } Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; @@ -238,7 +243,7 @@ struct ExprSelect : Expr PosIdx pos; Expr * e, * def; AttrPath attrPath; - ExprSelect(const PosIdx & pos, Expr * e, const AttrPath && attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { }; + ExprSelect(const PosIdx & pos, Expr * e, AttrPath attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { }; ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); }; PosIdx getPos() const override { return pos; } COMMON_METHODS @@ -248,7 +253,7 @@ struct ExprOpHasAttr : Expr { Expr * e; AttrPath attrPath; - ExprOpHasAttr(Expr * e, const AttrPath && attrPath) : e(e), attrPath(std::move(attrPath)) { }; + ExprOpHasAttr(Expr * e, AttrPath attrPath) : e(e), attrPath(std::move(attrPath)) { }; PosIdx getPos() const override { return e->getPos(); } COMMON_METHODS }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 792f51fde32..f6cf1f6893b 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -19,6 +19,7 @@ #include #include "util.hh" +#include "users.hh" #include "nixexpr.hh" #include "eval.hh" @@ -520,7 +521,7 @@ path_start /* add back in the trailing '/' to the first segment */ if ($1.p[$1.l-1] == '/' && $1.l > 1) path += "/"; - $$ = new ExprPath(path); + $$ = new ExprPath(ref(data->state.rootFS), std::move(path)); } | HPATH { if (evalSettings.pureEval) { @@ -530,7 +531,7 @@ path_start ); } Path path(getHome() + std::string($1.p + 1, $1.l - 1)); - $$ = new ExprPath(path); + $$ = new ExprPath(ref(data->state.rootFS), std::move(path)); } ; @@ -646,13 +647,16 @@ formal #include "eval.hh" #include "filetransfer.hh" -#include "fetchers.hh" +#include "tarball.hh" #include "store-api.hh" #include "flake/flake.hh" +#include "fs-input-accessor.hh" +#include "memory-input-accessor.hh" namespace nix { +unsigned long Expr::nrExprs = 0; Expr * EvalState::parse( char * text, @@ -682,17 +686,25 @@ Expr * EvalState::parse( } -SourcePath resolveExprPath(const SourcePath & path) +SourcePath resolveExprPath(SourcePath path) { + unsigned int followCount = 0, maxFollow = 1024; + /* If `path' is a symlink, follow it. This is so that relative path references work. */ - auto path2 = path.resolveSymlinks(); + while (true) { + // Basic cycle/depth limit to avoid infinite loops. + if (++followCount >= maxFollow) + throw Error("too many symbolic links encountered while traversing the path '%s'", path); + if (path.lstat().type != InputAccessor::tSymlink) break; + path = {path.accessor, CanonPath(path.readLink(), path.path.parent().value_or(CanonPath::root))}; + } /* If `path' refers to a directory, append `/default.nix'. */ - if (path2.lstat().type == InputAccessor::tDirectory) - return path2 + "default.nix"; + if (path.lstat().type == InputAccessor::tDirectory) + return path + "default.nix"; - return path2; + return path; } @@ -755,11 +767,11 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_ auto r = *rOpt; Path res = suffix == "" ? r : concatStrings(r, "/", suffix); - if (pathExists(res)) return CanonPath(canonPath(res)); + if (pathExists(res)) return rootPath(CanonPath(canonPath(res))); } if (hasPrefix(path, "nix/")) - return CanonPath(concatStrings(corepkgsPrefix, path.substr(4))); + return {corepkgsFS, CanonPath(path.substr(3))}; debugThrow(ThrownError({ .msg = hintfmt(evalSettings.pureEval @@ -782,7 +794,7 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa if (EvalSettings::isPseudoUrl(value)) { try { auto storePath = fetchers::downloadTarball( - store, EvalSettings::resolvePseudoUrl(value), "source", false).tree.storePath; + store, EvalSettings::resolvePseudoUrl(value), "source", false).storePath; res = { store->toRealPath(storePath) }; } catch (FileTransferError & e) { logWarning({ @@ -796,7 +808,7 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false); debug("fetching flake search path element '%s''", value); - auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath; + auto storePath = flakeRef.resolve(store).fetchTree(store).first; res = { store->toRealPath(storePath) }; } diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index 1d690b72210..099607638e3 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -1,10 +1,11 @@ #include "eval.hh" +#include "fs-input-accessor.hh" namespace nix { SourcePath EvalState::rootPath(CanonPath path) { - return std::move(path); + return {rootFS, std::move(path)}; } } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ef2d7e76810..ce916d85790 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -10,6 +10,7 @@ #include "path-references.hh" #include "store-api.hh" #include "util.hh" +#include "processes.hh" #include "value-to-json.hh" #include "value-to-xml.hh" #include "primops.hh" @@ -28,7 +29,6 @@ #include - namespace nix { @@ -129,13 +129,15 @@ static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, co auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path"); try { - StringMap rewrites = state.realiseContext(context); - - auto realPath = state.rootPath(CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context))); + if (!context.empty()) { + auto rewrites = state.realiseContext(context); + auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context); + return {path.accessor, CanonPath(realPath)}; + } return flags.checkForPureEval - ? state.checkSourcePath(realPath) - : realPath; + ? state.checkSourcePath(path) + : path; } catch (Error & e) { e.addTrace(state.positions[pos], "while realising the context of path '%s'", path); throw; @@ -210,7 +212,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v state.vImportedDrvToDerivation = allocRootValue(state.allocValue()); state.eval(state.parseExprFromString( #include "imported-drv-to-derivation.nix.gen.hh" - , CanonPath::root), **state.vImportedDrvToDerivation); + , state.rootPath(CanonPath::root)), **state.vImportedDrvToDerivation); } state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh"); @@ -218,12 +220,6 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh"); } - else if (path2 == corepkgsPrefix + "fetchurl.nix") { - state.eval(state.parseExprFromString( - #include "fetchurl.nix.gen.hh" - , CanonPath::root), v); - } - else { if (!vScope) state.evalFile(path, v); @@ -266,64 +262,71 @@ static RegisterPrimOp primop_import({ .args = {"path"}, // TODO turn "normal path values" into link below .doc = R"( - Load, parse and return the Nix expression in the file *path*. - - The value *path* can be a path, a string, or an attribute set with an - `__toString` attribute or a `outPath` attribute (as derivations or flake - inputs typically have). - - If *path* is a directory, the file `default.nix` in that directory - is loaded. - - Evaluation aborts if the file doesn’t exist or contains - an incorrect Nix expression. `import` implements Nix’s module - system: you can put any Nix expression (such as a set or a - function) in a separate file, and use it from Nix expressions in - other files. + Load, parse, and return the Nix expression in the file *path*. > **Note** > > Unlike some languages, `import` is a regular function in Nix. - > Paths using the angle bracket syntax (e.g., `import` *\*) - > are normal [path values](@docroot@/language/values.md#type-path). - A Nix expression loaded by `import` must not contain any *free - variables* (identifiers that are not defined in the Nix expression - itself and are not built-in). Therefore, it cannot refer to - variables that are in scope at the call site. For instance, if you - have a calling expression + The *path* argument must meet the same criteria as an [interpolated expression](@docroot@/language/string-interpolation.md#interpolated-expression). - ```nix - rec { - x = 123; - y = import ./foo.nix; - } - ``` - - then the following `foo.nix` will give an error: + If *path* is a directory, the file `default.nix` in that directory is used if it exists. - ```nix - x + 456 - ``` - - since `x` is not in scope in `foo.nix`. If you want `x` to be - available in `foo.nix`, you should pass it as a function argument: - - ```nix - rec { - x = 123; - y = import ./foo.nix x; - } - ``` + > **Example** + > + > ```console + > $ echo 123 > default.nix + > ``` + > + > Import `default.nix` from the current directory. + > + > ```nix + > import ./. + > ``` + > + > 123 - and + Evaluation aborts if the file doesn’t exist or contains an invalid Nix expression. - ```nix - x: x + 456 - ``` + A Nix expression loaded by `import` must not contain any *free variables*, that is, identifiers that are not defined in the Nix expression itself and are not built-in. + Therefore, it cannot refer to variables that are in scope at the call site. - (The function argument doesn’t have to be called `x` in `foo.nix`; - any name would work.) + > **Example** + > + > If you have a calling expression + > + > ```nix + > rec { + > x = 123; + > y = import ./foo.nix; + > } + > ``` + > + > then the following `foo.nix` will give an error: + > + > ```nix + > # foo.nix + > x + 456 + > ``` + > + > since `x` is not in scope in `foo.nix`. + > If you want `x` to be available in `foo.nix`, pass it as a function argument: + > + > ```nix + > rec { + > x = 123; + > y = import ./foo.nix x; + > } + > ``` + > + > and + > + > ```nix + > # foo.nix + > x: x + 456 + > ``` + > + > The function argument doesn’t have to be called `x` in `foo.nix`; any name would work. )", .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { @@ -598,9 +601,12 @@ struct CompareValues case nFloat: return v1->fpoint < v2->fpoint; case nString: - return strcmp(v1->string.s, v2->string.s) < 0; + return v1->string_view().compare(v2->string_view()) < 0; case nPath: - return strcmp(v1->_path, v2->_path) < 0; + // Note: we don't take the accessor into account + // since it's not obvious how to compare them in a + // reproducible way. + return strcmp(v1->_path.path, v2->_path.path) < 0; case nList: // Lexicographic comparison for (size_t i = 0;; i++) { @@ -735,6 +741,14 @@ static RegisterPrimOp primop_genericClosure(PrimOp { ``` [ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ] ``` + + `key` can be one of the following types: + - [Number](@docroot@/language/values.md#type-number) + - [Boolean](@docroot@/language/values.md#type-boolean) + - [String](@docroot@/language/values.md#type-string) + - [Path](@docroot@/language/values.md#type-path) + - [List](@docroot@/language/values.md#list) + )", .fun = prim_genericClosure, }); @@ -818,7 +832,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * * auto message = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtins.addErrorContext", false, false).toOwned(); - e.addTrace(nullptr, message, true); + e.addTrace(nullptr, hintfmt(message), true); throw; } } @@ -990,7 +1004,7 @@ static void prim_trace(EvalState & state, const PosIdx pos, Value * * args, Valu { state.forceValue(*args[0], pos); if (args[0]->type() == nString) - printError("trace: %1%", args[0]->string.s); + printError("trace: %1%", args[0]->string_view()); else printError("trace: %1%", printValue(state, *args[0])); state.forceValue(*args[1], pos); @@ -1486,7 +1500,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, })); NixStringContext context; - auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.storePath")).path; + auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'")).path; /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink directly in the store. The latter condition is necessary so e.g. nix-push does the right thing. */ @@ -1536,14 +1550,14 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, auto path = realisePath(state, pos, arg, { .checkForPureEval = false }); /* SourcePath doesn't know about trailing slash. */ - auto mustBeDir = arg.type() == nString && arg.str().ends_with("/"); + auto mustBeDir = arg.type() == nString + && (arg.string_view().ends_with("/") + || arg.string_view().ends_with("/.")); try { auto checked = state.checkSourcePath(path); - auto exists = checked.pathExists(); - if (exists && mustBeDir) { - exists = checked.lstat().type == InputAccessor::tDirectory; - } + auto st = checked.maybeLstat(); + auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory); v.mkBool(exists); } catch (SysError & e) { /* Don't give away info from errors while canonicalising @@ -1697,13 +1711,14 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V static RegisterPrimOp primop_findFile(PrimOp { .name = "__findFile", - .args = {"search path", "lookup path"}, + .args = {"search-path", "lookup-path"}, .doc = R"( - Look up the given path with the given search path. + Find *lookup-path* in *search-path*. - A search path is represented list of [attribute sets](./values.md#attribute-set) with two attributes, `prefix`, and `path`. - `prefix` is a relative path. - `path` denotes a file system location; the exact syntax depends on the command line interface. + A search path is represented list of [attribute sets](./values.md#attribute-set) with two attributes: + - `prefix` is a relative path. + - `path` denotes a file system location + The exact syntax depends on the command line interface. Examples of search path attribute sets: @@ -1721,15 +1736,14 @@ static RegisterPrimOp primop_findFile(PrimOp { } ``` - The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/values.html#type-path) of the match. + The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/values.html#type-path) of the match: - This is the process for each entry: - If the lookup path matches `prefix`, then the remainder of the lookup path (the "suffix") is searched for within the directory denoted by `patch`. - Note that the `path` may need to be downloaded at this point to look inside. - If the suffix is found inside that directory, then the entry is a match; - the combined absolute path of the directory (now downloaded if need be) and the suffix is returned. + - If *lookup-path* matches `prefix`, then the remainder of *lookup-path* (the "suffix") is searched for within the directory denoted by `path`. + Note that the `path` may need to be downloaded at this point to look inside. + - If the suffix is found inside that directory, then the entry is a match. + The combined absolute path of the directory (now downloaded if need be) and the suffix is returned. - The syntax + [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions can be [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath): ```nix @@ -1757,7 +1771,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, V auto path = realisePath(state, pos, *args[1]); - v.mkString(hashString(*ht, path.readFile()).to_string(Base16, false)); + v.mkString(hashString(*ht, path.readFile()).to_string(HashFormat::Base16, false)); } static RegisterPrimOp primop_hashFile({ @@ -2202,7 +2216,7 @@ static void addPath( path = evalSettings.pureEval && expectedHash ? path - : state.checkSourcePath(CanonPath(path)).path.abs(); + : state.checkSourcePath(state.rootPath(CanonPath(path))).path.abs(); PathFilter filter = filterFun ? ([&](const Path & path) { auto st = lstat(path); @@ -2235,9 +2249,7 @@ static void addPath( }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { - StorePath dstPath = settings.readOnlyMode - ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first - : state.store->addToStore(name, path, method, htSHA256, filter, state.repair, refs); + auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, method, &filter, state.repair); if (expectedHash && expectedStorePath != dstPath) state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path)); state.allowAndSetStorePathString(dstPath, v); @@ -2254,7 +2266,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg { NixStringContext context; auto path = state.coerceToPath(pos, *args[1], context, - "while evaluating the second argument (the path to filter) passed to builtins.filterSource"); + "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"); state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); } @@ -2370,7 +2382,7 @@ static RegisterPrimOp primop_path({ like `@`. - filter\ - A function of the type expected by `builtins.filterSource`, + A function of the type expected by [`builtins.filterSource`](#builtins-filterSource), with the same semantics. - recursive\ @@ -2408,7 +2420,7 @@ static void prim_attrNames(EvalState & state, const PosIdx pos, Value * * args, (v.listElems()[n++] = state.allocValue())->mkString(state.symbols[i.name]); std::sort(v.listElems(), v.listElems() + n, - [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; }); + [](Value * v1, Value * v2) { return v1->string_view().compare(v2->string_view()) < 0; }); } static RegisterPrimOp primop_attrNames({ @@ -2545,11 +2557,12 @@ static void prim_removeAttrs(EvalState & state, const PosIdx pos, Value * * args /* Get the attribute names to be removed. We keep them as Attrs instead of Symbols so std::set_difference can be used to remove them from attrs[0]. */ + // 64: large enough to fit the attributes of a derivation boost::container::small_vector names; names.reserve(args[1]->listSize()); for (auto elem : args[1]->listItems()) { state.forceStringNoCtx(*elem, pos, "while evaluating the values of the second argument passed to builtins.removeAttrs"); - names.emplace_back(state.symbols.create(elem->string.s), nullptr); + names.emplace_back(state.symbols.create(elem->string_view()), nullptr); } std::sort(names.begin(), names.end()); @@ -2725,7 +2738,7 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs"); Value * res[args[1]->listSize()]; - unsigned int found = 0; + size_t found = 0; for (auto v2 : args[1]->listItems()) { state.forceAttrs(*v2, pos, "while evaluating an element in the list passed as second argument to builtins.catAttrs"); @@ -2999,7 +3012,7 @@ static RegisterPrimOp primop_tail({ .name = "__tail", .args = {"list"}, .doc = R"( - Return the second to last elements of a list; abort evaluation if + Return the list without its first item; abort evaluation if the argument isn’t a list or is an empty list. > **Warning** @@ -3061,7 +3074,7 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val // FIXME: putting this on the stack is risky. Value * vs[args[1]->listSize()]; - unsigned int k = 0; + size_t k = 0; bool same = true; for (unsigned int n = 0; n < args[1]->listSize(); ++n) { @@ -3186,10 +3199,14 @@ static void anyOrAll(bool any, EvalState & state, const PosIdx pos, Value * * ar state.forceFunction(*args[0], pos, std::string("while evaluating the first argument passed to builtins.") + (any ? "any" : "all")); state.forceList(*args[1], pos, std::string("while evaluating the second argument passed to builtins.") + (any ? "any" : "all")); + std::string_view errorCtx = any + ? "while evaluating the return value of the function passed to builtins.any" + : "while evaluating the return value of the function passed to builtins.all"; + Value vTmp; for (auto elem : args[1]->listItems()) { state.callFunction(*args[0], *elem, vTmp, pos); - bool res = state.forceBool(vTmp, pos, std::string("while evaluating the return value of the function passed to builtins.") + (any ? "any" : "all")); + bool res = state.forceBool(vTmp, pos, errorCtx); if (res == any) { v.mkBool(any); return; @@ -3451,7 +3468,7 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args, for (unsigned int n = 0; n < nrLists; ++n) { Value * vElem = args[1]->listElems()[n]; state.callFunction(*args[0], *vElem, lists[n], pos); - state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)), "while evaluating the return value of the function passed to buitlins.concatMap"); + state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)), "while evaluating the return value of the function passed to builtins.concatMap"); len += lists[n].listSize(); } @@ -3714,10 +3731,11 @@ static RegisterPrimOp primop_substring({ .doc = R"( Return the substring of *s* from character position *start* (zero-based) up to but not including *start + len*. If *start* is - greater than the length of the string, an empty string is returned, - and if *start + len* lies beyond the end of the string, only the - substring up to the end of the string is returned. *start* must be - non-negative. For example, + greater than the length of the string, an empty string is returned. + If *start + len* lies beyond the end of the string or *len* is `-1`, + only the substring up to the end of the string is returned. + *start* must be non-negative. + For example, ```nix builtins.substring 0 3 "nixos" @@ -3759,7 +3777,7 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args, NixStringContext context; // discarded auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString"); - v.mkString(hashString(*ht, s).to_string(Base16, false)); + v.mkString(hashString(*ht, s).to_string(HashFormat::Base16, false)); } static RegisterPrimOp primop_hashString({ @@ -3773,6 +3791,101 @@ static RegisterPrimOp primop_hashString({ .fun = prim_hashString, }); +static void prim_convertHash(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[0], pos, "while evaluating the first argument passed to builtins.convertHash"); + auto &inputAttrs = args[0]->attrs; + + Bindings::iterator iteratorHash = getAttr(state, state.symbols.create("hash"), inputAttrs, "while locating the attribute 'hash'"); + auto hash = state.forceStringNoCtx(*iteratorHash->value, pos, "while evaluating the attribute 'hash'"); + + Bindings::iterator iteratorHashAlgo = inputAttrs->find(state.symbols.create("hashAlgo")); + std::optional ht = std::nullopt; + if (iteratorHashAlgo != inputAttrs->end()) { + ht = parseHashType(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'")); + } + + Bindings::iterator iteratorToHashFormat = getAttr(state, state.symbols.create("toHashFormat"), args[0]->attrs, "while locating the attribute 'toHashFormat'"); + HashFormat hf = parseHashFormat(state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'")); + + v.mkString(Hash::parseAny(hash, ht).to_string(hf, hf == HashFormat::SRI)); +} + +static RegisterPrimOp primop_convertHash({ + .name = "__convertHash", + .args = {"args"}, + .doc = R"( + Return the specified representation of a hash string, based on the attributes presented in *args*: + + - `hash` + + The hash to be converted. + The hash format is detected automatically. + + - `hashAlgo` + + The algorithm used to create the hash. Must be one of + - `"md5"` + - `"sha1"` + - `"sha256"` + - `"sha512"` + + The attribute may be omitted when `hash` is an [SRI hash](https://www.w3.org/TR/SRI/#the-integrity-attribute) or when the hash is prefixed with the hash algorithm name followed by a colon. + That `:` syntax is supported for backwards compatibility with existing tooling. + + - `toHashFormat` + + The format of the resulting hash. Must be one of + - `"base16"` + - `"base32"` + - `"base64"` + - `"sri"` + + The result hash is the *toHashFormat* representation of the hash *hash*. + + > **Example** + > + > Convert a SHA256 hash in Base16 to SRI: + > + > ```nix + > builtins.convertHash { + > hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + > toHashFormat = "sri"; + > hashAlgo = "sha256"; + > } + > ``` + > + > "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" + + > **Example** + > + > Convert a SHA256 hash in SRI to Base16: + > + > ```nix + > builtins.convertHash { + > hash = "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="; + > toHashFormat = "base16"; + > } + > ``` + > + > "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + > **Example** + > + > Convert a hash in the form `:` in Base16 to SRI: + > + > ```nix + > builtins.convertHash { + > hash = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + > toHashFormat = "sri"; + > } + > ``` + > + > "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" + )", + .fun = prim_convertHash, +}); + struct RegexCache { // TODO use C++20 transparent comparison when available @@ -4395,9 +4508,9 @@ void EvalState::createBaseEnv() addConstant("__nixPath", v, { .type = nList, .doc = R"( - The search path used to resolve angle bracket path lookups. + List of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). - Angle bracket expressions can be + Lookup path expressions can be [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.findFile`](./builtins.html#builtins-findFile): @@ -4441,12 +4554,7 @@ void EvalState::createBaseEnv() /* Note: we have to initialize the 'derivation' constant *after* building baseEnv/staticBaseEnv because it uses 'builtins'. */ - char code[] = - #include "primops/derivation.nix.gen.hh" - // the parser needs two NUL bytes as terminators; one of them - // is implied by being a C string. - "\0"; - eval(parse(code, sizeof(code), derivationInternal, {CanonPath::root}, staticBaseEnv), *vDerivation); + evalFile(derivationInternal, *vDerivation); } diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 930e7f32a1a..45486608f77 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -8,6 +8,22 @@ namespace nix { +/** + * For functions where we do not expect deep recursion, we can use a sizable + * part of the stack a free allocation space. + * + * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes. + */ +constexpr size_t nonRecursiveStackReservation = 128; + +/** + * Functions that maybe applied to self-similar inputs, such as concatMap on a + * tree, should reserve a smaller part of the stack for allocation. + * + * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes. + */ +constexpr size_t conservativeStackReservation = 16; + struct RegisterPrimOp { typedef std::vector PrimOps; diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index e8542503a42..db940f277b2 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -30,20 +30,27 @@ static RegisterPrimOp primop_hasContext({ .name = "__hasContext", .args = {"s"}, .doc = R"( - Return `true` if string *s* has a non-empty context. The - context can be obtained with + Return `true` if string *s* has a non-empty context. + The context can be obtained with [`getContext`](#builtins-getContext). + + > **Example** + > + > Many operations require a string context to be empty because they are intended only to work with "regular" strings, and also to help users avoid unintentionally loosing track of string context elements. + > `builtins.hasContext` can help create better domain-specific errors in those case. + > + > ```nix + > name: meta: + > + > if builtins.hasContext name + > then throw "package name cannot contain string context" + > else { ${name} = meta; } + > ``` )", .fun = prim_hasContext }); -/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a - builder without causing the derivation to be built (for instance, - in the derivation that builds NARs in nix-push, when doing - source-only deployment). This primop marks the string context so - that builtins.derivation adds the path to drv.inputSrcs rather than - drv.inputDrvs. */ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx pos, Value * * args, Value & v) { NixStringContext context; @@ -66,11 +73,83 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p static RegisterPrimOp primop_unsafeDiscardOutputDependency({ .name = "__unsafeDiscardOutputDependency", - .arity = 1, + .args = {"s"}, + .doc = R"( + Create a copy of the given string where every "derivation deep" string context element is turned into a constant string context element. + + This is the opposite of [`builtins.addDrvOutputDependencies`](#builtins-addDrvOutputDependencies). + + This is unsafe because it allows us to "forget" store objects we would have otherwise refered to with the string context, + whereas Nix normally tracks all dependencies consistently. + Safe operations "grow" but never "shrink" string contexts. + [`builtins.addDrvOutputDependencies`] in contrast is safe because "derivation deep" string context element always refers to the underlying derivation (among many more things). + Replacing a constant string context element with a "derivation deep" element is a safe operation that just enlargens the string context without forgetting anything. + + [`builtins.addDrvOutputDependencies`]: #builtins-addDrvOutputDependencies + )", .fun = prim_unsafeDiscardOutputDependency }); +static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + NixStringContext context; + auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.addDrvOutputDependencies"); + + auto contextSize = context.size(); + if (contextSize != 1) { + throw EvalError({ + .msg = hintfmt("context of string '%s' must have exactly one element, but has %d", *s, contextSize), + .errPos = state.positions[pos] + }); + } + NixStringContext context2 { + (NixStringContextElem { std::visit(overloaded { + [&](const NixStringContextElem::Opaque & c) -> NixStringContextElem::DrvDeep { + if (!c.path.isDerivation()) { + throw EvalError({ + .msg = hintfmt("path '%s' is not a derivation", + state.store->printStorePath(c.path)), + .errPos = state.positions[pos], + }); + } + return NixStringContextElem::DrvDeep { + .drvPath = c.path, + }; + }, + [&](const NixStringContextElem::Built & c) -> NixStringContextElem::DrvDeep { + throw EvalError({ + .msg = hintfmt("`addDrvOutputDependencies` can only act on derivations, not on a derivation output such as '%1%'", c.output), + .errPos = state.positions[pos], + }); + }, + [&](const NixStringContextElem::DrvDeep & c) -> NixStringContextElem::DrvDeep { + /* Reuse original item because we want this to be idempotent. */ + return std::move(c); + }, + }, context.begin()->raw) }), + }; + + v.mkString(*s, context2); +} + +static RegisterPrimOp primop_addDrvOutputDependencies({ + .name = "__addDrvOutputDependencies", + .args = {"s"}, + .doc = R"( + Create a copy of the given string where a single consant string context element is turned into a "derivation deep" string context element. + + The store path that is the constant string context element should point to a valid derivation, and end in `.drv`. + + The original string context element must not be empty or have multiple elements, and it must not have any other type of element other than a constant or derivation deep element. + The latter is supported so this function is idempotent. + + This is the opposite of [`builtins.unsafeDiscardOutputDependency`](#builtins-addDrvOutputDependencies). + )", + .fun = prim_addDrvOutputDependencies +}); + + /* Extract the context of a string as a structured Nix value. The context is represented as an attribute set whose keys are the diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 7fe8203f453..b86ef6b93f2 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -133,7 +133,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg else if (attrName == "toPath") { state.forceValue(*attr.value, attr.pos); - bool isEmptyString = attr.value->type() == nString && attr.value->string.s == std::string(""); + bool isEmptyString = attr.value->type() == nString && attr.value->string_view() == ""; if (isEmptyString) { toPath = StorePathOrGap {}; } diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index b9ff01c16e1..e76ce455d38 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -71,10 +71,10 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a auto input = fetchers::Input::fromAttrs(std::move(attrs)); // FIXME: use name - auto [tree, input2] = input.fetch(state.store); + auto [storePath, input2] = input.fetch(state.store); auto attrs2 = state.buildBindings(8); - state.mkStorePathString(tree.storePath, attrs2.alloc(state.sOutPath)); + state.mkStorePathString(storePath, attrs2.alloc(state.sOutPath)); if (input2.getRef()) attrs2.alloc("branch").mkString(*input2.getRef()); // Backward compatibility: set 'rev' to @@ -86,7 +86,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a attrs2.alloc("revCount").mkInt(*revCount); v.mkAttrs(attrs2); - state.allowPath(tree.storePath); + state.allowPath(storePath); } static RegisterPrimOp r_fetchMercurial({ diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index f040a35109a..383ec7c5824 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -5,7 +5,9 @@ #include "fetchers.hh" #include "filetransfer.hh" #include "registry.hh" +#include "tarball.hh" #include "url.hh" +#include "value-to-json.hh" #include #include @@ -15,7 +17,7 @@ namespace nix { void emitTreeAttrs( EvalState & state, - const fetchers::Tree & tree, + const StorePath & storePath, const fetchers::Input & input, Value & v, bool emptyRevFallback, @@ -25,14 +27,13 @@ void emitTreeAttrs( auto attrs = state.buildBindings(10); - - state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath)); + state.mkStorePathString(storePath, attrs.alloc(state.sOutPath)); // FIXME: support arbitrary input attributes. auto narHash = input.getNarHash(); assert(narHash); - attrs.alloc("narHash").mkString(narHash->to_string(SRI, true)); + attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true)); if (input.getType() == "git") attrs.alloc("submodules").mkBool( @@ -71,36 +72,10 @@ void emitTreeAttrs( v.mkAttrs(attrs); } -std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file") -{ - state.checkURI(uri); - if (uri.find("://") == std::string::npos) { - const auto p = ParsedURL { - .scheme = defaultScheme, - .authority = "", - .path = uri - }; - return p.to_string(); - } else { - return uri; - } -} - -std::string fixURIForGit(std::string uri, EvalState & state) -{ - /* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes - * them by removing the `:` and assuming a scheme of `ssh://` - * */ - static std::regex scp_uri("([^/]*)@(.*):(.*)"); - if (uri[0] != '/' && std::regex_match(uri, scp_uri)) - return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh"); - else - return fixURI(uri, state); -} - struct FetchTreeParams { bool emptyRevFallback = false; bool allowNameArgument = false; + bool isFetchGit = false; }; static void fetchTree( @@ -108,11 +83,12 @@ static void fetchTree( const PosIdx pos, Value * * args, Value & v, - std::optional type, const FetchTreeParams & params = FetchTreeParams{} ) { fetchers::Input input; NixStringContext context; + std::optional type; + if (params.isFetchGit) type = "git"; state.forceValue(*args[0], pos); @@ -142,16 +118,18 @@ static void fetchTree( if (attr.value->type() == nPath || attr.value->type() == nString) { auto s = state.coerceToString(attr.pos, *attr.value, context, "", false, false).toOwned(); attrs.emplace(state.symbols[attr.name], - state.symbols[attr.name] == "url" - ? type == "git" - ? fixURIForGit(s, state) - : fixURI(s, state) + params.isFetchGit && state.symbols[attr.name] == "url" + ? fixGitURL(s) : s); } else if (attr.value->type() == nBool) attrs.emplace(state.symbols[attr.name], Explicit{attr.value->boolean}); else if (attr.value->type() == nInt) attrs.emplace(state.symbols[attr.name], uint64_t(attr.value->integer)); + else if (state.symbols[attr.name] == "publicKeys") { + experimentalFeatureSettings.require(Xp::VerifiedFetches); + attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, context).dump()); + } else state.debugThrowLastTrace(TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", state.symbols[attr.name], showType(*attr.value))); @@ -170,40 +148,87 @@ static void fetchTree( "while evaluating the first argument passed to the fetcher", false, false).toOwned(); - if (type == "git") { + if (params.isFetchGit) { fetchers::Attrs attrs; attrs.emplace("type", "git"); - attrs.emplace("url", fixURIForGit(url, state)); + attrs.emplace("url", fixGitURL(url)); input = fetchers::Input::fromAttrs(std::move(attrs)); } else { - input = fetchers::Input::fromURL(fixURI(url, state)); + if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) + state.debugThrowLastTrace(EvalError({ + .msg = hintfmt("passing a string argument to 'fetchTree' requires the 'flakes' experimental feature"), + .errPos = state.positions[pos] + })); + input = fetchers::Input::fromURL(url); } } - if (!evalSettings.pureEval && !input.isDirect()) + if (!evalSettings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) input = lookupInRegistries(state.store, input).first; if (evalSettings.pureEval && !input.isLocked()) state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", state.positions[pos])); - auto [tree, input2] = input.fetch(state.store); + state.checkURI(input.toURLString()); - state.allowPath(tree.storePath); + auto [storePath, input2] = input.fetch(state.store); - emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false); + state.allowPath(storePath); + + emitTreeAttrs(state, storePath, input2, v, params.emptyRevFallback, false); } static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - experimentalFeatureSettings.require(Xp::Flakes); - fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false }); + fetchTree(state, pos, args, v, { }); } -// FIXME: document static RegisterPrimOp primop_fetchTree({ .name = "fetchTree", - .arity = 1, - .fun = prim_fetchTree + .args = {"input"}, + .doc = R"( + Fetch a source tree or a plain file using one of the supported backends. + *input* must be a [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references), either in attribute set representation or in the URL-like syntax. + The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is enabled. + + > **Note** + > + > The URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. + + Here are some examples of how to use `fetchTree`: + + - Fetch a GitHub repository using the attribute set representation: + + ```nix + builtins.fetchTree { + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; + rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; + } + ``` + + This evaluates to the following attribute set: + + ``` + { + lastModified = 1686503798; + lastModifiedDate = "20230611171638"; + narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc="; + outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source"; + rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; + shortRev = "ae2e6b3"; + } + ``` + + - Fetch the same GitHub repository using the URL-like syntax: + + ``` + builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd" + ``` + )", + .fun = prim_fetchTree, + .experimentalFeature = Xp::FetchTree, }); static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v, @@ -270,7 +295,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v // https://github.com/NixOS/nix/issues/4313 auto storePath = unpack - ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).tree.storePath + ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; if (expectedHash) { @@ -279,7 +304,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v : hashFile(htSHA256, state.store->toRealPath(storePath)); if (hash != *expectedHash) state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", - *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true))); + *url, expectedHash->to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true))); } state.allowAndSetStorePathString(storePath, v); @@ -353,7 +378,12 @@ static RegisterPrimOp primop_fetchTarball({ static void prim_fetchGit(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true }); + fetchTree(state, pos, args, v, + FetchTreeParams { + .emptyRevFallback = true, + .allowNameArgument = true, + .isFetchGit = true + }); } static RegisterPrimOp primop_fetchGit({ @@ -368,7 +398,7 @@ static RegisterPrimOp primop_fetchGit({ The URL of the repo. - - `name` (default: *basename of the URL*) + - `name` (default: `source`) The name of the directory the repo should be exported to in the store. @@ -395,7 +425,8 @@ static RegisterPrimOp primop_fetchGit({ - `shallow` (default: `false`) - A Boolean parameter that specifies whether fetching a shallow clone is allowed. + A Boolean parameter that specifies whether fetching from a shallow remote repository is allowed. + This still performs a full clone of what is available on the remote. - `allRefs` @@ -403,6 +434,42 @@ static RegisterPrimOp primop_fetchGit({ With this argument being true, it's possible to load a `rev` from *any* `ref` (by default only `rev`s from the specified `ref` are supported). + - `verifyCommit` (default: `true` if `publicKey` or `publicKeys` are provided, otherwise `false`) + + Whether to check `rev` for a signature matching `publicKey` or `publicKeys`. + If `verifyCommit` is enabled, then `fetchGit` cannot use a local repository with uncommitted changes. + Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + + - `publicKey` + + The public key against which `rev` is verified if `verifyCommit` is enabled. + Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + + - `keytype` (default: `"ssh-ed25519"`) + + The key type of `publicKey`. + Possible values: + - `"ssh-dsa"` + - `"ssh-ecdsa"` + - `"ssh-ecdsa-sk"` + - `"ssh-ed25519"` + - `"ssh-ed25519-sk"` + - `"ssh-rsa"` + Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + + - `publicKeys` + + The public keys against which `rev` is verified if `verifyCommit` is enabled. + Must be given as a list of attribute sets with the following form: + ```nix + { + key = ""; + type = ""; # optional, default: "ssh-ed25519" + } + ``` + Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + + Here are some examples of how to use `fetchGit`. - To fetch a private repository over SSH: @@ -477,6 +544,21 @@ static RegisterPrimOp primop_fetchGit({ } ``` + - To verify the commit signature: + + ```nix + builtins.fetchGit { + url = "ssh://git@github.com/nixos/nix.git"; + verifyCommit = true; + publicKeys = [ + { + type = "ssh-ed25519"; + key = "AAAAC3NzaC1lZDI1NTE5AAAAIArPKULJOid8eS6XETwUjO48/HKBWl7FTCK0Z//fplDi"; + } + ]; + } + ``` + Nix will refetch the branch according to the [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) setting. This behavior is disabled in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). diff --git a/src/libexpr/search-path.cc b/src/libexpr/search-path.cc index 180d5f8b1a8..a2576749636 100644 --- a/src/libexpr/search-path.cc +++ b/src/libexpr/search-path.cc @@ -1,5 +1,4 @@ #include "search-path.hh" -#include "util.hh" namespace nix { diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk deleted file mode 100644 index 331a5ead60e..00000000000 --- a/src/libexpr/tests/local.mk +++ /dev/null @@ -1,19 +0,0 @@ -check: libexpr-tests_RUN - -programs += libexpr-tests - -libexpr-tests_NAME := libnixexpr-tests - -libexpr-tests_DIR := $(d) - -libexpr-tests_INSTALL_DIR := - -libexpr-tests_SOURCES := \ - $(wildcard $(d)/*.cc) \ - $(wildcard $(d)/value/*.cc) - -libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -I src/libfetchers - -libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers - -libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index ac3986c8700..74b3ebf136b 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -1,7 +1,7 @@ #include "value-to-json.hh" #include "eval-inline.hh" -#include "util.hh" #include "store-api.hh" +#include "signals.hh" #include #include @@ -31,7 +31,7 @@ json printValueAsJSON(EvalState & state, bool strict, case nString: copyContext(v, context); - out = v.string.s; + out = v.c_str(); break; case nPath: diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 2539ad1c128..5032115bbb5 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -1,7 +1,7 @@ #include "value-to-xml.hh" #include "xml-writer.hh" #include "eval-inline.hh" -#include "util.hh" +#include "signals.hh" #include @@ -74,7 +74,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, case nString: /* !!! show the context? */ copyContext(v, context); - doc.writeEmptyElement("string", singletonAttrs("value", v.string.s)); + doc.writeEmptyElement("string", singletonAttrs("value", v.c_str())); break; case nPath: @@ -96,14 +96,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, if (a != v.attrs->end()) { if (strict) state.forceValue(*a->value, a->pos); if (a->value->type() == nString) - xmlAttrs["drvPath"] = drvPath = a->value->string.s; + xmlAttrs["drvPath"] = drvPath = a->value->c_str(); } a = v.attrs->find(state.sOutPath); if (a != v.attrs->end()) { if (strict) state.forceValue(*a->value, a->pos); if (a->value->type() == nString) - xmlAttrs["outPath"] = a->value->string.s; + xmlAttrs["outPath"] = a->value->c_str(); } XMLOpenElement _(doc, "derivation", xmlAttrs); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index c44683e501e..bcff8ae55b7 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -3,6 +3,7 @@ #include #include +#include #include "symbol-table.hh" #include "value/context.hh" @@ -158,60 +159,72 @@ public: inline bool isPrimOp() const { return internalType == tPrimOp; }; inline bool isPrimOpApp() const { return internalType == tPrimOpApp; }; + /** + * Strings in the evaluator carry a so-called `context` which + * is a list of strings representing store paths. This is to + * allow users to write things like + * + * "--with-freetype2-library=" + freetype + "/lib" + * + * where `freetype` is a derivation (or a source to be copied + * to the store). If we just concatenated the strings without + * keeping track of the referenced store paths, then if the + * string is used as a derivation attribute, the derivation + * will not have the correct dependencies in its inputDrvs and + * inputSrcs. + + * The semantics of the context is as follows: when a string + * with context C is used as a derivation attribute, then the + * derivations in C will be added to the inputDrvs of the + * derivation, and the other store paths in C will be added to + * the inputSrcs of the derivations. + + * For canonicity, the store paths should be in sorted order. + */ + struct StringWithContext { + const char * c_str; + const char * * context; // must be in sorted order + }; + + struct Path { + InputAccessor * accessor; + const char * path; + }; + + struct ClosureThunk { + Env * env; + Expr * expr; + }; + + struct FunctionApplicationThunk { + Value * left, * right; + }; + + struct Lambda { + Env * env; + ExprLambda * fun; + }; + union { NixInt integer; bool boolean; - /** - * Strings in the evaluator carry a so-called `context` which - * is a list of strings representing store paths. This is to - * allow users to write things like - - * "--with-freetype2-library=" + freetype + "/lib" + StringWithContext string; - * where `freetype` is a derivation (or a source to be copied - * to the store). If we just concatenated the strings without - * keeping track of the referenced store paths, then if the - * string is used as a derivation attribute, the derivation - * will not have the correct dependencies in its inputDrvs and - * inputSrcs. + Path _path; - * The semantics of the context is as follows: when a string - * with context C is used as a derivation attribute, then the - * derivations in C will be added to the inputDrvs of the - * derivation, and the other store paths in C will be added to - * the inputSrcs of the derivations. - - * For canonicity, the store paths should be in sorted order. - */ - struct { - const char * s; - const char * * context; // must be in sorted order - } string; - - const char * _path; Bindings * attrs; struct { size_t size; Value * * elems; } bigList; Value * smallList[2]; - struct { - Env * env; - Expr * expr; - } thunk; - struct { - Value * left, * right; - } app; - struct { - Env * env; - ExprLambda * fun; - } lambda; + ClosureThunk thunk; + FunctionApplicationThunk app; + Lambda lambda; PrimOp * primOp; - struct { - Value * left, * right; - } primOpApp; + FunctionApplicationThunk primOpApp; ExternalValueBase * external; NixFloat fpoint; }; @@ -270,7 +283,7 @@ public: inline void mkString(const char * s, const char * * context = 0) { internalType = tString; - string.s = s; + string.c_str = s; string.context = context; } @@ -287,11 +300,12 @@ public: void mkPath(const SourcePath & path); - inline void mkPath(const char * path) + inline void mkPath(InputAccessor * accessor, const char * path) { clearValue(); internalType = tPath; - _path = path; + _path.accessor = accessor; + _path.path = path; } inline void mkNull() @@ -349,13 +363,7 @@ public: // Value will be overridden anyways } - inline void mkPrimOp(PrimOp * p) - { - clearValue(); - internalType = tPrimOp; - primOp = p; - } - + void mkPrimOp(PrimOp * p); inline void mkPrimOpApp(Value * l, Value * r) { @@ -388,7 +396,13 @@ public: return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems; } - const Value * const * listElems() const + std::span listItems() const + { + assert(isList()); + return std::span(listElems(), listSize()); + } + + Value * const * listElems() const { return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems; } @@ -407,44 +421,30 @@ public: */ bool isTrivial() const; - auto listItems() + SourcePath path() const { - struct ListIterable - { - typedef Value * const * iterator; - iterator _begin, _end; - iterator begin() const { return _begin; } - iterator end() const { return _end; } + assert(internalType == tPath); + return SourcePath { + .accessor = ref(_path.accessor->shared_from_this()), + .path = CanonPath(CanonPath::unchecked_t(), _path.path) }; - assert(isList()); - auto begin = listElems(); - return ListIterable { begin, begin + listSize() }; } - auto listItems() const + std::string_view string_view() const { - struct ConstListIterable - { - typedef const Value * const * iterator; - iterator _begin, _end; - iterator begin() const { return _begin; } - iterator end() const { return _end; } - }; - assert(isList()); - auto begin = listElems(); - return ConstListIterable { begin, begin + listSize() }; + assert(internalType == tString); + return std::string_view(string.c_str); } - SourcePath path() const + const char * const c_str() const { - assert(internalType == tPath); - return SourcePath{CanonPath(_path)}; + assert(internalType == tString); + return string.c_str; } - std::string_view str() const + const char * * context() const { - assert(internalType == tString); - return std::string_view(string.s); + return string.context; } }; diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index 22361d8fa24..6d9633268df 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -1,3 +1,4 @@ +#include "util.hh" #include "value/context.hh" #include diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index 9f1d5931748..51fd30a44cb 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "util.hh" #include "comparator.hh" #include "derived-path.hh" #include "variant-wrapper.hh" diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index 9f885a7935e..b9a2c824ea7 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -13,6 +13,12 @@ namespace nix::fetchers { typedef std::variant> Attr; + +/** + * An `Attrs` can be thought of a JSON object restricted or simplified + * to be "flat", not containing any subcontainers (arrays or objects) + * and also not containing any `null`s. + */ typedef std::map Attrs; Attrs jsonToAttrs(const nlohmann::json & json); diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index 0c8ecac9d48..b72a464e885 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -1,4 +1,5 @@ #include "cache.hh" +#include "users.hh" #include "sqlite.hh" #include "sync.hh" #include "store-api.hh" diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh index ae398d0404b..af34e66ce13 100644 --- a/src/libfetchers/cache.hh +++ b/src/libfetchers/cache.hh @@ -2,6 +2,7 @@ ///@file #include "fetchers.hh" +#include "path.hh" namespace nix::fetchers { diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index 6108a179cda..f095963a834 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -3,7 +3,6 @@ #include "types.hh" #include "config.hh" -#include "util.hh" #include #include diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index e683b9f804a..89551532794 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -5,12 +5,31 @@ namespace nix::fetchers { -std::unique_ptr>> inputSchemes = nullptr; +using InputSchemeMap = std::map>; + +std::unique_ptr inputSchemes = nullptr; void registerInputScheme(std::shared_ptr && inputScheme) { - if (!inputSchemes) inputSchemes = std::make_unique>>(); - inputSchemes->push_back(std::move(inputScheme)); + if (!inputSchemes) + inputSchemes = std::make_unique(); + auto schemeName = inputScheme->schemeName(); + if (inputSchemes->count(schemeName) > 0) + throw Error("Input scheme with name %s already registered", schemeName); + inputSchemes->insert_or_assign(schemeName, std::move(inputScheme)); +} + +nlohmann::json dumpRegisterInputSchemeInfo() { + using nlohmann::json; + + auto res = json::object(); + + for (auto & [name, scheme] : *inputSchemes) { + auto & r = res[name] = json::object(); + r["allowedAttrs"] = scheme->allowedAttrs(); + } + + return res; } Input Input::fromURL(const std::string & url, bool requireTree) @@ -33,9 +52,10 @@ static void fixupInput(Input & input) Input Input::fromURL(const ParsedURL & url, bool requireTree) { - for (auto & inputScheme : *inputSchemes) { + for (auto & [_, inputScheme] : *inputSchemes) { auto res = inputScheme->inputFromURL(url, requireTree); if (res) { + experimentalFeatureSettings.require(inputScheme->experimentalFeature()); res->scheme = inputScheme; fixupInput(*res); return std::move(*res); @@ -47,19 +67,44 @@ Input Input::fromURL(const ParsedURL & url, bool requireTree) Input Input::fromAttrs(Attrs && attrs) { - for (auto & inputScheme : *inputSchemes) { - auto res = inputScheme->inputFromAttrs(attrs); - if (res) { - res->scheme = inputScheme; - fixupInput(*res); - return std::move(*res); - } - } + auto schemeName = ({ + auto schemeNameOpt = maybeGetStrAttr(attrs, "type"); + if (!schemeNameOpt) + throw Error("'type' attribute to specify input scheme is required but not provided"); + *std::move(schemeNameOpt); + }); - Input input; - input.attrs = attrs; - fixupInput(input); - return input; + auto raw = [&]() { + // Return an input without a scheme; most operations will fail, + // but not all of them. Doing this is to support those other + // operations which are supposed to be robust on + // unknown/uninterpretable inputs. + Input input; + input.attrs = attrs; + fixupInput(input); + return input; + }; + + std::shared_ptr inputScheme = ({ + auto i = inputSchemes->find(schemeName); + i == inputSchemes->end() ? nullptr : i->second; + }); + + if (!inputScheme) return raw(); + + experimentalFeatureSettings.require(inputScheme->experimentalFeature()); + + auto allowedAttrs = inputScheme->allowedAttrs(); + + for (auto & [name, _] : attrs) + if (name != "type" && allowedAttrs.count(name) == 0) + throw Error("input attribute '%s' not supported by scheme '%s'", name, schemeName); + + auto res = inputScheme->inputFromAttrs(attrs); + if (!res) return raw(); + res->scheme = inputScheme; + fixupInput(*res); + return std::move(*res); } ParsedURL Input::toURL() const @@ -82,14 +127,14 @@ std::string Input::to_string() const return toURL().to_string(); } -Attrs Input::toAttrs() const +bool Input::isDirect() const { - return attrs; + return !scheme || scheme->isDirect(*this); } -bool Input::hasAllInfo() const +Attrs Input::toAttrs() const { - return getNarHash() && scheme && scheme->hasAllInfo(*this); + return attrs; } bool Input::operator ==(const Input & other) const @@ -107,7 +152,7 @@ bool Input::contains(const Input & other) const return false; } -std::pair Input::fetch(ref store) const +std::pair Input::fetch(ref store) const { if (!scheme) throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); @@ -115,7 +160,7 @@ std::pair Input::fetch(ref store) const /* The tree may already be in the Nix store, or it could be substituted (which is often faster than fetching from the original source). So check that. */ - if (hasAllInfo()) { + if (getNarHash()) { try { auto storePath = computeStorePath(*store); @@ -124,7 +169,7 @@ std::pair Input::fetch(ref store) const debug("using substituted/cached input '%s' in '%s'", to_string(), store->printStorePath(storePath)); - return {Tree { .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath) }, *this}; + return {std::move(storePath), *this}; } catch (Error & e) { debug("substitution of input '%s' failed: %s", to_string(), e.what()); } @@ -139,18 +184,16 @@ std::pair Input::fetch(ref store) const } }(); - Tree tree { - .actualPath = store->toRealPath(storePath), - .storePath = storePath, - }; - - auto narHash = store->queryPathInfo(tree.storePath)->narHash; - input.attrs.insert_or_assign("narHash", narHash.to_string(SRI, true)); + auto narHash = store->queryPathInfo(storePath)->narHash; + input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); if (auto prevNarHash = getNarHash()) { if (narHash != *prevNarHash) throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", - to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash.to_string(SRI, true)); + to_string(), + store->printStorePath(storePath), + prevNarHash->to_string(HashFormat::SRI, true), + narHash.to_string(HashFormat::SRI, true)); } if (auto prevLastModified = getLastModified()) { @@ -173,9 +216,7 @@ std::pair Input::fetch(ref store) const input.locked = true; - assert(input.hasAllInfo()); - - return {std::move(tree), input}; + return {std::move(storePath), input}; } Input Input::applyOverrides( @@ -198,12 +239,13 @@ std::optional Input::getSourcePath() const return scheme->getSourcePath(*this); } -void Input::markChangedFile( - std::string_view file, +void Input::putFile( + const CanonPath & path, + std::string_view contents, std::optional commitMsg) const { assert(scheme); - return scheme->markChangedFile(*this, file, commitMsg); + return scheme->putFile(*this, path, contents, commitMsg); } std::string Input::getName() const @@ -254,7 +296,8 @@ std::optional Input::getRev() const try { hash = Hash::parseAnyPrefixed(*s); } catch (BadHash &e) { - // Default to sha1 for backwards compatibility with existing flakes + // Default to sha1 for backwards compatibility with existing + // usages (e.g. `builtins.fetchTree` calls or flake inputs). hash = Hash::parseAny(*s, htSHA1); } } @@ -293,14 +336,18 @@ Input InputScheme::applyOverrides( return input; } -std::optional InputScheme::getSourcePath(const Input & input) +std::optional InputScheme::getSourcePath(const Input & input) const { return {}; } -void InputScheme::markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) +void InputScheme::putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const { - assert(false); + throw Error("input '%s' does not support modifying file '%s'", input.to_string(), path); } void InputScheme::clone(const Input & input, const Path & destDir) const @@ -308,4 +355,14 @@ void InputScheme::clone(const Input & input, const Path & destDir) const throw Error("do not know how to clone input '%s'", input.to_string()); } +std::optional InputScheme::experimentalFeature() const +{ + return {}; +} + +std::string publicKeys_to_string(const std::vector& publicKeys) +{ + return ((nlohmann::json) publicKeys).dump(); +} + } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 6e10e95134f..a056c8939f0 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -3,31 +3,25 @@ #include "types.hh" #include "hash.hh" -#include "path.hh" +#include "canon-path.hh" #include "attrs.hh" #include "url.hh" #include +#include -namespace nix { class Store; } +namespace nix { class Store; class StorePath; } namespace nix::fetchers { -struct Tree -{ - Path actualPath; - StorePath storePath; -}; - struct InputScheme; /** - * The Input object is generated by a specific fetcher, based on the - * user-supplied input attribute in the flake.nix file, and contains + * The `Input` object is generated by a specific fetcher, based on + * user-supplied information, and contains * the information that the specific fetcher needs to perform the * actual fetch. The Input object is most commonly created via the - * "fromURL()" or "fromAttrs()" static functions which are provided - * the url or attrset specified in the flake file. + * `fromURL()` or `fromAttrs()` static functions. */ struct Input { @@ -36,7 +30,6 @@ struct Input std::shared_ptr scheme; // note: can be null Attrs attrs; bool locked = false; - bool direct = true; /** * path of the parent of this input, used for relative path resolution @@ -44,10 +37,20 @@ struct Input std::optional parent; public: + /** + * Create an `Input` from a URL. + * + * The URL indicate which sort of fetcher, and provides information to that fetcher. + */ static Input fromURL(const std::string & url, bool requireTree = true); static Input fromURL(const ParsedURL & url, bool requireTree = true); + /** + * Create an `Input` from a an `Attrs`. + * + * The URL indicate which sort of fetcher, and provides information to that fetcher. + */ static Input fromAttrs(Attrs && attrs); ParsedURL toURL() const; @@ -62,7 +65,7 @@ public: * Check whether this is a "direct" input, that is, not * one that goes through a registry. */ - bool isDirect() const { return direct; } + bool isDirect() const; /** * Check whether this is a "locked" input, that is, @@ -70,24 +73,15 @@ public: */ bool isLocked() const { return locked; } - /** - * Check whether the input carries all necessary info required - * for cache insertion and substitution. - * These fields are used to uniquely identify cached trees - * within the "tarball TTL" window without necessarily - * indicating that the input's origin is unchanged. - */ - bool hasAllInfo() const; - bool operator ==(const Input & other) const; bool contains(const Input & other) const; /** - * Fetch the input into the Nix store, returning the location in - * the Nix store and the locked input. + * Fetch the entire input into the Nix store, returning the + * location in the Nix store and the locked input. */ - std::pair fetch(ref store) const; + std::pair fetch(ref store) const; Input applyOverrides( std::optional ref, @@ -97,8 +91,13 @@ public: std::optional getSourcePath() const; - void markChangedFile( - std::string_view file, + /** + * Write a file to this input, for input types that support + * writing. Optionally commit the change (for e.g. Git inputs). + */ + void putFile( + const CanonPath & path, + std::string_view contents, std::optional commitMsg) const; std::string getName() const; @@ -116,13 +115,13 @@ public: /** - * The InputScheme represents a type of fetcher. Each fetcher - * registers with nix at startup time. When processing an input for a - * flake, each scheme is given an opportunity to "recognize" that - * input from the url or attributes in the flake file's specification - * and return an Input object to represent the input if it is - * recognized. The Input object contains the information the fetcher - * needs to actually perform the "fetch()" when called. + * The `InputScheme` represents a type of fetcher. Each fetcher + * registers with nix at startup time. When processing an `Input`, + * each scheme is given an opportunity to "recognize" that + * input from the user-provided url or attributes + * and return an `Input` object to represent the input if it is + * recognized. The `Input` object contains the information the fetcher + * needs to actually perform the `fetch()` when called. */ struct InputScheme { @@ -133,9 +132,25 @@ struct InputScheme virtual std::optional inputFromAttrs(const Attrs & attrs) const = 0; - virtual ParsedURL toURL(const Input & input) const; + /** + * What is the name of the scheme? + * + * The `type` attribute is used to select which input scheme is + * used, and then the other fields are forwarded to that input + * scheme. + */ + virtual std::string_view schemeName() const = 0; - virtual bool hasAllInfo(const Input & input) const = 0; + /** + * Allowed attributes in an attribute set that is converted to an + * input. + * + * `type` is not included from this set, because the `type` field is + parsed first to choose which scheme; `type` is always required. + */ + virtual StringSet allowedAttrs() const = 0; + + virtual ParsedURL toURL(const Input & input) const; virtual Input applyOverrides( const Input & input, @@ -144,42 +159,36 @@ struct InputScheme virtual void clone(const Input & input, const Path & destDir) const; - virtual std::optional getSourcePath(const Input & input); + virtual std::optional getSourcePath(const Input & input) const; - virtual void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg); + virtual void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const; virtual std::pair fetch(ref store, const Input & input) = 0; -}; -void registerInputScheme(std::shared_ptr && fetcher); + /** + * Is this `InputScheme` part of an experimental feature? + */ + virtual std::optional experimentalFeature() const; -struct DownloadFileResult -{ - StorePath storePath; - std::string etag; - std::string effectiveUrl; - std::optional immutableUrl; + virtual bool isDirect(const Input & input) const + { return true; } }; -DownloadFileResult downloadFile( - ref store, - const std::string & url, - const std::string & name, - bool locked, - const Headers & headers = {}); +void registerInputScheme(std::shared_ptr && fetcher); + +nlohmann::json dumpRegisterInputSchemeInfo(); -struct DownloadTarballResult +struct PublicKey { - Tree tree; - time_t lastModified; - std::optional immutableUrl; + std::string type = "ssh-ed25519"; + std::string key; }; +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(PublicKey, type, key) -DownloadTarballResult downloadTarball( - ref store, - const std::string & url, - const std::string & name, - bool locked, - const Headers & headers = {}); +std::string publicKeys_to_string(const std::vector&); } diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc new file mode 100644 index 00000000000..81be64482fb --- /dev/null +++ b/src/libfetchers/fs-input-accessor.cc @@ -0,0 +1,130 @@ +#include "fs-input-accessor.hh" +#include "posix-source-accessor.hh" +#include "store-api.hh" + +namespace nix { + +struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor +{ + CanonPath root; + std::optional> allowedPaths; + MakeNotAllowedError makeNotAllowedError; + + FSInputAccessorImpl( + const CanonPath & root, + std::optional> && allowedPaths, + MakeNotAllowedError && makeNotAllowedError) + : root(root) + , allowedPaths(std::move(allowedPaths)) + , makeNotAllowedError(std::move(makeNotAllowedError)) + { + } + + void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) override + { + auto absPath = makeAbsPath(path); + checkAllowed(absPath); + PosixSourceAccessor::readFile(absPath, sink, sizeCallback); + } + + bool pathExists(const CanonPath & path) override + { + auto absPath = makeAbsPath(path); + return isAllowed(absPath) && PosixSourceAccessor::pathExists(absPath); + } + + std::optional maybeLstat(const CanonPath & path) override + { + auto absPath = makeAbsPath(path); + checkAllowed(absPath); + return PosixSourceAccessor::maybeLstat(absPath); + } + + DirEntries readDirectory(const CanonPath & path) override + { + auto absPath = makeAbsPath(path); + checkAllowed(absPath); + DirEntries res; + for (auto & entry : PosixSourceAccessor::readDirectory(absPath)) + if (isAllowed(absPath + entry.first)) + res.emplace(entry); + return res; + } + + std::string readLink(const CanonPath & path) override + { + auto absPath = makeAbsPath(path); + checkAllowed(absPath); + return PosixSourceAccessor::readLink(absPath); + } + + CanonPath makeAbsPath(const CanonPath & path) + { + return root + path; + } + + void checkAllowed(const CanonPath & absPath) override + { + if (!isAllowed(absPath)) + throw makeNotAllowedError + ? makeNotAllowedError(absPath) + : RestrictedPathError("access to path '%s' is forbidden", absPath); + } + + bool isAllowed(const CanonPath & absPath) + { + if (!absPath.isWithin(root)) + return false; + + if (allowedPaths) { + auto p = absPath.removePrefix(root); + if (!p.isAllowed(*allowedPaths)) + return false; + } + + return true; + } + + void allowPath(CanonPath path) override + { + if (allowedPaths) + allowedPaths->insert(std::move(path)); + } + + bool hasAccessControl() override + { + return (bool) allowedPaths; + } + + std::optional getPhysicalPath(const CanonPath & path) override + { + return makeAbsPath(path); + } +}; + +ref makeFSInputAccessor( + const CanonPath & root, + std::optional> && allowedPaths, + MakeNotAllowedError && makeNotAllowedError) +{ + return make_ref(root, std::move(allowedPaths), std::move(makeNotAllowedError)); +} + +ref makeStorePathAccessor( + ref store, + const StorePath & storePath, + MakeNotAllowedError && makeNotAllowedError) +{ + return makeFSInputAccessor(CanonPath(store->toRealPath(storePath)), {}, std::move(makeNotAllowedError)); +} + +SourcePath getUnfilteredRootPath(CanonPath path) +{ + static auto rootFS = makeFSInputAccessor(CanonPath::root); + return {rootFS, path}; +} + +} diff --git a/src/libfetchers/fs-input-accessor.hh b/src/libfetchers/fs-input-accessor.hh new file mode 100644 index 00000000000..19a5211c8b4 --- /dev/null +++ b/src/libfetchers/fs-input-accessor.hh @@ -0,0 +1,33 @@ +#pragma once + +#include "input-accessor.hh" + +namespace nix { + +class StorePath; +class Store; + +struct FSInputAccessor : InputAccessor +{ + virtual void checkAllowed(const CanonPath & absPath) = 0; + + virtual void allowPath(CanonPath path) = 0; + + virtual bool hasAccessControl() = 0; +}; + +typedef std::function MakeNotAllowedError; + +ref makeFSInputAccessor( + const CanonPath & root, + std::optional> && allowedPaths = {}, + MakeNotAllowedError && makeNotAllowedError = {}); + +ref makeStorePathAccessor( + ref store, + const StorePath & storePath, + MakeNotAllowedError && makeNotAllowedError = {}); + +SourcePath getUnfilteredRootPath(CanonPath path); + +} diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index f8d89ab2fcd..cc735996bc6 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -1,11 +1,12 @@ #include "fetchers.hh" +#include "users.hh" #include "cache.hh" #include "globals.hh" #include "tarfile.hh" #include "store-api.hh" #include "url-parts.hh" #include "pathlocks.hh" -#include "util.hh" +#include "processes.hh" #include "git.hh" #include "fetch-settings.hh" @@ -46,7 +47,7 @@ bool touchCacheFile(const Path & path, time_t touch_time) Path getCachePath(std::string_view key) { return getCacheDir() + "/nix/gitv3/" + - hashString(htSHA256, key).to_string(Base32, false); + hashString(htSHA256, key).to_string(HashFormat::Base32, false); } // Returns the name of the HEAD branch. @@ -143,6 +144,69 @@ struct WorkdirInfo bool hasHead = false; }; +std::vector getPublicKeys(const Attrs & attrs) { + std::vector publicKeys; + if (attrs.contains("publicKeys")) { + nlohmann::json publicKeysJson = nlohmann::json::parse(getStrAttr(attrs, "publicKeys")); + ensureType(publicKeysJson, nlohmann::json::value_t::array); + publicKeys = publicKeysJson.get>(); + } + else { + publicKeys = {}; + } + if (attrs.contains("publicKey")) + publicKeys.push_back(PublicKey{maybeGetStrAttr(attrs, "keytype").value_or("ssh-ed25519"),getStrAttr(attrs, "publicKey")}); + return publicKeys; +} + +void doCommitVerification(const Path repoDir, const Path gitDir, const std::string rev, const std::vector& publicKeys) { + // Create ad-hoc allowedSignersFile and populate it with publicKeys + auto allowedSignersFile = createTempFile().second; + std::string allowedSigners; + for (const PublicKey& k : publicKeys) { + if (k.type != "ssh-dsa" + && k.type != "ssh-ecdsa" + && k.type != "ssh-ecdsa-sk" + && k.type != "ssh-ed25519" + && k.type != "ssh-ed25519-sk" + && k.type != "ssh-rsa") + warn("Unknown keytype: %s\n" + "Please use one of\n" + "- ssh-dsa\n" + " ssh-ecdsa\n" + " ssh-ecdsa-sk\n" + " ssh-ed25519\n" + " ssh-ed25519-sk\n" + " ssh-rsa", k.type); + allowedSigners += "* " + k.type + " " + k.key + "\n"; + } + writeFile(allowedSignersFile, allowedSigners); + + // Run verification command + auto [status, output] = runProgram(RunOptions { + .program = "git", + .args = {"-c", "gpg.ssh.allowedSignersFile=" + allowedSignersFile, "-C", repoDir, + "--git-dir", gitDir, "verify-commit", rev}, + .mergeStderrToStdout = true, + }); + + /* Evaluate result through status code and checking if public key fingerprints appear on stderr + * This is neccessary because the git command might also succeed due to the commit being signed by gpg keys + * that are present in the users key agent. */ + std::string re = R"(Good "git" signature for \* with .* key SHA256:[)"; + for (const PublicKey& k : publicKeys){ + // Calculate sha256 fingerprint from public key and escape the regex symbol '+' to match the key literally + auto fingerprint = trim(hashString(htSHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "="); + auto escaped_fingerprint = std::regex_replace(fingerprint, std::regex("\\+"), "\\+" ); + re += "(" + escaped_fingerprint + ")"; + } + re += "]"; + if (status == 0 && std::regex_search(output, std::regex(re))) + printTalkative("Signature verification on commit %s succeeded", rev); + else + throw Error("Commit signature verification on commit %s failed: \n%s", rev, output); +} + // Returns whether a git workdir is clean and has commits. WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir) { @@ -272,9 +336,9 @@ struct GitInputScheme : InputScheme attrs.emplace("type", "git"); for (auto & [name, value] : url.query) { - if (name == "rev" || name == "ref") + if (name == "rev" || name == "ref" || name == "keytype" || name == "publicKey" || name == "publicKeys") attrs.emplace(name, value); - else if (name == "shallow" || name == "submodules" || name == "allRefs") + else if (name == "shallow" || name == "submodules" || name == "allRefs" || name == "verifyCommit") attrs.emplace(name, Explicit { value == "1" }); else url2.query.emplace(name, value); @@ -285,18 +349,47 @@ struct GitInputScheme : InputScheme return inputFromAttrs(attrs); } - std::optional inputFromAttrs(const Attrs & attrs) const override + + std::string_view schemeName() const override { - if (maybeGetStrAttr(attrs, "type") != "git") return {}; + return "git"; + } - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name" && name != "dirtyRev" && name != "dirtyShortRev") - throw Error("unsupported Git input attribute '%s'", name); + StringSet allowedAttrs() const override + { + return { + "url", + "ref", + "rev", + "shallow", + "submodules", + "lastModified", + "revCount", + "narHash", + "allRefs", + "name", + "dirtyRev", + "dirtyShortRev", + "verifyCommit", + "keytype", + "publicKey", + "publicKeys", + }; + } + + std::optional inputFromAttrs(const Attrs & attrs) const override + { + for (auto & [name, _] : attrs) + if (name == "verifyCommit" + || name == "keytype" + || name == "publicKey" + || name == "publicKeys") + experimentalFeatureSettings.require(Xp::VerifiedFetches); - parseURL(getStrAttr(attrs, "url")); maybeGetBoolAttr(attrs, "shallow"); maybeGetBoolAttr(attrs, "submodules"); maybeGetBoolAttr(attrs, "allRefs"); + maybeGetBoolAttr(attrs, "verifyCommit"); if (auto ref = maybeGetStrAttr(attrs, "ref")) { if (std::regex_search(*ref, badGitRefRegex)) @@ -305,6 +398,9 @@ struct GitInputScheme : InputScheme Input input; input.attrs = attrs; + auto url = fixGitURL(getStrAttr(attrs, "url")); + parseURL(url); + input.attrs["url"] = url; return input; } @@ -316,18 +412,18 @@ struct GitInputScheme : InputScheme if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref); if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false)) url.query.insert_or_assign("shallow", "1"); + if (maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(false)) + url.query.insert_or_assign("verifyCommit", "1"); + auto publicKeys = getPublicKeys(input.attrs); + if (publicKeys.size() == 1) { + url.query.insert_or_assign("keytype", publicKeys.at(0).type); + url.query.insert_or_assign("publicKey", publicKeys.at(0).key); + } + else if (publicKeys.size() > 1) + url.query.insert_or_assign("publicKeys", publicKeys_to_string(publicKeys)); return url; } - bool hasAllInfo(const Input & input) const override - { - bool maybeDirty = !input.getRef(); - bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); - return - maybeGetIntAttr(input.attrs, "lastModified") - && (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount")); - } - Input applyOverrides( const Input & input, std::optional ref, @@ -361,7 +457,7 @@ struct GitInputScheme : InputScheme runProgram("git", true, args, {}, true); } - std::optional getSourcePath(const Input & input) override + std::optional getSourcePath(const Input & input) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); if (url.scheme == "file" && !input.getRef() && !input.getRev()) @@ -369,18 +465,26 @@ struct GitInputScheme : InputScheme return {}; } - void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override + void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const override { - auto sourcePath = getSourcePath(input); - assert(sourcePath); + auto root = getSourcePath(input); + if (!root) + throw Error("cannot commit '%s' to Git repository '%s' because it's not a working tree", path, input.to_string()); + + writeFile((CanonPath(*root) + path).abs(), contents); + auto gitDir = ".git"; runProgram("git", true, - { "-C", *sourcePath, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(file) }); + { "-C", *root, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) }); if (commitMsg) runProgram("git", true, - { "-C", *sourcePath, "--git-dir", gitDir, "commit", std::string(file), "-m", *commitMsg }); + { "-C", *root, "--git-dir", gitDir, "commit", std::string(path.rel()), "-m", *commitMsg }); } std::pair getActualUrl(const Input & input) const @@ -406,6 +510,8 @@ struct GitInputScheme : InputScheme bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false); + std::vector publicKeys = getPublicKeys(input.attrs); + bool verifyCommit = maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(!publicKeys.empty()); std::string cacheType = "git"; if (shallow) cacheType += "-shallow"; @@ -415,7 +521,7 @@ struct GitInputScheme : InputScheme auto checkHashType = [&](const std::optional & hash) { if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256)) - throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(Base16, true)); + throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true)); }; auto getLockedAttrs = [&]() @@ -426,6 +532,8 @@ struct GitInputScheme : InputScheme {"type", cacheType}, {"name", name}, {"rev", input.getRev()->gitRev()}, + {"verifyCommit", verifyCommit}, + {"publicKeys", publicKeys_to_string(publicKeys)}, }); }; @@ -448,12 +556,15 @@ struct GitInputScheme : InputScheme auto [isLocal, actualUrl_] = getActualUrl(input); auto actualUrl = actualUrl_; // work around clang bug - /* If this is a local directory and no ref or revision is given, + /* If this is a local directory, no ref or revision is given and no signature verification is needed, allow fetching directly from a dirty workdir. */ if (!input.getRef() && !input.getRev() && isLocal) { auto workdirInfo = getWorkdirInfo(input, actualUrl); if (!workdirInfo.clean) { - return fetchFromWorkdir(store, input, actualUrl, workdirInfo); + if (verifyCommit) + throw Error("Can't fetch from a dirty workdir with commit signature verification enabled."); + else + return fetchFromWorkdir(store, input, actualUrl, workdirInfo); } } @@ -461,6 +572,8 @@ struct GitInputScheme : InputScheme {"type", cacheType}, {"name", name}, {"url", actualUrl}, + {"verifyCommit", verifyCommit}, + {"publicKeys", publicKeys_to_string(publicKeys)}, }); Path repoDir; @@ -618,6 +731,9 @@ struct GitInputScheme : InputScheme ); } + if (verifyCommit) + doCommitVerification(repoDir, gitDir, input.getRev()->gitRev(), publicKeys); + if (submodules) { Path tmpGitDir = createTempDir(); AutoDelete delTmpGitDir(tmpGitDir, true); diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 291f457f0d3..6c9b29721e3 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -7,6 +7,7 @@ #include "git.hh" #include "fetchers.hh" #include "fetch-settings.hh" +#include "tarball.hh" #include #include @@ -26,13 +27,11 @@ std::regex hostRegex(hostRegexS, std::regex::ECMAScript); struct GitArchiveInputScheme : InputScheme { - virtual std::string type() const = 0; - virtual std::optional> accessHeaderFromToken(const std::string & token) const = 0; std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { - if (url.scheme != type()) return {}; + if (url.scheme != schemeName()) return {}; auto path = tokenizeString>(url.path, "/"); @@ -90,7 +89,7 @@ struct GitArchiveInputScheme : InputScheme throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev()); Input input; - input.attrs.insert_or_assign("type", type()); + input.attrs.insert_or_assign("type", std::string { schemeName() }); input.attrs.insert_or_assign("owner", path[0]); input.attrs.insert_or_assign("repo", path[1]); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); @@ -100,14 +99,21 @@ struct GitArchiveInputScheme : InputScheme return input; } - std::optional inputFromAttrs(const Attrs & attrs) const override + StringSet allowedAttrs() const override { - if (maybeGetStrAttr(attrs, "type") != type()) return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host") - throw Error("unsupported input attribute '%s'", name); + return { + "owner", + "repo", + "ref", + "rev", + "narHash", + "lastModified", + "host", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override + { getStrAttr(attrs, "owner"); getStrAttr(attrs, "repo"); @@ -125,18 +131,13 @@ struct GitArchiveInputScheme : InputScheme auto path = owner + "/" + repo; assert(!(ref && rev)); if (ref) path += "/" + *ref; - if (rev) path += "/" + rev->to_string(Base16, false); + if (rev) path += "/" + rev->to_string(HashFormat::Base16, false); return ParsedURL { - .scheme = type(), + .scheme = std::string { schemeName() }, .path = path, }; } - bool hasAllInfo(const Input & input) const override - { - return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified"); - } - Input applyOverrides( const Input & _input, std::optional ref, @@ -218,16 +219,21 @@ struct GitArchiveInputScheme : InputScheme {"rev", rev->gitRev()}, {"lastModified", uint64_t(result.lastModified)} }, - result.tree.storePath, + result.storePath, true); - return {result.tree.storePath, input}; + return {result.storePath, input}; + } + + std::optional experimentalFeature() const override + { + return Xp::Flakes; } }; struct GitHubInputScheme : GitArchiveInputScheme { - std::string type() const override { return "github"; } + std::string_view schemeName() const override { return "github"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -291,7 +297,7 @@ struct GitHubInputScheme : GitArchiveInputScheme : "https://api.%s/repos/%s/%s/tarball/%s"; const auto url = fmt(urlFmt, host, getOwner(input), getRepo(input), - input.getRev()->to_string(Base16, false)); + input.getRev()->to_string(HashFormat::Base16, false)); return DownloadUrl { url, headers }; } @@ -308,7 +314,7 @@ struct GitHubInputScheme : GitArchiveInputScheme struct GitLabInputScheme : GitArchiveInputScheme { - std::string type() const override { return "gitlab"; } + std::string_view schemeName() const override { return "gitlab"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -357,7 +363,7 @@ struct GitLabInputScheme : GitArchiveInputScheme auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), - input.getRev()->to_string(Base16, false)); + input.getRev()->to_string(HashFormat::Base16, false)); Headers headers = makeHeadersWithAuthTokens(host); return DownloadUrl { url, headers }; @@ -376,7 +382,7 @@ struct GitLabInputScheme : GitArchiveInputScheme struct SourceHutInputScheme : GitArchiveInputScheme { - std::string type() const override { return "sourcehut"; } + std::string_view schemeName() const override { return "sourcehut"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -444,7 +450,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); auto url = fmt("https://%s/%s/%s/archive/%s.tar.gz", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), - input.getRev()->to_string(Base16, false)); + input.getRev()->to_string(HashFormat::Base16, false)); Headers headers = makeHeadersWithAuthTokens(host); return DownloadUrl { url, headers }; diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 4874a43ff97..8e30284c6ac 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -1,5 +1,6 @@ #include "fetchers.hh" #include "url-parts.hh" +#include "path.hh" namespace nix::fetchers { @@ -41,7 +42,6 @@ struct IndirectInputScheme : InputScheme // FIXME: forbid query params? Input input; - input.direct = false; input.attrs.insert_or_assign("type", "indirect"); input.attrs.insert_or_assign("id", id); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); @@ -50,20 +50,28 @@ struct IndirectInputScheme : InputScheme return input; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::string_view schemeName() const override { - if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; + return "indirect"; + } - for (auto & [name, value] : attrs) - if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash") - throw Error("unsupported indirect input attribute '%s'", name); + StringSet allowedAttrs() const override + { + return { + "id", + "ref", + "rev", + "narHash", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override + { auto id = getStrAttr(attrs, "id"); if (!std::regex_match(id, flakeRegex)) throw BadURL("'%s' is not a valid flake ID", id); Input input; - input.direct = false; input.attrs = attrs; return input; } @@ -78,11 +86,6 @@ struct IndirectInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) const override - { - return false; - } - Input applyOverrides( const Input & _input, std::optional ref, @@ -98,6 +101,14 @@ struct IndirectInputScheme : InputScheme { throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); } + + std::optional experimentalFeature() const override + { + return Xp::Flakes; + } + + bool isDirect(const Input & input) const override + { return false; } }; static auto rIndirectInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index f37a8058b68..d1d450cf71f 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -3,75 +3,67 @@ namespace nix { -std::ostream & operator << (std::ostream & str, const SourcePath & path) +StorePath InputAccessor::fetchToStore( + ref store, + const CanonPath & path, + std::string_view name, + FileIngestionMethod method, + PathFilter * filter, + RepairFlag repair) { - str << path.to_string(); - return str; -} + Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path))); -std::string_view SourcePath::baseName() const -{ - return path.baseName().value_or("source"); -} + auto source = sinkToSource([&](Sink & sink) { + if (method == FileIngestionMethod::Recursive) + dumpPath(path, sink, filter ? *filter : defaultPathFilter); + else + readFile(path, sink); + }); -SourcePath SourcePath::parent() const -{ - auto p = path.parent(); - assert(p); - return std::move(*p); -} + auto storePath = + settings.readOnlyMode + ? store->computeStorePathFromDump(*source, name, method, htSHA256).first + : store->addToStoreFromDump(*source, name, method, htSHA256, repair); -InputAccessor::Stat SourcePath::lstat() const -{ - auto st = nix::lstat(path.abs()); - return InputAccessor::Stat { - .type = - S_ISREG(st.st_mode) ? InputAccessor::tRegular : - S_ISDIR(st.st_mode) ? InputAccessor::tDirectory : - S_ISLNK(st.st_mode) ? InputAccessor::tSymlink : - InputAccessor::tMisc, - .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR - }; + return storePath; } -std::optional SourcePath::maybeLstat() const +SourcePath InputAccessor::root() { - // FIXME: merge these into one operation. - if (!pathExists()) - return {}; - return lstat(); + return {ref(shared_from_this()), CanonPath::root}; } -InputAccessor::DirEntries SourcePath::readDirectory() const +std::ostream & operator << (std::ostream & str, const SourcePath & path) { - InputAccessor::DirEntries res; - for (auto & entry : nix::readDirectory(path.abs())) { - std::optional type; - switch (entry.type) { - case DT_REG: type = InputAccessor::Type::tRegular; break; - case DT_LNK: type = InputAccessor::Type::tSymlink; break; - case DT_DIR: type = InputAccessor::Type::tDirectory; break; - } - res.emplace(entry.name, type); - } - return res; + str << path.to_string(); + return str; } StorePath SourcePath::fetchToStore( ref store, std::string_view name, + FileIngestionMethod method, PathFilter * filter, RepairFlag repair) const { - return - settings.readOnlyMode - ? store->computeStorePathForPath(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter).first - : store->addToStore(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter, repair); + return accessor->fetchToStore(store, path, name, method, filter, repair); +} + +std::string_view SourcePath::baseName() const +{ + return path.baseName().value_or("source"); +} + +SourcePath SourcePath::parent() const +{ + auto p = path.parent(); + assert(p); + return {accessor, std::move(*p)}; } SourcePath SourcePath::resolveSymlinks() const { - SourcePath res(CanonPath::root); + auto res = accessor->root(); int linksAllowed = 1024; diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index 5a2f17f6288..9c688a234ca 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -1,40 +1,41 @@ #pragma once +///@file +#include "source-accessor.hh" #include "ref.hh" #include "types.hh" -#include "archive.hh" -#include "canon-path.hh" +#include "file-system.hh" #include "repair-flag.hh" +#include "content-address.hh" namespace nix { +MakeError(RestrictedPathError, Error); + +struct SourcePath; class StorePath; class Store; -struct InputAccessor +struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this { - enum Type { - tRegular, tSymlink, tDirectory, - /** - Any other node types that may be encountered on the file system, such as device nodes, sockets, named pipe, and possibly even more exotic things. - - Responsible for `"unknown"` from `builtins.readFileType "/dev/null"`. - - Unlike `DT_UNKNOWN`, this must not be used for deferring the lookup of types. - */ - tMisc - }; - - struct Stat + /** + * Return the maximum last-modified time of the files in this + * tree, if available. + */ + virtual std::optional getLastModified() { - Type type = tMisc; - //uint64_t fileSize = 0; // regular files only - bool isExecutable = false; // regular files only - }; + return std::nullopt; + } - typedef std::optional DirEntry; + StorePath fetchToStore( + ref store, + const CanonPath & path, + std::string_view name = "source", + FileIngestionMethod method = FileIngestionMethod::Recursive, + PathFilter * filter = nullptr, + RepairFlag repair = NoRepair); - typedef std::map DirEntries; + SourcePath root(); }; /** @@ -45,12 +46,9 @@ struct InputAccessor */ struct SourcePath { + ref accessor; CanonPath path; - SourcePath(CanonPath path) - : path(std::move(path)) - { } - std::string_view baseName() const; /** @@ -64,39 +62,42 @@ struct SourcePath * return its contents; otherwise throw an error. */ std::string readFile() const - { return nix::readFile(path.abs()); } + { return accessor->readFile(path); } /** * Return whether this `SourcePath` denotes a file (of any type) * that exists */ bool pathExists() const - { return nix::pathExists(path.abs()); } + { return accessor->pathExists(path); } /** * Return stats about this `SourcePath`, or throw an exception if * it doesn't exist. */ - InputAccessor::Stat lstat() const; + InputAccessor::Stat lstat() const + { return accessor->lstat(path); } /** * Return stats about this `SourcePath`, or std::nullopt if it * doesn't exist. */ - std::optional maybeLstat() const; + std::optional maybeLstat() const + { return accessor->maybeLstat(path); } /** * If this `SourcePath` denotes a directory (not a symlink), * return its directory entries; otherwise throw an error. */ - InputAccessor::DirEntries readDirectory() const; + InputAccessor::DirEntries readDirectory() const + { return accessor->readDirectory(path); } /** * If this `SourcePath` denotes a symlink, return its target; * otherwise throw an error. */ std::string readLink() const - { return nix::readLink(path.abs()); } + { return accessor->readLink(path); } /** * Dump this `SourcePath` to `sink` as a NAR archive. @@ -104,7 +105,7 @@ struct SourcePath void dumpPath( Sink & sink, PathFilter & filter = defaultPathFilter) const - { return nix::dumpPath(path.abs(), sink, filter); } + { return accessor->dumpPath(path, sink, filter); } /** * Copy this `SourcePath` to the Nix store. @@ -112,6 +113,7 @@ struct SourcePath StorePath fetchToStore( ref store, std::string_view name = "source", + FileIngestionMethod method = FileIngestionMethod::Recursive, PathFilter * filter = nullptr, RepairFlag repair = NoRepair) const; @@ -120,7 +122,7 @@ struct SourcePath * it has a physical location. */ std::optional getPhysicalPath() const - { return path; } + { return accessor->getPhysicalPath(path); } std::string to_string() const { return path.abs(); } @@ -129,7 +131,7 @@ struct SourcePath * Append a `CanonPath` to this path. */ SourcePath operator + (const CanonPath & x) const - { return {path + x}; } + { return {accessor, path + x}; } /** * Append a single component `c` to this path. `c` must not @@ -137,21 +139,21 @@ struct SourcePath * and `c`. */ SourcePath operator + (std::string_view c) const - { return {path + c}; } + { return {accessor, path + c}; } bool operator == (const SourcePath & x) const { - return path == x.path; + return std::tie(accessor, path) == std::tie(x.accessor, x.path); } bool operator != (const SourcePath & x) const { - return path != x.path; + return std::tie(accessor, path) != std::tie(x.accessor, x.path); } bool operator < (const SourcePath & x) const { - return path < x.path; + return std::tie(accessor, path) < std::tie(x.accessor, x.path); } /** diff --git a/src/libfetchers/memory-input-accessor.cc b/src/libfetchers/memory-input-accessor.cc new file mode 100644 index 00000000000..057f3e37f73 --- /dev/null +++ b/src/libfetchers/memory-input-accessor.cc @@ -0,0 +1,22 @@ +#include "memory-input-accessor.hh" +#include "memory-source-accessor.hh" + +namespace nix { + +struct MemoryInputAccessorImpl : MemoryInputAccessor, MemorySourceAccessor +{ + SourcePath addFile(CanonPath path, std::string && contents) override + { + return { + ref(shared_from_this()), + MemorySourceAccessor::addFile(path, std::move(contents)) + }; + } +}; + +ref makeMemoryInputAccessor() +{ + return make_ref(); +} + +} diff --git a/src/libfetchers/memory-input-accessor.hh b/src/libfetchers/memory-input-accessor.hh new file mode 100644 index 00000000000..b75b02bfd61 --- /dev/null +++ b/src/libfetchers/memory-input-accessor.hh @@ -0,0 +1,15 @@ +#include "input-accessor.hh" + +namespace nix { + +/** + * An input accessor for an in-memory file system. + */ +struct MemoryInputAccessor : InputAccessor +{ + virtual SourcePath addFile(CanonPath path, std::string && contents) = 0; +}; + +ref makeMemoryInputAccessor(); + +} diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 51fd1ed428b..9244acf3982 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -1,4 +1,6 @@ #include "fetchers.hh" +#include "processes.hh" +#include "users.hh" #include "cache.hh" #include "globals.hh" #include "tarfile.hh" @@ -69,14 +71,25 @@ struct MercurialInputScheme : InputScheme return inputFromAttrs(attrs); } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::string_view schemeName() const override { - if (maybeGetStrAttr(attrs, "type") != "hg") return {}; + return "hg"; + } - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash" && name != "name") - throw Error("unsupported Mercurial input attribute '%s'", name); + StringSet allowedAttrs() const override + { + return { + "url", + "ref", + "rev", + "revCount", + "narHash", + "name", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override + { parseURL(getStrAttr(attrs, "url")); if (auto ref = maybeGetStrAttr(attrs, "ref")) { @@ -98,13 +111,6 @@ struct MercurialInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) const override - { - // FIXME: ugly, need to distinguish between dirty and clean - // default trees. - return input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount"); - } - Input applyOverrides( const Input & input, std::optional ref, @@ -116,7 +122,7 @@ struct MercurialInputScheme : InputScheme return res; } - std::optional getSourcePath(const Input & input) override + std::optional getSourcePath(const Input & input) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); if (url.scheme == "file" && !input.getRef() && !input.getRev()) @@ -124,18 +130,27 @@ struct MercurialInputScheme : InputScheme return {}; } - void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override + void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const override { - auto sourcePath = getSourcePath(input); - assert(sourcePath); + auto [isLocal, repoPath] = getActualUrl(input); + if (!isLocal) + throw Error("cannot commit '%s' to Mercurial repository '%s' because it's not a working tree", path, input.to_string()); + + auto absPath = CanonPath(repoPath) + path; + + writeFile(absPath.abs(), contents); // FIXME: shut up if file is already tracked. runHg( - { "add", *sourcePath + "/" + std::string(file) }); + { "add", absPath.abs() }); if (commitMsg) runHg( - { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg }); + { "commit", absPath.abs(), "-m", *commitMsg }); } std::pair getActualUrl(const Input & input) const @@ -206,7 +221,7 @@ struct MercurialInputScheme : InputScheme auto checkHashType = [&](const std::optional & hash) { if (hash.has_value() && hash->type != htSHA1) - throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(Base16, true)); + throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(HashFormat::Base16, true)); }; @@ -252,7 +267,7 @@ struct MercurialInputScheme : InputScheme } } - Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false)); + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(HashFormat::Base32, false)); /* If this is a commit hash that we already have, we don't have to pull again. */ diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 01f1be97822..f9b973320b0 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -32,22 +32,29 @@ struct PathInputScheme : InputScheme return input; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::string_view schemeName() const override { - if (maybeGetStrAttr(attrs, "type") != "path") return {}; - - getStrAttr(attrs, "path"); + return "path"; + } - for (auto & [name, value] : attrs) + StringSet allowedAttrs() const override + { + return { + "path", /* Allow the user to pass in "fake" tree info - attributes. This is useful for making a pinned tree - work the same as the repository from which is exported - (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ - if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path") - // checked in Input::fromAttrs - ; - else - throw Error("unsupported path input attribute '%s'", name); + attributes. This is useful for making a pinned tree work + the same as the repository from which is exported (e.g. + path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). + */ + "rev", + "revCount", + "lastModified", + "narHash", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override + { + getStrAttr(attrs, "path"); Input input; input.attrs = attrs; @@ -66,19 +73,28 @@ struct PathInputScheme : InputScheme }; } - bool hasAllInfo(const Input & input) const override + std::optional getSourcePath(const Input & input) const override { - return true; + return getStrAttr(input.attrs, "path"); } - std::optional getSourcePath(const Input & input) override + void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const override { - return getStrAttr(input.attrs, "path"); + writeFile((CanonPath(getAbsPath(input)) + path).abs(), contents); } - void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override + CanonPath getAbsPath(const Input & input) const { - // nothing to do + auto path = getStrAttr(input.attrs, "path"); + + if (path[0] == '/') + return CanonPath(path); + + throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string()); } std::pair fetch(ref store, const Input & _input) override @@ -125,6 +141,11 @@ struct PathInputScheme : InputScheme return {std::move(*storePath), input}; } + + std::optional experimentalFeature() const override + { + return Xp::Flakes; + } }; static auto rPathInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 43c03beec17..9c7bc0cfe19 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -1,6 +1,6 @@ #include "registry.hh" -#include "fetchers.hh" -#include "util.hh" +#include "tarball.hh" +#include "users.hh" #include "globals.hh" #include "store-api.hh" #include "local-fs-store.hh" diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index e3a41e75e44..0062878a9d4 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -1,3 +1,4 @@ +#include "tarball.hh" #include "fetchers.hh" #include "cache.hh" #include "filetransfer.hh" @@ -133,7 +134,7 @@ DownloadTarballResult downloadTarball( if (cached && !cached->expired) return { - .tree = Tree { .actualPath = store->toRealPath(cached->storePath), .storePath = std::move(cached->storePath) }, + .storePath = std::move(cached->storePath), .lastModified = (time_t) getIntAttr(cached->infoAttrs, "lastModified"), .immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"), }; @@ -174,7 +175,7 @@ DownloadTarballResult downloadTarball( locked); return { - .tree = Tree { .actualPath = store->toRealPath(*unpackedStorePath), .storePath = std::move(*unpackedStorePath) }, + .storePath = std::move(*unpackedStorePath), .lastModified = lastModified, .immutableUrl = res.immutableUrl, }; @@ -183,7 +184,6 @@ DownloadTarballResult downloadTarball( // An input scheme corresponding to a curl-downloadable resource. struct CurlInputScheme : InputScheme { - virtual const std::string inputType() const = 0; const std::set transportUrlSchemes = {"file", "http", "https"}; const bool hasTarballExtension(std::string_view path) const @@ -221,22 +221,27 @@ struct CurlInputScheme : InputScheme url.query.erase("rev"); url.query.erase("revCount"); - input.attrs.insert_or_assign("type", inputType()); + input.attrs.insert_or_assign("type", std::string { schemeName() }); input.attrs.insert_or_assign("url", url.to_string()); return input; } - std::optional inputFromAttrs(const Attrs & attrs) const override + StringSet allowedAttrs() const override { - auto type = maybeGetStrAttr(attrs, "type"); - if (type != inputType()) return {}; - - // FIXME: some of these only apply to TarballInputScheme. - std::set allowedNames = {"type", "url", "narHash", "name", "unpack", "rev", "revCount", "lastModified"}; - for (auto & [name, value] : attrs) - if (!allowedNames.count(name)) - throw Error("unsupported %s input attribute '%s'", *type, name); + return { + "type", + "url", + "narHash", + "name", + "unpack", + "rev", + "revCount", + "lastModified", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override + { Input input; input.attrs = attrs; @@ -250,27 +255,21 @@ struct CurlInputScheme : InputScheme // NAR hashes are preferred over file hashes since tar/zip // files don't have a canonical representation. if (auto narHash = input.getNarHash()) - url.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); + url.query.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true)); return url; } - - bool hasAllInfo(const Input & input) const override - { - return true; - } - }; struct FileInputScheme : CurlInputScheme { - const std::string inputType() const override { return "file"; } + std::string_view schemeName() const override { return "file"; } bool isValidURL(const ParsedURL & url, bool requireTree) const override { auto parsedUrlScheme = parseUrlScheme(url.scheme); return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) && (parsedUrlScheme.application - ? parsedUrlScheme.application.value() == inputType() + ? parsedUrlScheme.application.value() == schemeName() : (!requireTree && !hasTarballExtension(url.path))); } @@ -283,7 +282,7 @@ struct FileInputScheme : CurlInputScheme struct TarballInputScheme : CurlInputScheme { - const std::string inputType() const override { return "tarball"; } + std::string_view schemeName() const override { return "tarball"; } bool isValidURL(const ParsedURL & url, bool requireTree) const override { @@ -291,7 +290,7 @@ struct TarballInputScheme : CurlInputScheme return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) && (parsedUrlScheme.application - ? parsedUrlScheme.application.value() == inputType() + ? parsedUrlScheme.application.value() == schemeName() : (requireTree || hasTarballExtension(url.path))); } @@ -313,7 +312,7 @@ struct TarballInputScheme : CurlInputScheme if (result.lastModified && !input.attrs.contains("lastModified")) input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); - return {result.tree.storePath, std::move(input)}; + return {result.storePath, std::move(input)}; } }; diff --git a/src/libfetchers/tarball.hh b/src/libfetchers/tarball.hh new file mode 100644 index 00000000000..9e6b50b31de --- /dev/null +++ b/src/libfetchers/tarball.hh @@ -0,0 +1,43 @@ +#pragma once + +#include "types.hh" +#include "path.hh" + +#include + +namespace nix { +class Store; +} + +namespace nix::fetchers { + +struct DownloadFileResult +{ + StorePath storePath; + std::string etag; + std::string effectiveUrl; + std::optional immutableUrl; +}; + +DownloadFileResult downloadFile( + ref store, + const std::string & url, + const std::string & name, + bool locked, + const Headers & headers = {}); + +struct DownloadTarballResult +{ + StorePath storePath; + time_t lastModified; + std::optional immutableUrl; +}; + +DownloadTarballResult downloadTarball( + ref store, + const std::string & url, + const std::string & name, + bool locked, + const Headers & headers = {}); + +} diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index f92920d18e5..5b49aaabcef 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,6 +1,9 @@ #include "common-args.hh" +#include "args/root.hh" #include "globals.hh" +#include "logging.hh" #include "loggers.hh" +#include "util.hh" namespace nix { @@ -34,21 +37,21 @@ MixCommonArgs::MixCommonArgs(const std::string & programName) .description = "Set the Nix configuration setting *name* to *value* (overriding `nix.conf`).", .category = miscCategory, .labels = {"name", "value"}, - .handler = {[](std::string name, std::string value) { + .handler = {[this](std::string name, std::string value) { try { globalConfig.set(name, value); } catch (UsageError & e) { - if (!completions) + if (!getRoot().completions) warn(e.what()); } }}, - .completer = [](size_t index, std::string_view prefix) { + .completer = [](AddCompletions & completions, size_t index, std::string_view prefix) { if (index == 0) { std::map settings; globalConfig.getSettings(settings); for (auto & s : settings) if (hasPrefix(s.first, prefix)) - completions->add(s.first, fmt("Set the `%s` setting.", s.first)); + completions.add(s.first, fmt("Set the `%s` setting.", s.first)); } } }); diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index cda5cb939ac..9829859de32 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -1,6 +1,6 @@ #include "loggers.hh" +#include "environment-variables.hh" #include "progress-bar.hh" -#include "util.hh" namespace nix { diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 45b1fdfd115..a7aee47c3ac 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -1,5 +1,5 @@ #include "progress-bar.hh" -#include "util.hh" +#include "terminal.hh" #include "sync.hh" #include "store-api.hh" #include "names.hh" diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 56f47a4ac81..862ef355b95 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -1,10 +1,11 @@ #include "globals.hh" +#include "current-process.hh" #include "shared.hh" #include "store-api.hh" #include "gc-store.hh" -#include "util.hh" #include "loggers.hh" #include "progress-bar.hh" +#include "signals.hh" #include #include @@ -379,9 +380,9 @@ RunPager::RunPager() }); pid.setKillSignal(SIGINT); - stdout = fcntl(STDOUT_FILENO, F_DUPFD_CLOEXEC, 0); + std_out = fcntl(STDOUT_FILENO, F_DUPFD_CLOEXEC, 0); if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); + throw SysError("dupping standard output"); } @@ -390,7 +391,7 @@ RunPager::~RunPager() try { if (pid != -1) { std::cout.flush(); - dup2(stdout, STDOUT_FILENO); + dup2(std_out, STDOUT_FILENO); pid.wait(); } } catch (...) { diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 7a9e83c6c29..c68f6cd83e9 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -1,8 +1,9 @@ #pragma once ///@file -#include "util.hh" +#include "processes.hh" #include "args.hh" +#include "args/root.hh" #include "common-args.hh" #include "path.hh" #include "derived-path.hh" @@ -66,7 +67,7 @@ template N getIntArg(const std::string & opt, } -struct LegacyArgs : public MixCommonArgs +struct LegacyArgs : public MixCommonArgs, public RootArgs { std::function parseArg; @@ -85,8 +86,9 @@ struct LegacyArgs : public MixCommonArgs void showManPage(const std::string & name); /** - * The constructor of this class starts a pager if stdout is a - * terminal and $PAGER is set. Stdout is redirected to the pager. + * The constructor of this class starts a pager if standard output is a + * terminal and $PAGER is set. Standard output is redirected to the + * pager. */ class RunPager { @@ -96,7 +98,7 @@ public: private: Pid pid; - int stdout; + int std_out; }; extern volatile ::sig_atomic_t blockInt; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index b4fea693f5b..ae483c95efc 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -2,7 +2,7 @@ #include "binary-cache-store.hh" #include "compression.hh" #include "derivations.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "globals.hh" #include "nar-info.hh" #include "sync.hh" @@ -11,6 +11,7 @@ #include "nar-accessor.hh" #include "thread-pool.hh" #include "callback.hh" +#include "signals.hh" #include #include @@ -143,7 +144,7 @@ ref BinaryCacheStore::addToStoreCommon( write the compressed NAR to disk), into a HashSink (to get the NAR hash), and into a NarAccessor (to get the NAR listing). */ HashSink fileHashSink { htSHA256 }; - std::shared_ptr narAccessor; + std::shared_ptr narAccessor; HashSink narHashSink { htSHA256 }; { FdSink fileSink(fdTemp.get()); @@ -164,7 +165,7 @@ ref BinaryCacheStore::addToStoreCommon( auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; - narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" + narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Base32, false) + ".nar" + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "zstd" ? ".zst" : @@ -195,7 +196,7 @@ ref BinaryCacheStore::addToStoreCommon( if (writeNARListing) { nlohmann::json j = { {"version", 1}, - {"root", listNar(ref(narAccessor), "", true)}, + {"root", listNar(ref(narAccessor), CanonPath::root, true)}, }; upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json"); @@ -206,9 +207,9 @@ ref BinaryCacheStore::addToStoreCommon( specify the NAR file and member containing the debug info. */ if (writeDebugInfo) { - std::string buildIdDir = "/lib/debug/.build-id"; + CanonPath buildIdDir("lib/debug/.build-id"); - if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) { + if (auto st = narAccessor->maybeLstat(buildIdDir); st && st->type == SourceAccessor::tDirectory) { ThreadPool threadPool(25); @@ -231,17 +232,17 @@ ref BinaryCacheStore::addToStoreCommon( std::regex regex1("^[0-9a-f]{2}$"); std::regex regex2("^[0-9a-f]{38}\\.debug$"); - for (auto & s1 : narAccessor->readDirectory(buildIdDir)) { - auto dir = buildIdDir + "/" + s1; + for (auto & [s1, _type] : narAccessor->readDirectory(buildIdDir)) { + auto dir = buildIdDir + s1; - if (narAccessor->stat(dir).type != FSAccessor::tDirectory + if (narAccessor->lstat(dir).type != SourceAccessor::tDirectory || !std::regex_match(s1, regex1)) continue; - for (auto & s2 : narAccessor->readDirectory(dir)) { - auto debugPath = dir + "/" + s2; + for (auto & [s2, _type] : narAccessor->readDirectory(dir)) { + auto debugPath = dir + s2; - if (narAccessor->stat(debugPath).type != FSAccessor::tRegular + if (narAccessor->lstat(debugPath).type != SourceAccessor::tRegular || !std::regex_match(s2, regex2)) continue; @@ -250,7 +251,7 @@ ref BinaryCacheStore::addToStoreCommon( std::string key = "debuginfo/" + buildId; std::string target = "../" + narInfo->url; - threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target)); + threadPool.enqueue(std::bind(doFile, std::string(debugPath.rel()), key, target)); } } @@ -503,9 +504,9 @@ void BinaryCacheStore::registerDrvOutput(const Realisation& info) { upsertFile(filePath, info.toJSON().dump(), "application/json"); } -ref BinaryCacheStore::getFSAccessor() +ref BinaryCacheStore::getFSAccessor(bool requireValidPath) { - return make_ref(ref(shared_from_this()), localNarCache); + return make_ref(ref(shared_from_this()), requireValidPath, localNarCache); } void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 49f271d248c..cea2a571f13 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -17,28 +17,28 @@ struct BinaryCacheStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting compression{(StoreConfig*) this, "xz", "compression", + const Setting compression{this, "xz", "compression", "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."}; - const Setting writeNARListing{(StoreConfig*) this, false, "write-nar-listing", + const Setting writeNARListing{this, false, "write-nar-listing", "Whether to write a JSON file that lists the files in each NAR."}; - const Setting writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", + const Setting writeDebugInfo{this, false, "index-debug-info", R"( Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to fetch debug info on demand )"}; - const Setting secretKeyFile{(StoreConfig*) this, "", "secret-key", + const Setting secretKeyFile{this, "", "secret-key", "Path to the secret key used to sign the binary cache."}; - const Setting localNarCache{(StoreConfig*) this, "", "local-nar-cache", + const Setting localNarCache{this, "", "local-nar-cache", "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."}; - const Setting parallelCompression{(StoreConfig*) this, false, "parallel-compression", + const Setting parallelCompression{this, false, "parallel-compression", "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; - const Setting compressionLevel{(StoreConfig*) this, -1, "compression-level", + const Setting compressionLevel{this, -1, "compression-level", R"( The *preset level* to be used when compressing NARs. The meaning and accepted values depend on the compression method selected. @@ -148,7 +148,7 @@ public: void narFromPath(const StorePath & path, Sink & sink) override; - ref getFSAccessor() override; + ref getFSAccessor(bool requireValidPath) override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override; diff --git a/src/libstore/build-result.cc b/src/libstore/build-result.cc new file mode 100644 index 00000000000..18f519c5c61 --- /dev/null +++ b/src/libstore/build-result.cc @@ -0,0 +1,18 @@ +#include "build-result.hh" + +namespace nix { + +GENERATE_CMP_EXT( + , + BuildResult, + me->status, + me->errorMsg, + me->timesBuilt, + me->isNonDeterministic, + me->builtOutputs, + me->startTime, + me->stopTime, + me->cpuUser, + me->cpuSystem); + +} diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index b7a56e7917d..8840fa7e340 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -3,6 +3,7 @@ #include "realisation.hh" #include "derived-path.hh" +#include "comparator.hh" #include #include @@ -100,6 +101,8 @@ struct BuildResult */ std::optional cpuUser, cpuSystem; + DECLARE_CMP(BuildResult); + bool success() { return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid; diff --git a/src/libstore/build/child.cc b/src/libstore/build/child.cc new file mode 100644 index 00000000000..aa31c3caf24 --- /dev/null +++ b/src/libstore/build/child.cc @@ -0,0 +1,37 @@ +#include "child.hh" +#include "current-process.hh" +#include "logging.hh" + +#include +#include + +namespace nix { + +void commonChildInit() +{ + logger = makeSimpleLogger(); + + const static std::string pathNullDevice = "/dev/null"; + restoreProcessContext(false); + + /* Put the child in a separate session (and thus a separate + process group) so that it has no controlling terminal (meaning + that e.g. ssh cannot open /dev/tty) and it doesn't receive + terminal signals. */ + if (setsid() == -1) + throw SysError("creating a new session"); + + /* Dup stderr to stdout. */ + if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) + throw SysError("cannot dup stderr into stdout"); + + /* Reroute stdin to /dev/null. */ + int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); + if (fdDevNull == -1) + throw SysError("cannot open '%1%'", pathNullDevice); + if (dup2(fdDevNull, STDIN_FILENO) == -1) + throw SysError("cannot dup null device into stdin"); + close(fdDevNull); +} + +} diff --git a/src/libstore/build/child.hh b/src/libstore/build/child.hh new file mode 100644 index 00000000000..3dfc552b93d --- /dev/null +++ b/src/libstore/build/child.hh @@ -0,0 +1,11 @@ +#pragma once +///@file + +namespace nix { + +/** + * Common initialisation performed in child processes. + */ +void commonChildInit(); + +} diff --git a/src/libstore/build/create-derivation-and-realise-goal.cc b/src/libstore/build/create-derivation-and-realise-goal.cc deleted file mode 100644 index 60f67956d6f..00000000000 --- a/src/libstore/build/create-derivation-and-realise-goal.cc +++ /dev/null @@ -1,157 +0,0 @@ -#include "create-derivation-and-realise-goal.hh" -#include "worker.hh" - -namespace nix { - -CreateDerivationAndRealiseGoal::CreateDerivationAndRealiseGoal(ref drvReq, - const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker, DerivedPath::Built { .drvPath = drvReq, .outputs = wantedOutputs }) - , drvReq(drvReq) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - state = &CreateDerivationAndRealiseGoal::getDerivation; - name = fmt( - "outer obtaining drv from '%s' and then building outputs %s", - drvReq->to_string(worker.store), - std::visit(overloaded { - [&](const OutputsSpec::All) -> std::string { - return "* (all of them)"; - }, - [&](const OutputsSpec::Names os) { - return concatStringsSep(", ", quoteStrings(os)); - }, - }, wantedOutputs.raw)); - trace("created outer"); - - worker.updateProgress(); -} - - -CreateDerivationAndRealiseGoal::~CreateDerivationAndRealiseGoal() -{ -} - - -static StorePath pathPartOfReq(const SingleDerivedPath & req) -{ - return std::visit(overloaded { - [&](const SingleDerivedPath::Opaque & bo) { - return bo.path; - }, - [&](const SingleDerivedPath::Built & bfd) { - return pathPartOfReq(*bfd.drvPath); - }, - }, req.raw()); -} - - -std::string CreateDerivationAndRealiseGoal::key() -{ - /* Ensure that derivations get built in order of their name, - i.e. a derivation named "aardvark" always comes before "baboon". And - substitution goals and inner derivation goals always happen before - derivation goals (due to "b$"). */ - return "c$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + drvReq->to_string(worker.store); -} - - -void CreateDerivationAndRealiseGoal::timedOut(Error && ex) -{ -} - - -void CreateDerivationAndRealiseGoal::work() -{ - (this->*state)(); -} - - -void CreateDerivationAndRealiseGoal::addWantedOutputs(const OutputsSpec & outputs) -{ - /* If we already want all outputs, there is nothing to do. */ - auto newWanted = wantedOutputs.union_(outputs); - bool needRestart = !newWanted.isSubsetOf(wantedOutputs); - wantedOutputs = newWanted; - - if (!needRestart) return; - - if (!optDrvPath) - // haven't started steps where the outputs matter yet - return; - worker.makeDerivationGoal(*optDrvPath, outputs, buildMode); -} - - -void CreateDerivationAndRealiseGoal::getDerivation() -{ - trace("outer init"); - - /* The first thing to do is to make sure that the derivation - exists. If it doesn't, it may be created through a - substitute. */ - if (auto optDrvPath = [this]() -> std::optional { - if (buildMode != bmNormal) return std::nullopt; - - auto drvPath = StorePath::dummy; - try { - drvPath = resolveDerivedPath(worker.store, *drvReq); - } catch (MissingRealisation &) { - return std::nullopt; - } - return worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath) - ? std::optional { drvPath } - : std::nullopt; - }()) { - trace(fmt("already have drv '%s' for '%s', can go straight to building", - worker.store.printStorePath(*optDrvPath), - drvReq->to_string(worker.store))); - - loadAndBuildDerivation(); - } else { - trace("need to obtain drv we want to build"); - - addWaitee(worker.makeGoal(DerivedPath::fromSingle(*drvReq))); - - state = &CreateDerivationAndRealiseGoal::loadAndBuildDerivation; - if (waitees.empty()) work(); - } -} - - -void CreateDerivationAndRealiseGoal::loadAndBuildDerivation() -{ - trace("outer load and build derivation"); - - if (nrFailed != 0) { - amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store))); - return; - } - - StorePath drvPath = resolveDerivedPath(worker.store, *drvReq); - /* Build this step! */ - concreteDrvGoal = worker.makeDerivationGoal(drvPath, wantedOutputs, buildMode); - addWaitee(upcast_goal(concreteDrvGoal)); - state = &CreateDerivationAndRealiseGoal::buildDone; - optDrvPath = std::move(drvPath); - if (waitees.empty()) work(); -} - - -void CreateDerivationAndRealiseGoal::buildDone() -{ - trace("outer build done"); - - buildResult = upcast_goal(concreteDrvGoal)->getBuildResult(DerivedPath::Built { - .drvPath = drvReq, - .outputs = wantedOutputs, - }); - - if (buildResult.success()) - amDone(ecSuccess); - else - amDone(ecFailed, Error("building '%s' failed", drvReq->to_string(worker.store))); -} - - -} diff --git a/src/libstore/build/create-derivation-and-realise-goal.hh b/src/libstore/build/create-derivation-and-realise-goal.hh deleted file mode 100644 index ca936fc95de..00000000000 --- a/src/libstore/build/create-derivation-and-realise-goal.hh +++ /dev/null @@ -1,96 +0,0 @@ -#pragma once - -#include "parsed-derivations.hh" -#include "lock.hh" -#include "store-api.hh" -#include "pathlocks.hh" -#include "goal.hh" - -namespace nix { - -struct DerivationGoal; - -/** - * This goal type is essentially the serial composition (like function - * composition) of a goal for getting a derivation, and then a - * `DerivationGoal` using the newly-obtained derivation. - * - * In the (currently experimental) general inductive case of derivations - * that are themselves build outputs, that first goal will be *another* - * `CreateDerivationAndRealiseGoal`. In the (much more common) base-case - * where the derivation has no provence and is just referred to by - * (content-addressed) store path, that first goal is a - * `SubstitutionGoal`. - * - * If we already have the derivation (e.g. if the evalutator has created - * the derivation locally and then instructured the store to build it), - * we can skip the first goal entirely as a small optimization. - */ -struct CreateDerivationAndRealiseGoal : public Goal -{ - /** - * How to obtain a store path of the derivation to build. - */ - ref drvReq; - - /** - * The path of the derivation, once obtained. - **/ - std::optional optDrvPath; - - /** - * The goal for the corresponding concrete derivation. - **/ - std::shared_ptr concreteDrvGoal; - - /** - * The specific outputs that we need to build. - */ - OutputsSpec wantedOutputs; - - typedef void (CreateDerivationAndRealiseGoal::*GoalState)(); - GoalState state; - - /** - * The final output paths of the build. - * - * - For input-addressed derivations, always the precomputed paths - * - * - For content-addressed derivations, calcuated from whatever the - * hash ends up being. (Note that fixed outputs derivations that - * produce the "wrong" output still install that data under its - * true content-address.) - */ - OutputPathMap finalOutputs; - - BuildMode buildMode; - - CreateDerivationAndRealiseGoal(ref drvReq, - const OutputsSpec & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - virtual ~CreateDerivationAndRealiseGoal(); - - void timedOut(Error && ex) override; - - std::string key() override; - - void work() override; - - /** - * Add wanted outputs to an already existing derivation goal. - */ - void addWantedOutputs(const OutputsSpec & outputs); - - /** - * The states. - */ - void getDerivation(); - void loadAndBuildDerivation(); - void buildDone(); - - JobCategory jobCategory() const override { - return JobCategory::Administration; - }; -}; - -} diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 6472ecd99f4..81eef7c4748 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -8,8 +8,8 @@ #include "util.hh" #include "archive.hh" #include "compression.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include "topo-sort.hh" #include "callback.hh" #include "local-store.hh" // TODO remove, along with remaining downcasts @@ -71,7 +71,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, , wantedOutputs(wantedOutputs) , buildMode(buildMode) { - state = &DerivationGoal::loadDerivation; + state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); @@ -164,6 +164,24 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) } +void DerivationGoal::getDerivation() +{ + trace("init"); + + /* The first thing to do is to make sure that the derivation + exists. If it doesn't, it may be created through a + substitute. */ + if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) { + loadDerivation(); + return; + } + + addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath))); + + state = &DerivationGoal::loadDerivation; +} + + void DerivationGoal::loadDerivation() { trace("loading derivation"); @@ -543,7 +561,7 @@ void DerivationGoal::inputsRealised() attempt = fullDrv.tryResolve(worker.store); } assert(attempt); - Derivation drvResolved { *std::move(attempt) }; + Derivation drvResolved { std::move(*attempt) }; auto pathResolved = writeDerivation(worker.store, drvResolved); @@ -1167,11 +1185,11 @@ HookReply DerivationGoal::tryBuildHook() throw; } - WorkerProto::WriteConn conn { hook->sink }; + CommonProto::WriteConn conn { hook->sink }; /* Tell the hook all the inputs that have to be copied to the remote system. */ - WorkerProto::write(worker.store, conn, inputPaths); + CommonProto::write(worker.store, conn, inputPaths); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ @@ -1182,7 +1200,7 @@ HookReply DerivationGoal::tryBuildHook() if (buildMode != bmCheck && status.known && status.known->isValid()) continue; missingOutputs.insert(outputName); } - WorkerProto::write(worker.store, conn, missingOutputs); + CommonProto::write(worker.store, conn, missingOutputs); } hook->sink = FdSink(); @@ -1299,9 +1317,26 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data) auto s = handleJSONLogMessage(*json, worker.act, hook->activities, true); // ensure that logs from a builder using `ssh-ng://` as protocol // are also available to `nix log`. - if (s && !isWrittenToLog && logSink && (*json)["type"] == resBuildLogLine) { - auto f = (*json)["fields"]; - (*logSink)((f.size() > 0 ? f.at(0).get() : "") + "\n"); + if (s && !isWrittenToLog && logSink) { + const auto type = (*json)["type"]; + const auto fields = (*json)["fields"]; + if (type == resBuildLogLine) { + (*logSink)((fields.size() > 0 ? fields[0].get() : "") + "\n"); + } else if (type == resSetPhase && ! fields.is_null()) { + const auto phase = fields[0]; + if (! phase.is_null()) { + // nixpkgs' stdenv produces lines in the log to signal + // phase changes. + // We want to get the same lines in case of remote builds. + // The format is: + // @nix { "action": "setPhase", "phase": "$curPhase" } + const auto logLine = nlohmann::json::object({ + {"action", "setPhase"}, + {"phase", phase} + }); + (*logSink)("@nix " + logLine.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace) + "\n"); + } + } } } currentHookLine.clear(); @@ -1456,6 +1491,7 @@ void DerivationGoal::done( SingleDrvOutputs builtOutputs, std::optional ex) { + outputLocks.unlock(); buildResult.status = status; if (ex) buildResult.errorMsg = fmt("%s", normaltxt(ex->info().msg)); @@ -1498,24 +1534,23 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) if (!useDerivation) return; auto & fullDrv = *dynamic_cast(drv.get()); - std::optional info = tryGetConcreteDrvGoal(waitee); - if (!info) return; - const auto & [dg, drvReq] = *info; + auto * dg = dynamic_cast(&*waitee); + if (!dg) return; - auto * nodeP = fullDrv.inputDrvs.findSlot(drvReq.get()); + auto * nodeP = fullDrv.inputDrvs.findSlot(DerivedPath::Opaque { .path = dg->drvPath }); if (!nodeP) return; auto & outputs = nodeP->value; for (auto & outputName : outputs) { - auto buildResult = dg.get().getBuildResult(DerivedPath::Built { - .drvPath = makeConstantStorePathRef(dg.get().drvPath), + auto buildResult = dg->getBuildResult(DerivedPath::Built { + .drvPath = makeConstantStorePathRef(dg->drvPath), .outputs = OutputsSpec::Names { outputName }, }); if (buildResult.success()) { auto i = buildResult.builtOutputs.find(outputName); if (i != buildResult.builtOutputs.end()) inputDrvOutputs.insert_or_assign( - { dg.get().drvPath, outputName }, + { dg->drvPath, outputName }, i->second.outPath); } } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 62b122c27e6..ddb5ee1e34a 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -52,10 +52,6 @@ struct InitialOutput { /** * A goal for building some or all of the outputs of a derivation. - * - * The derivation must already be present, either in the store in a drv - * or in memory. If the derivation itself needs to be gotten first, a - * `CreateDerivationAndRealiseGoal` goal must be used instead. */ struct DerivationGoal : public Goal { @@ -235,6 +231,7 @@ struct DerivationGoal : public Goal /** * The states. */ + void getDerivation(); void loadDerivation(); void haveDerivation(); void outputsSubstitutionTried(); diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index f0f0e551935..13ff22f4589 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -1,6 +1,5 @@ #include "worker.hh" #include "substitution-goal.hh" -#include "create-derivation-and-realise-goal.hh" #include "derivation-goal.hh" #include "local-store.hh" @@ -16,7 +15,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod worker.run(goals); - StringSet failed; + StorePathSet failed; std::optional ex; for (auto & i : goals) { if (i->ex) { @@ -26,10 +25,10 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod ex = std::move(i->ex); } if (i->exitCode != Goal::ecSuccess) { - if (auto i2 = dynamic_cast(i.get())) - failed.insert(i2->drvReq->to_string(*this)); + if (auto i2 = dynamic_cast(i.get())) + failed.insert(i2->drvPath); else if (auto i2 = dynamic_cast(i.get())) - failed.insert(printStorePath(i2->storePath)); + failed.insert(i2->storePath); } } @@ -38,7 +37,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod throw std::move(*ex); } else if (!failed.empty()) { if (ex) logError(ex->info()); - throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed))); + throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed)); } } diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 01d3c3491a5..9af083230ee 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -49,16 +49,6 @@ enum struct JobCategory { * A substitution an arbitrary store object; it will use network resources. */ Substitution, - /** - * A goal that does no "real" work by itself, and just exists to depend on - * other goals which *do* do real work. These goals therefore are not - * limited. - * - * These goals cannot infinitely create themselves, so there is no risk of - * a "fork bomb" type situation (which would be a problem even though the - * goal do no real work) either. - */ - Administration, }; struct Goal : public std::enable_shared_from_this diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index 337c60bd460..5d045ec3d07 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -1,5 +1,7 @@ #include "globals.hh" #include "hook-instance.hh" +#include "file-system.hh" +#include "child.hh" namespace nix { diff --git a/src/libstore/build/hook-instance.hh b/src/libstore/build/hook-instance.hh index d84f6287775..61cf534f4e9 100644 --- a/src/libstore/build/hook-instance.hh +++ b/src/libstore/build/hook-instance.hh @@ -3,6 +3,7 @@ #include "logging.hh" #include "serialise.hh" +#include "processes.hh" namespace nix { diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 64b55ca6ac2..a9f9307737a 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -15,7 +15,10 @@ #include "json-utils.hh" #include "cgroup.hh" #include "personality.hh" +#include "current-process.hh" #include "namespaces.hh" +#include "child.hh" +#include "unix-domain-socket.hh" #include #include @@ -227,7 +230,7 @@ void LocalDerivationGoal::tryLocalBuild() if (!buildUser) { if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); + fmt("waiting for a free build user ID for '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); return; } @@ -386,27 +389,27 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() cleanupPostOutputsRegisteredModeCheck(); } - #if __linux__ -static void linkOrCopy(const Path & from, const Path & to) -{ - if (link(from.c_str(), to.c_str()) == -1) { - /* Hard-linking fails if we exceed the maximum link count on a - file (e.g. 32000 of ext3), which is quite possible after a - 'nix-store --optimise'. FIXME: actually, why don't we just - bind-mount in this case? - - It can also fail with EPERM in BeegFS v7 and earlier versions - or fail with EXDEV in OpenAFS - which don't allow hard-links to other directories */ - if (errno != EMLINK && errno != EPERM && errno != EXDEV) - throw SysError("linking '%s' to '%s'", to, from); - copyPath(from, to); +static void doBind(const Path & source, const Path & target, bool optional = false) { + debug("bind mounting '%1%' to '%2%'", source, target); + struct stat st; + if (stat(source.c_str(), &st) == -1) { + if (optional && errno == ENOENT) + return; + else + throw SysError("getting attributes of path '%1%'", source); } -} + if (S_ISDIR(st.st_mode)) + createDirs(target); + else { + createDirs(dirOf(target)); + writeFile(target, ""); + } + if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) + throw SysError("bind mount from '%1%' to '%2%' failed", source, target); +}; #endif - void LocalDerivationGoal::startBuilder() { if ((buildUser && buildUser->getUIDCount() != 1) @@ -581,7 +584,7 @@ void LocalDerivationGoal::startBuilder() /* Allow a user-configurable set of directories from the host file system. */ - dirsInChroot.clear(); + pathsInChroot.clear(); for (auto i : settings.sandboxPaths.get()) { if (i.empty()) continue; @@ -592,19 +595,19 @@ void LocalDerivationGoal::startBuilder() } size_t p = i.find('='); if (p == std::string::npos) - dirsInChroot[i] = {i, optional}; + pathsInChroot[i] = {i, optional}; else - dirsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; + pathsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; } if (hasPrefix(worker.store.storeDir, tmpDirInSandbox)) { throw Error("`sandbox-build-dir` must not contain the storeDir"); } - dirsInChroot[tmpDirInSandbox] = tmpDir; + pathsInChroot[tmpDirInSandbox] = tmpDir; /* Add the closure of store paths to the chroot. */ StorePathSet closure; - for (auto & i : dirsInChroot) + for (auto & i : pathsInChroot) try { if (worker.store.isInStore(i.second.source)) worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); @@ -615,7 +618,7 @@ void LocalDerivationGoal::startBuilder() } for (auto & i : closure) { auto p = worker.store.printStorePath(i); - dirsInChroot.insert_or_assign(p, p); + pathsInChroot.insert_or_assign(p, p); } PathSet allowedPaths = settings.allowedImpureHostPrefixes; @@ -643,14 +646,14 @@ void LocalDerivationGoal::startBuilder() /* Allow files in __impureHostDeps to be missing; e.g. macOS 11+ has no /usr/lib/libSystem*.dylib */ - dirsInChroot[i] = {i, true}; + pathsInChroot[i] = {i, true}; } #if __linux__ /* Create a temporary directory in which we set up the chroot environment using bind-mounts. We put it in the Nix store - to ensure that we can create hard-links to non-directory - inputs in the fake Nix store in the chroot (see below). */ + so that the build outputs can be moved efficiently from the + chroot to their final location. */ chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; deletePath(chrootRootDir); @@ -711,15 +714,12 @@ void LocalDerivationGoal::startBuilder() for (auto & i : inputPaths) { auto p = worker.store.printStorePath(i); Path r = worker.store.toRealPath(p); - if (S_ISDIR(lstat(r).st_mode)) - dirsInChroot.insert_or_assign(p, r); - else - linkOrCopy(r, chrootRootDir + p); + pathsInChroot.insert_or_assign(p, r); } /* If we're repairing, checking or rebuilding part of a multiple-outputs derivation, it's possible that we're - rebuilding a path that is in settings.dirsInChroot + rebuilding a path that is in settings.sandbox-paths (typically the dependencies of /bin/sh). Throw them out. */ for (auto & i : drv->outputsAndOptPaths(worker.store)) { @@ -729,7 +729,7 @@ void LocalDerivationGoal::startBuilder() is already in the sandbox, so we don't need to worry about removing it. */ if (i.second.second) - dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); + pathsInChroot.erase(worker.store.printStorePath(*i.second.second)); } if (cgroup) { @@ -787,9 +787,9 @@ void LocalDerivationGoal::startBuilder() } else { auto p = line.find('='); if (p == std::string::npos) - dirsInChroot[line] = line; + pathsInChroot[line] = line; else - dirsInChroot[line.substr(0, p)] = line.substr(p + 1); + pathsInChroot[line.substr(0, p)] = line.substr(p + 1); } } } @@ -1066,7 +1066,7 @@ void LocalDerivationGoal::initTmpDir() { env[i.first] = i.second; } else { auto hash = hashString(htSHA256, i.first); - std::string fn = ".attr-" + hash.to_string(Base32, false); + std::string fn = ".attr-" + hash.to_string(HashFormat::Base32, false); Path p = tmpDir + "/" + fn; writeFile(p, rewriteStrings(i.second, inputRewrites)); chownToBuilder(p); @@ -1135,8 +1135,18 @@ void LocalDerivationGoal::initEnv() fixed-output derivations is by definition pure (since we already know the cryptographic hash of the output). */ if (!derivationType->isSandboxed()) { - for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) - env[i] = getEnv(i).value_or(""); + auto & impureEnv = settings.impureEnv.get(); + if (!impureEnv.empty()) + experimentalFeatureSettings.require(Xp::ConfigurableImpureEnv); + + for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) { + auto envVar = impureEnv.find(i); + if (envVar != impureEnv.end()) { + env[i] = envVar->second; + } else { + env[i] = getEnv(i).value_or(""); + } + } } /* Currently structured log messages piggyback on stderr, but we @@ -1555,41 +1565,33 @@ void LocalDerivationGoal::addDependency(const StorePath & path) Path source = worker.store.Store::toRealPath(path); Path target = chrootRootDir + worker.store.printStorePath(path); - debug("bind-mounting %s -> %s", target, source); - if (pathExists(target)) + if (pathExists(target)) { + // There is a similar debug message in doBind, so only run it in this block to not have double messages. + debug("bind-mounting %s -> %s", target, source); throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); + } - auto st = lstat(source); - - if (S_ISDIR(st.st_mode)) { - - /* Bind-mount the path into the sandbox. This requires - entering its mount namespace, which is not possible - in multithreaded programs. So we do this in a - child process.*/ - Pid child(startProcess([&]() { - - if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1)) - throw SysError("entering sandbox user namespace"); - - if (setns(sandboxMountNamespace.get(), 0) == -1) - throw SysError("entering sandbox mount namespace"); + /* Bind-mount the path into the sandbox. This requires + entering its mount namespace, which is not possible + in multithreaded programs. So we do this in a + child process.*/ + Pid child(startProcess([&]() { - createDirs(target); + if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1)) + throw SysError("entering sandbox user namespace"); - if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) - throw SysError("bind mount from '%s' to '%s' failed", source, target); + if (setns(sandboxMountNamespace.get(), 0) == -1) + throw SysError("entering sandbox mount namespace"); - _exit(0); - })); + doBind(source, target); - int status = child.wait(); - if (status != 0) - throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); + _exit(0); + })); - } else - linkOrCopy(source, target); + int status = child.wait(); + if (status != 0) + throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); #else throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", @@ -1621,6 +1623,8 @@ void setupSeccomp() seccomp_release(ctx); }); + constexpr std::string_view nativeSystem = SYSTEM; + if (nativeSystem == "x86_64-linux" && seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) throw SysError("unable to add 32-bit seccomp architecture"); @@ -1779,7 +1783,7 @@ void LocalDerivationGoal::runChild() /* Set up a nearly empty /dev, unless the user asked to bind-mount the host /dev. */ Strings ss; - if (dirsInChroot.find("/dev") == dirsInChroot.end()) { + if (pathsInChroot.find("/dev") == pathsInChroot.end()) { createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/pts"); ss.push_back("/dev/full"); @@ -1814,34 +1818,15 @@ void LocalDerivationGoal::runChild() ss.push_back(path); if (settings.caFile != "") - dirsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true); + pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true); } - for (auto & i : ss) dirsInChroot.emplace(i, i); + for (auto & i : ss) pathsInChroot.emplace(i, i); /* Bind-mount all the directories from the "host" filesystem that we want in the chroot environment. */ - auto doBind = [&](const Path & source, const Path & target, bool optional = false) { - debug("bind mounting '%1%' to '%2%'", source, target); - struct stat st; - if (stat(source.c_str(), &st) == -1) { - if (optional && errno == ENOENT) - return; - else - throw SysError("getting attributes of path '%1%'", source); - } - if (S_ISDIR(st.st_mode)) - createDirs(target); - else { - createDirs(dirOf(target)); - writeFile(target, ""); - } - if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("bind mount from '%1%' to '%2%' failed", source, target); - }; - - for (auto & i : dirsInChroot) { + for (auto & i : pathsInChroot) { if (i.second.source == "/proc") continue; // backwards compatibility #if HAVE_EMBEDDED_SANDBOX_SHELL @@ -1882,7 +1867,7 @@ void LocalDerivationGoal::runChild() if /dev/ptx/ptmx exists). */ if (pathExists("/dev/pts/ptmx") && !pathExists(chrootRootDir + "/dev/ptmx") - && !dirsInChroot.count("/dev/pts")) + && !pathsInChroot.count("/dev/pts")) { if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) { @@ -2017,7 +2002,7 @@ void LocalDerivationGoal::runChild() /* We build the ancestry before adding all inputPaths to the store because we know they'll all have the same parents (the store), and there might be lots of inputs. This isn't particularly efficient... I doubt it'll be a bottleneck in practice */ - for (auto & i : dirsInChroot) { + for (auto & i : pathsInChroot) { Path cur = i.first; while (cur.compare("/") != 0) { cur = dirOf(cur); @@ -2025,7 +2010,7 @@ void LocalDerivationGoal::runChild() } } - /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost + /* And we want the store in there regardless of how empty pathsInChroot. We include the innermost path component this time, since it's typically /nix/store and we care about that. */ Path cur = worker.store.storeDir; while (cur.compare("/") != 0) { @@ -2036,7 +2021,7 @@ void LocalDerivationGoal::runChild() /* Add all our input paths to the chroot */ for (auto & i : inputPaths) { auto p = worker.store.printStorePath(i); - dirsInChroot[p] = p; + pathsInChroot[p] = p; } /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ @@ -2067,7 +2052,7 @@ void LocalDerivationGoal::runChild() without file-write* allowed, access() incorrectly returns EPERM */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : dirsInChroot) { + for (auto & i : pathsInChroot) { if (i.first != i.second.source) throw Error( "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", @@ -2511,7 +2496,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() ValidPathInfo newInfo0 { worker.store, outputPathName(drv->name, outputName), - *std::move(optCA), + std::move(*optCA), Hash::dummy, }; if (*scratchPath != newInfo0.path) { @@ -2573,8 +2558,8 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", worker.store.printStorePath(drvPath), - wanted.to_string(SRI, true), - got.to_string(SRI, true))); + wanted.to_string(HashFormat::SRI, true), + got.to_string(HashFormat::SRI, true))); } if (!newInfo0.references.empty()) delayedException = std::make_exception_ptr( diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 0a05081c78a..88152a645cf 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -3,6 +3,7 @@ #include "derivation-goal.hh" #include "local-store.hh" +#include "processes.hh" namespace nix { @@ -86,8 +87,8 @@ struct LocalDerivationGoal : public DerivationGoal : source(source), optional(optional) { } }; - typedef map DirsInChroot; // maps target path to source path - DirsInChroot dirsInChroot; + typedef map PathsInChroot; // maps target path to source path + PathsInChroot pathsInChroot; typedef map Environment; Environment env; @@ -120,14 +121,6 @@ struct LocalDerivationGoal : public DerivationGoal */ OutputPathMap scratchOutputs; - /** - * Path registration info from the previous round, if we're - * building multiple times. Since this contains the hash, it - * allows us to compare whether two rounds produced the same - * result. - */ - std::map prevInfos; - uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } gid_t sandboxGid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); } diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 93867007d3a..c7e8e282508 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -2,6 +2,7 @@ #include "substitution-goal.hh" #include "nar-info.hh" #include "finally.hh" +#include "signals.hh" namespace nix { @@ -217,6 +218,8 @@ void PathSubstitutionGoal::tryToRun() thr = std::thread([this]() { try { + ReceiveInterrupts receiveInterrupts; + /* Wake up the worker loop when we're done. */ Finally updateStats([this]() { outPipe.writeSide.close(); }); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index b4a634e7b1f..01914e2d613 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -2,9 +2,9 @@ #include "worker.hh" #include "substitution-goal.hh" #include "drv-output-substitution-goal.hh" -#include "create-derivation-and-realise-goal.hh" #include "local-derivation-goal.hh" #include "hook-instance.hh" +#include "signals.hh" #include @@ -42,24 +42,6 @@ Worker::~Worker() } -std::shared_ptr Worker::makeCreateDerivationAndRealiseGoal( - ref drvReq, - const OutputsSpec & wantedOutputs, - BuildMode buildMode) -{ - std::weak_ptr & goal_weak = outerDerivationGoals.ensureSlot(*drvReq).value; - std::shared_ptr goal = goal_weak.lock(); - if (!goal) { - goal = std::make_shared(drvReq, wantedOutputs, *this, buildMode); - goal_weak = goal; - wakeUp(goal); - } else { - goal->addWantedOutputs(wantedOutputs); - } - return goal; -} - - std::shared_ptr Worker::makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, @@ -130,7 +112,10 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) { return std::visit(overloaded { [&](const DerivedPath::Built & bfd) -> GoalPtr { - return makeCreateDerivationAndRealiseGoal(bfd.drvPath, bfd.outputs, buildMode); + if (auto bop = std::get_if(&*bfd.drvPath)) + return makeDerivationGoal(bop->path, bfd.outputs, buildMode); + else + throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented."); }, [&](const DerivedPath::Opaque & bo) -> GoalPtr { return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); @@ -139,46 +124,24 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) } -template -static void cullMap(std::map & goalMap, F f) -{ - for (auto i = goalMap.begin(); i != goalMap.end();) - if (!f(i->second)) - i = goalMap.erase(i); - else ++i; -} - - template static void removeGoal(std::shared_ptr goal, std::map> & goalMap) { /* !!! inefficient */ - cullMap(goalMap, [&](const std::weak_ptr & gp) -> bool { - return gp.lock() != goal; - }); -} - -template -static void removeGoal(std::shared_ptr goal, std::map>::ChildNode> & goalMap); - -template -static void removeGoal(std::shared_ptr goal, std::map>::ChildNode> & goalMap) -{ - /* !!! inefficient */ - cullMap(goalMap, [&](DerivedPathMap>::ChildNode & node) -> bool { - if (node.value.lock() == goal) - node.value.reset(); - removeGoal(goal, node.childMap); - return !node.value.expired() || !node.childMap.empty(); - }); + for (auto i = goalMap.begin(); + i != goalMap.end(); ) + if (i->second.lock() == goal) { + auto j = i; ++j; + goalMap.erase(i); + i = j; + } + else ++i; } void Worker::removeGoal(GoalPtr goal) { - if (auto drvGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(drvGoal, outerDerivationGoals.map); - else if (auto drvGoal = std::dynamic_pointer_cast(goal)) + if (auto drvGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(drvGoal, derivationGoals); else if (auto subGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(subGoal, substitutionGoals); @@ -236,19 +199,8 @@ void Worker::childStarted(GoalPtr goal, const std::set & fds, child.respectTimeouts = respectTimeouts; children.emplace_back(child); if (inBuildSlot) { - switch (goal->jobCategory()) { - case JobCategory::Substitution: - nrSubstitutions++; - break; - case JobCategory::Build: - nrLocalBuilds++; - break; - case JobCategory::Administration: - /* Intentionally not limited, see docs */ - break; - default: - abort(); - } + if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++; + else nrLocalBuilds++; } } @@ -260,20 +212,12 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers) if (i == children.end()) return; if (i->inBuildSlot) { - switch (goal->jobCategory()) { - case JobCategory::Substitution: + if (goal->jobCategory() == JobCategory::Substitution) { assert(nrSubstitutions > 0); nrSubstitutions--; - break; - case JobCategory::Build: + } else { assert(nrLocalBuilds > 0); nrLocalBuilds--; - break; - case JobCategory::Administration: - /* Intentionally not limited, see docs */ - break; - default: - abort(); } } @@ -324,9 +268,9 @@ void Worker::run(const Goals & _topGoals) for (auto & i : _topGoals) { topGoals.insert(i); - if (auto goal = dynamic_cast(i.get())) { + if (auto goal = dynamic_cast(i.get())) { topPaths.push_back(DerivedPath::Built { - .drvPath = goal->drvReq, + .drvPath = makeConstantStorePathRef(goal->drvPath), .outputs = goal->wantedOutputs, }); } else if (auto goal = dynamic_cast(i.get())) { @@ -589,19 +533,4 @@ GoalPtr upcast_goal(std::shared_ptr subGoal) return subGoal; } -GoalPtr upcast_goal(std::shared_ptr subGoal) -{ - return subGoal; -} - -std::optional, std::reference_wrapper>> tryGetConcreteDrvGoal(GoalPtr waitee) -{ - auto * odg = dynamic_cast(&*waitee); - if (!odg) return std::nullopt; - return {{ - std::cref(*odg->concreteDrvGoal), - std::cref(*odg->drvReq), - }}; -} - } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 6f6d25d7d76..23ad879148d 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -4,7 +4,6 @@ #include "types.hh" #include "lock.hh" #include "store-api.hh" -#include "derived-path-map.hh" #include "goal.hh" #include "realisation.hh" @@ -14,7 +13,6 @@ namespace nix { /* Forward definition. */ -struct CreateDerivationAndRealiseGoal; struct DerivationGoal; struct PathSubstitutionGoal; class DrvOutputSubstitutionGoal; @@ -33,25 +31,9 @@ class DrvOutputSubstitutionGoal; */ GoalPtr upcast_goal(std::shared_ptr subGoal); GoalPtr upcast_goal(std::shared_ptr subGoal); -GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; -/** - * The current implementation of impure derivations has - * `DerivationGoal`s accumulate realisations from their waitees. - * Unfortunately, `DerivationGoal`s don't directly depend on other - * goals, but instead depend on `CreateDerivationAndRealiseGoal`s. - * - * We try not to share any of the details of any goal type with any - * other, for sake of modularity and quicker rebuilds. This means we - * cannot "just" downcast and fish out the field. So as an escape hatch, - * we have made the function, written in `worker.cc` where all the goal - * types are visible, and use it instead. - */ - -std::optional, std::reference_wrapper>> tryGetConcreteDrvGoal(GoalPtr waitee); - /** * A mapping used to remember for each child process to what goal it * belongs, and file descriptors for receiving log data and output @@ -119,9 +101,6 @@ private: * Maps used to prevent multiple instantiations of a goal for the * same derivation / path. */ - - DerivedPathMap> outerDerivationGoals; - std::map> derivationGoals; std::map> substitutionGoals; std::map> drvOutputSubstitutionGoals; @@ -209,9 +188,6 @@ public: * @ref DerivationGoal "derivation goal" */ private: - std::shared_ptr makeCreateDerivationAndRealiseGoal( - ref drvPath, - const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, std::function()> mkDrvGoal); diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 7bba33fb9a6..c8911d153fe 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -174,15 +174,19 @@ void builtinBuildenv(const BasicDerivation & drv) /* Convert the stuff we get from the environment back into a * coherent data type. */ Packages pkgs; - auto derivations = tokenizeString(getAttr("derivations")); - while (!derivations.empty()) { - /* !!! We're trusting the caller to structure derivations env var correctly */ - auto active = derivations.front(); derivations.pop_front(); - auto priority = stoi(derivations.front()); derivations.pop_front(); - auto outputs = stoi(derivations.front()); derivations.pop_front(); - for (auto n = 0; n < outputs; n++) { - auto path = derivations.front(); derivations.pop_front(); - pkgs.emplace_back(path, active != "false", priority); + { + auto derivations = tokenizeString(getAttr("derivations")); + + auto itemIt = derivations.begin(); + while (itemIt != derivations.end()) { + /* !!! We're trusting the caller to structure derivations env var correctly */ + const bool active = "false" != *itemIt++; + const int priority = stoi(*itemIt++); + const size_t outputs = stoul(*itemIt++); + + for (size_t n {0}; n < outputs; n++) { + pkgs.emplace_back(std::move(*itemIt++), active, priority); + } } } diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 7d7924d7733..35780033370 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -65,7 +65,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; std::optional ht = parseHashTypeOpt(getAttr("outputHashAlgo")); Hash h = newHashAllowEmpty(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); + fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(HashFormat::Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/common-protocol-impl.hh b/src/libstore/common-protocol-impl.hh new file mode 100644 index 00000000000..079c182b86a --- /dev/null +++ b/src/libstore/common-protocol-impl.hh @@ -0,0 +1,41 @@ +#pragma once +/** + * @file + * + * Template implementations (as opposed to mere declarations). + * + * This file is an exmample of the "impl.hh" pattern. See the + * contributing guide. + */ + +#include "common-protocol.hh" +#include "length-prefixed-protocol-helper.hh" + +namespace nix { + +/* protocol-agnostic templates */ + +#define COMMON_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ + TEMPLATE T CommonProto::Serialise< T >::read(const Store & store, CommonProto::ReadConn conn) \ + { \ + return LengthPrefixedProtoHelper::read(store, conn); \ + } \ + TEMPLATE void CommonProto::Serialise< T >::write(const Store & store, CommonProto::WriteConn conn, const T & t) \ + { \ + LengthPrefixedProtoHelper::write(store, conn, t); \ + } + +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) + +#define COMMA_ , +COMMON_USE_LENGTH_PREFIX_SERIALISER( + template, + std::map) +#undef COMMA_ + + +/* protocol-specific templates */ + +} diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc new file mode 100644 index 00000000000..68445258ff1 --- /dev/null +++ b/src/libstore/common-protocol.cc @@ -0,0 +1,97 @@ +#include "serialise.hh" +#include "path-with-outputs.hh" +#include "store-api.hh" +#include "build-result.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" +#include "archive.hh" +#include "derivations.hh" + +#include + +namespace nix { + +/* protocol-agnostic definitions */ + +std::string CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return readString(conn.from); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const std::string & str) +{ + conn.to << str; +} + + +StorePath CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return store.parseStorePath(readString(conn.from)); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const StorePath & storePath) +{ + conn.to << store.printStorePath(storePath); +} + + +ContentAddress CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return ContentAddress::parse(readString(conn.from)); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const ContentAddress & ca) +{ + conn.to << renderContentAddress(ca); +} + + +Realisation CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + std::string rawInput = readString(conn.from); + return Realisation::fromJSON( + nlohmann::json::parse(rawInput), + "remote-protocol" + ); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const Realisation & realisation) +{ + conn.to << realisation.toJSON().dump(); +} + + +DrvOutput CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return DrvOutput::parse(readString(conn.from)); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput) +{ + conn.to << drvOutput.to_string(); +} + + +std::optional CommonProto::Serialise>::read(const Store & store, CommonProto::ReadConn conn) +{ + auto s = readString(conn.from); + return s == "" ? std::optional {} : store.parseStorePath(s); +} + +void CommonProto::Serialise>::write(const Store & store, CommonProto::WriteConn conn, const std::optional & storePathOpt) +{ + conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); +} + + +std::optional CommonProto::Serialise>::read(const Store & store, CommonProto::ReadConn conn) +{ + return ContentAddress::parseOpt(readString(conn.from)); +} + +void CommonProto::Serialise>::write(const Store & store, CommonProto::WriteConn conn, const std::optional & caOpt) +{ + conn.to << (caOpt ? renderContentAddress(*caOpt) : ""); +} + +} diff --git a/src/libstore/common-protocol.hh b/src/libstore/common-protocol.hh new file mode 100644 index 00000000000..f3f28972afd --- /dev/null +++ b/src/libstore/common-protocol.hh @@ -0,0 +1,106 @@ +#pragma once +///@file + +#include "serialise.hh" + +namespace nix { + +class Store; +struct Source; + +// items being serialized +class StorePath; +struct ContentAddress; +struct DrvOutput; +struct Realisation; + + +/** + * Shared serializers between the worker protocol, serve protocol, and a + * few others. + * + * This `struct` is basically just a `namespace`; We use a type rather + * than a namespace just so we can use it as a template argument. + */ +struct CommonProto +{ + /** + * A unidirectional read connection, to be used by the read half of the + * canonical serializers below. + */ + struct ReadConn { + Source & from; + }; + + /** + * A unidirectional write connection, to be used by the write half of the + * canonical serializers below. + */ + struct WriteConn { + Sink & to; + }; + + template + struct Serialise; + + /** + * Wrapper function around `CommonProto::Serialise::write` that allows us to + * infer the type instead of having to write it down explicitly. + */ + template + static void write(const Store & store, WriteConn conn, const T & t) + { + CommonProto::Serialise::write(store, conn, t); + } +}; + +#define DECLARE_COMMON_SERIALISER(T) \ + struct CommonProto::Serialise< T > \ + { \ + static T read(const Store & store, CommonProto::ReadConn conn); \ + static void write(const Store & store, CommonProto::WriteConn conn, const T & str); \ + } + +template<> +DECLARE_COMMON_SERIALISER(std::string); +template<> +DECLARE_COMMON_SERIALISER(StorePath); +template<> +DECLARE_COMMON_SERIALISER(ContentAddress); +template<> +DECLARE_COMMON_SERIALISER(DrvOutput); +template<> +DECLARE_COMMON_SERIALISER(Realisation); + +template +DECLARE_COMMON_SERIALISER(std::vector); +template +DECLARE_COMMON_SERIALISER(std::set); +template +DECLARE_COMMON_SERIALISER(std::tuple); + +#define COMMA_ , +template +DECLARE_COMMON_SERIALISER(std::map); +#undef COMMA_ + +/** + * These use the empty string for the null case, relying on the fact + * that the underlying types never serialize to the empty string. + * + * We do this instead of a generic std::optional instance because + * ordinal tags (0 or 1, here) are a bit of a compatability hazard. For + * the same reason, we don't have a std::variant instances (ordinal + * tags 0...n). + * + * We could the generic instances and then these as specializations for + * compatability, but that's proven a bit finnicky, and also makes the + * worker protocol harder to implement in other languages where such + * specializations may not be allowed. + */ +template<> +DECLARE_COMMON_SERIALISER(std::optional); +template<> +DECLARE_COMMON_SERIALISER(std::optional); + +} diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index e290a8d387e..a5f7cdf8124 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -29,12 +29,13 @@ std::string ContentAddressMethod::renderPrefix() const ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) { - ContentAddressMethod method = FileIngestionMethod::Flat; - if (splitPrefix(m, "r:")) - method = FileIngestionMethod::Recursive; - else if (splitPrefix(m, "text:")) - method = TextIngestionMethod {}; - return method; + if (splitPrefix(m, "r:")) { + return FileIngestionMethod::Recursive; + } + else if (splitPrefix(m, "text:")) { + return TextIngestionMethod {}; + } + return FileIngestionMethod::Flat; } std::string ContentAddressMethod::render(HashType ht) const @@ -60,7 +61,7 @@ std::string ContentAddress::render() const + makeFileIngestionPrefix(method); }, }, method.raw) - + this->hash.to_string(Base32, true); + + this->hash.to_string(HashFormat::Base32, true); } /** @@ -83,7 +84,7 @@ static std::pair parseContentAddressMethodPrefix if (!hashTypeRaw) throw UsageError("content address hash must be in form ':', but found: %s", wholeInput); HashType hashType = parseHashType(*hashTypeRaw); - return std::move(hashType); + return hashType; }; // Switch on prefix diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 1027469c9ee..1b705733c30 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -1,4 +1,5 @@ #include "crypto.hh" +#include "file-system.hh" #include "util.hh" #include "globals.hh" diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 8cbf6f044fa..105d92f2531 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -45,9 +45,9 @@ struct TunnelLogger : public Logger Sync state_; - unsigned int clientVersion; + WorkerProto::Version clientVersion; - TunnelLogger(FdSink & to, unsigned int clientVersion) + TunnelLogger(FdSink & to, WorkerProto::Version clientVersion) : to(to), clientVersion(clientVersion) { } void enqueueMsg(const std::string & s) @@ -261,24 +261,18 @@ struct ClientSettings } }; -static std::vector readDerivedPaths(Store & store, unsigned int clientVersion, WorkerProto::ReadConn conn) -{ - std::vector reqs; - if (GET_PROTOCOL_MINOR(clientVersion) >= 30) { - reqs = WorkerProto::Serialise>::read(store, conn); - } else { - for (auto & s : readStrings(conn.from)) - reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath()); - } - return reqs; -} - static void performOp(TunnelLogger * logger, ref store, - TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion, + TrustedFlag trusted, RecursiveFlag recursive, WorkerProto::Version clientVersion, Source & from, BufferedSink & to, WorkerProto::Op op) { - WorkerProto::ReadConn rconn { .from = from }; - WorkerProto::WriteConn wconn { .to = to }; + WorkerProto::ReadConn rconn { + .from = from, + .version = clientVersion, + }; + WorkerProto::WriteConn wconn { + .to = to, + .version = clientVersion, + }; switch (op) { @@ -334,7 +328,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; logger->stopWork(); - to << hash.to_string(Base16, false); + to << hash.to_string(HashFormat::Base16, false); break; } @@ -428,7 +422,7 @@ static void performOp(TunnelLogger * logger, ref store, }(); logger->stopWork(); - pathInfo->write(to, *store, GET_PROTOCOL_MINOR(clientVersion)); + WorkerProto::Serialise::write(*store, wconn, *pathInfo); } else { HashType hashAlgo; std::string baseName; @@ -460,13 +454,13 @@ static void performOp(TunnelLogger * logger, ref store, eagerly consume the entire stream it's given, past the length of the Nar. */ TeeSource savedNARSource(from, saved); - ParseSink sink; /* null sink; just parse the NAR */ + NullParseSink sink; /* just parse the NAR */ parseDump(sink, savedNARSource); } else { /* Incrementally parse the NAR file, stripping the metadata, and streaming the sole file we expect into `saved`. */ - RetrieveRegularNARSink savedRegular { saved }; + RegularFileSink savedRegular { saved }; parseDump(savedRegular, from); if (!savedRegular.regular) throw Error("regular file expected"); } @@ -532,7 +526,7 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::BuildPaths: { - auto drvs = readDerivedPaths(*store, clientVersion, rconn); + auto drvs = WorkerProto::Serialise::read(*store, rconn); BuildMode mode = bmNormal; if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { mode = (BuildMode) readInt(from); @@ -557,7 +551,7 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::BuildPathsWithResults: { - auto drvs = readDerivedPaths(*store, clientVersion, rconn); + auto drvs = WorkerProto::Serialise::read(*store, rconn); BuildMode mode = bmNormal; mode = (BuildMode) readInt(from); @@ -641,16 +635,7 @@ static void performOp(TunnelLogger * logger, ref store, auto res = store->buildDerivation(drvPath, drv, buildMode); logger->stopWork(); - to << res.status << res.errorMsg; - if (GET_PROTOCOL_MINOR(clientVersion) >= 29) { - to << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime; - } - if (GET_PROTOCOL_MINOR(clientVersion) >= 28) { - DrvOutputs builtOutputs; - for (auto & [output, realisation] : res.builtOutputs) - builtOutputs.insert_or_assign(realisation.id, realisation); - WorkerProto::write(*store, wconn, builtOutputs); - } + WorkerProto::write(*store, wconn, res); break; } @@ -834,7 +819,7 @@ static void performOp(TunnelLogger * logger, ref store, if (info) { if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; - info->write(to, *store, GET_PROTOCOL_MINOR(clientVersion), false); + WorkerProto::write(*store, wconn, static_cast(*info)); } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); to << 0; @@ -914,7 +899,7 @@ static void performOp(TunnelLogger * logger, ref store, source = std::make_unique(from, to); else { TeeSource tee { from, saved }; - ParseSink ether; + NullParseSink ether; parseDump(ether, tee); source = std::make_unique(saved.s); } @@ -932,7 +917,7 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QueryMissing: { - auto targets = readDerivedPaths(*store, clientVersion, rconn); + auto targets = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; @@ -1017,7 +1002,7 @@ void processConnection( if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); to << WORKER_MAGIC_2 << PROTOCOL_VERSION; to.flush(); - unsigned int clientVersion = readInt(from); + WorkerProto::Version clientVersion = readInt(from); if (clientVersion < 0x10a) throw Error("the Nix client version is too old"); @@ -1052,7 +1037,10 @@ void processConnection( auto temp = trusted ? store->isTrustedClient() : std::optional { NotTrusted }; - WorkerProto::WriteConn wconn { .to = to }; + WorkerProto::WriteConn wconn { + .to = to, + .version = clientVersion, + }; WorkerProto::write(*store, wconn, temp); } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 67069c3c932..10962d142ff 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -4,9 +4,8 @@ #include "globals.hh" #include "util.hh" #include "split.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" -#include "fs-accessor.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include #include @@ -152,11 +151,10 @@ StorePath writeDerivation(Store & store, /* Read string `s' from stream `str'. */ static void expect(std::istream & str, std::string_view s) { - char s2[s.size()]; - str.read(s2, s.size()); - std::string_view s2View { s2, s.size() }; - if (s2View != s) - throw FormatError("expected string '%s', got '%s'", s, s2View); + for (auto & c : s) { + if (str.get() != c) + throw FormatError("expected string '%1%'", s); + } } @@ -353,7 +351,7 @@ Derivation parseDerivation( expect(str, "erive("); version = DerivationATermVersion::Traditional; break; - case 'r': + case 'r': { expect(str, "rvWithVersion("); auto versionS = parseString(str); if (versionS == "xp-dyn-drv") { @@ -366,6 +364,9 @@ Derivation parseDerivation( expect(str, ","); break; } + default: + throw Error("derivation does not start with 'Derive' or 'DrvWithVersion'"); + } /* Parse the list of outputs. */ expect(str, "["); @@ -542,7 +543,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, [&](const DerivationOutput::CAFixed & dof) { s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); s += ','; printUnquotedString(s, dof.ca.printMethodAlgo()); - s += ','; printUnquotedString(s, dof.ca.hash.to_string(Base16, false)); + s += ','; printUnquotedString(s, dof.ca.hash.to_string(HashFormat::Base16, false)); }, [&](const DerivationOutput::CAFloating & dof) { s += ','; printUnquotedString(s, ""); @@ -775,7 +776,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut auto & dof = std::get(i.second.raw); auto hash = hashString(htSHA256, "fixed:out:" + dof.ca.printMethodAlgo() + ":" - + dof.ca.hash.to_string(Base16, false) + ":" + + dof.ca.hash.to_string(HashFormat::Base16, false) + ":" + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } @@ -820,7 +821,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut const auto h = get(res.hashes, outputName); if (!h) throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); - inputs2[h->to_string(Base16, false)].value.insert(outputName); + inputs2[h->to_string(HashFormat::Base16, false)].value.insert(outputName); } } @@ -895,8 +896,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, drv.outputs.emplace(std::move(name), std::move(output)); } - drv.inputSrcs = WorkerProto::Serialise::read(store, - WorkerProto::ReadConn { .from = in }); + drv.inputSrcs = CommonProto::Serialise::read(store, + CommonProto::ReadConn { .from = in }); in >> drv.platform >> drv.builder; drv.args = readStrings(in); @@ -925,7 +926,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr [&](const DerivationOutput::CAFixed & dof) { out << store.printStorePath(dof.path(store, drv.name, i.first)) << dof.ca.printMethodAlgo() - << dof.ca.hash.to_string(Base16, false); + << dof.ca.hash.to_string(HashFormat::Base16, false); }, [&](const DerivationOutput::CAFloating & dof) { out << "" @@ -944,8 +945,8 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr }, }, i.second.raw); } - WorkerProto::write(store, - WorkerProto::WriteConn { .to = out }, + CommonProto::write(store, + CommonProto::WriteConn { .to = out }, drv.inputSrcs); out << drv.platform << drv.builder << drv.args; out << drv.env.size(); @@ -957,7 +958,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr std::string hashPlaceholder(const OutputNameView outputName) { // FIXME: memoize? - return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false); + return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Base32, false); } @@ -1001,13 +1002,13 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String } -std::optional Derivation::tryResolve(Store & store) const +std::optional Derivation::tryResolve(Store & store, Store * evalStore) const { std::map, StorePath> inputDrvOutputs; std::function::ChildNode &)> accum; accum = [&](auto & inputDrv, auto & node) { - for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv)) { + for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv, evalStore)) { if (outputPath) { inputDrvOutputs.insert_or_assign({inputDrv, outputName}, *outputPath); if (auto p = get(node.childMap, outputName)) @@ -1162,7 +1163,7 @@ nlohmann::json DerivationOutput::toJSON( [&](const DerivationOutput::CAFixed & dof) { res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); res["hashAlgo"] = dof.ca.printMethodAlgo(); - res["hash"] = dof.ca.hash.to_string(Base16, false); + res["hash"] = dof.ca.hash.to_string(HashFormat::Base16, false); // FIXME print refs? }, [&](const DerivationOutput::CAFloating & dof) { diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index fa14e75362d..7309918ce55 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -340,7 +340,7 @@ struct Derivation : BasicDerivation * 2. Input placeholders are replaced with realized input store * paths. */ - std::optional tryResolve(Store & store) const; + std::optional tryResolve(Store & store, Store * evalStore = nullptr) const; /** * Like the above, but instead of querying the Nix database for diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc index 437b6a71a97..4c1ea417a36 100644 --- a/src/libstore/derived-path-map.cc +++ b/src/libstore/derived-path-map.cc @@ -1,4 +1,5 @@ #include "derived-path-map.hh" +#include "util.hh" namespace nix { @@ -51,11 +52,8 @@ typename DerivedPathMap::ChildNode * DerivedPathMap::findSlot(const Single // instantiations -#include "create-derivation-and-realise-goal.hh" namespace nix { -template struct DerivedPathMap>; - GENERATE_CMP_EXT( template<>, DerivedPathMap>::ChildNode, diff --git a/src/libstore/derived-path-map.hh b/src/libstore/derived-path-map.hh index 4a2c907333c..393cdedf747 100644 --- a/src/libstore/derived-path-map.hh +++ b/src/libstore/derived-path-map.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "derived-path.hh" @@ -20,11 +21,8 @@ namespace nix { * * @param V A type to instantiate for each output. It should probably * should be an "optional" type so not every interior node has to have a - * value. For example, the scheduler uses - * `DerivedPathMap>` to - * remember which goals correspond to which outputs. `* const Something` - * or `std::optional` would also be good choices for - * "optional" types. + * value. `* const Something` or `std::optional` would be + * good choices for "optional" types. */ template struct DerivedPathMap { diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 4d7033df2f1..6c5dfeed9cc 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -1,10 +1,10 @@ #pragma once ///@file -#include "util.hh" #include "path.hh" #include "outputs-spec.hh" #include "comparator.hh" +#include "config.hh" #include diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index 7e3f7548d99..ca9f7476e7b 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -5,7 +5,7 @@ namespace nix { std::string DownstreamPlaceholder::render() const { - return "/" + hash.to_string(Base32, false); + return "/" + hash.to_string(HashFormat::Base32, false); } @@ -31,7 +31,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( xpSettings.require(Xp::DynamicDerivations); auto compressed = compressHash(placeholder.hash, 20); auto clearText = "nix-computed-output:" - + compressed.to_string(Base32, false) + + compressed.to_string(HashFormat::Base32, false) + ":" + std::string { outputName }; return DownstreamPlaceholder { hashString(htSHA256, clearText) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 74d6ed3b518..821cda399c9 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -72,7 +72,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store Callback> callback) noexcept override { callback(nullptr); } - virtual ref getFSAccessor() override + virtual ref getFSAccessor(bool requireValidPath) override { unsupported("getFSAccessor"); } }; diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index e866aeb42d2..52130f8f6e4 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -1,8 +1,8 @@ #include "serialise.hh" #include "store-api.hh" #include "archive.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include @@ -41,13 +41,13 @@ void Store::exportPath(const StorePath & path, Sink & sink) Hash hash = hashSink.currentHash().first; if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); + printStorePath(path), info->narHash.to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true)); teeSink << exportMagic << printStorePath(path); - WorkerProto::write(*this, - WorkerProto::WriteConn { .to = teeSink }, + CommonProto::write(*this, + CommonProto::WriteConn { .to = teeSink }, info->references); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") @@ -65,7 +65,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) /* Extract the NAR from the source. */ StringSink saved; TeeSource tee { source, saved }; - ParseSink ether; + NullParseSink ether; parseDump(ether, tee); uint32_t magic = readInt(source); @@ -76,8 +76,8 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) //Activity act(*logger, lvlInfo, "importing path '%s'", info.path); - auto references = WorkerProto::Serialise::read(*this, - WorkerProto::ReadConn { .from = source }); + auto references = CommonProto::Serialise::read(*this, + CommonProto::ReadConn { .from = source }); auto deriver = readString(source); auto narHash = hashString(htSHA256, saved.s); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index a283af5a2b3..dcbec4acd84 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -1,11 +1,12 @@ #include "filetransfer.hh" -#include "util.hh" +#include "namespaces.hh" #include "globals.hh" #include "store-api.hh" #include "s3.hh" #include "compression.hh" #include "finally.hh" #include "callback.hh" +#include "signals.hh" #if ENABLE_S3 #include diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh deleted file mode 100644 index 1df19e64709..00000000000 --- a/src/libstore/fs-accessor.hh +++ /dev/null @@ -1,52 +0,0 @@ -#pragma once -///@file - -#include "types.hh" - -namespace nix { - -/** - * An abstract class for accessing a filesystem-like structure, such - * as a (possibly remote) Nix store or the contents of a NAR file. - */ -class FSAccessor -{ -public: - enum Type { tMissing, tRegular, tSymlink, tDirectory }; - - struct Stat - { - Type type = tMissing; - /** - * regular files only - */ - uint64_t fileSize = 0; - /** - * regular files only - */ - bool isExecutable = false; // regular files only - /** - * regular files only - */ - uint64_t narOffset = 0; // regular files only - }; - - virtual ~FSAccessor() { } - - virtual Stat stat(const Path & path) = 0; - - virtual StringSet readDirectory(const Path & path) = 0; - - /** - * Read a file inside the store. - * - * If `requireValidPath` is set to `true` (the default), the path must be - * inside a valid store path, otherwise it just needs to be physically - * present (but not necessarily properly registered) - */ - virtual std::string readFile(const Path & path, bool requireValidPath = true) = 0; - - virtual std::string readLink(const Path & path) = 0; -}; - -} diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 26c87391c50..93fa60682ce 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -2,6 +2,13 @@ #include "globals.hh" #include "local-store.hh" #include "finally.hh" +#include "unix-domain-socket.hh" +#include "signals.hh" + +#if !defined(__linux__) +// For shelling out to lsof +# include "processes.hh" +#endif #include #include @@ -43,7 +50,7 @@ static void makeSymlink(const Path & link, const Path & target) void LocalStore::addIndirectRoot(const Path & path) { - std::string hash = hashString(htSHA1, path).to_string(Base32, false); + std::string hash = hashString(htSHA1, path).to_string(HashFormat::Base32, false); Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash)); makeSymlink(realRoot, path); } @@ -323,9 +330,7 @@ typedef std::unordered_map> UncheckedRoots static void readProcLink(const std::string & file, UncheckedRoots & roots) { - /* 64 is the starting buffer size gnu readlink uses... */ - auto bufsiz = ssize_t{64}; -try_again: + constexpr auto bufsiz = PATH_MAX; char buf[bufsiz]; auto res = readlink(file.c_str(), buf, bufsiz); if (res == -1) { @@ -334,10 +339,7 @@ static void readProcLink(const std::string & file, UncheckedRoots & roots) throw SysError("reading symlink"); } if (res == bufsiz) { - if (SSIZE_MAX / 2 < bufsiz) - throw Error("stupidly long symlink"); - bufsiz *= 2; - goto try_again; + throw Error("overly long symlink starting with '%1%'", std::string_view(buf, bufsiz)); } if (res > 0 && buf[0] == '/') roots[std::string(static_cast(buf), res)] @@ -776,7 +778,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } }; - /* Synchronisation point for testing, see tests/gc-concurrent.sh. */ + /* Synchronisation point for testing, see tests/functional/gc-concurrent.sh. */ if (auto p = getEnv("_NIX_TEST_GC_SYNC")) readFile(*p); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 5a4cb18240c..cc416a4d621 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -1,7 +1,8 @@ #include "globals.hh" -#include "util.hh" +#include "current-process.hh" #include "archive.hh" #include "args.hh" +#include "users.hh" #include "abstract-setting-to-json.hh" #include "compute-levels.hh" @@ -17,13 +18,20 @@ #include #ifdef __GLIBC__ -#include -#include -#include +# include +# include +# include +#endif + +#if __APPLE__ +# include "processes.hh" #endif #include "config-impl.hh" +#ifdef __APPLE__ +#include +#endif namespace nix { @@ -154,6 +162,29 @@ unsigned int Settings::getDefaultCores() return concurrency; } +#if __APPLE__ +static bool hasVirt() { + + int hasVMM; + int hvSupport; + size_t size; + + size = sizeof(hasVMM); + if (sysctlbyname("kern.hv_vmm_present", &hasVMM, &size, NULL, 0) == 0) { + if (hasVMM) + return false; + } + + // whether the kernel and hardware supports virt + size = sizeof(hvSupport); + if (sysctlbyname("kern.hv_support", &hvSupport, &size, NULL, 0) == 0) { + return hvSupport == 1; + } else { + return false; + } +} +#endif + StringSet Settings::getDefaultSystemFeatures() { /* For backwards compatibility, accept some "features" that are @@ -170,6 +201,11 @@ StringSet Settings::getDefaultSystemFeatures() features.insert("kvm"); #endif + #if __APPLE__ + if (hasVirt()) + features.insert("apple-virt"); + #endif + return features; } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index cf10edebde4..27caf42c4ed 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -3,7 +3,8 @@ #include "types.hh" #include "config.hh" -#include "util.hh" +#include "environment-variables.hh" +#include "experimental-features.hh" #include #include @@ -261,6 +262,14 @@ public: For the exact format and examples, see [the manual chapter on remote builds](../advanced-topics/distributed-builds.md) )"}; + Setting alwaysAllowSubstitutes{ + this, false, "always-allow-substitutes", + R"( + If set to `true`, Nix will ignore the `allowSubstitutes` attribute in + derivations and always attempt to use available substituters. + For more information on `allowSubstitutes`, see [the manual chapter on advanced attributes](../language/advanced-attributes.md). + )"}; + Setting buildersUseSubstitutes{ this, false, "builders-use-substitutes", R"( @@ -343,7 +352,7 @@ public: users in `build-users-group`. UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS. - )"}; + )", {}, true, Xp::AutoAllocateUids}; Setting startId{this, #if __linux__ @@ -697,19 +706,44 @@ public: getDefaultSystemFeatures(), "system-features", R"( - A set of system “features” supported by this machine, e.g. `kvm`. - Derivations can express a dependency on such features through the - derivation attribute `requiredSystemFeatures`. For example, the - attribute + A set of system “features” supported by this machine. + + This complements the [`system`](#conf-system) and [`extra-platforms`](#conf-extra-platforms) configuration options and the corresponding [`system`](@docroot@/language/derivations.md#attr-system) attribute on derivations. + + A derivation can require system features in the [`requiredSystemFeatures` attribute](@docroot@/language/advanced-attributes.md#adv-attr-requiredSystemFeatures), and the machine to build the derivation must have them. + + System features are user-defined, but Nix sets the following defaults: + + - `apple-virt` + + Included on Darwin if virtualization is available. + + - `kvm` + + Included on Linux if `/dev/kvm` is accessible. + + - `nixos-test`, `benchmark`, `big-parallel` + + These historical pseudo-features are always enabled for backwards compatibility, as they are used in Nixpkgs to route Hydra builds to specific machines. + + - `ca-derivations` + + Included by default if the [`ca-derivations` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) is enabled. + + This system feature is implicitly required by derivations with the [`__contentAddressed` attribute](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed). - requiredSystemFeatures = [ "kvm" ]; + - `recursive-nix` - ensures that the derivation can only be built on a machine with the - `kvm` feature. + Included by default if the [`recursive-nix` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-recursive-nix) is enabled. - This setting by default includes `kvm` if `/dev/kvm` is accessible, - and the pseudo-features `nixos-test`, `benchmark` and `big-parallel` - that are used in Nixpkgs to route builds to specific machines. + - `uid-range` + + On Linux, Nix can run builds in a user namespace where they run as root (UID 0) and have 65,536 UIDs available. + This is primarily useful for running containers such as `systemd-nspawn` inside a Nix build. For an example, see [`tests/systemd-nspawn/nix`][nspawn]. + + [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. + + Included by default on Linux if the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting is enabled. )", {}, false}; Setting substituters{ @@ -1031,6 +1065,35 @@ public: ``` )" }; + + Setting impureEnv {this, {}, "impure-env", + R"( + A list of items, each in the format of: + + - `name=value`: Set environment variable `name` to `value`. + + If the user is trusted (see `trusted-users` option), when building + a fixed-output derivation, environment variables set in this option + will be passed to the builder if they are listed in [`impureEnvVars`](@docroot@/language/advanced-attributes.md##adv-attr-impureEnvVars). + + This option is useful for, e.g., setting `https_proxy` for + fixed-output derivations and in a multi-user Nix installation, or + setting private access tokens when fetching a private repository. + )", + {}, // aliases + true, // document default + Xp::ConfigurableImpureEnv + }; + + Setting upgradeNixStorePathUrl{ + this, + "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix", + "upgrade-nix-store-path-url", + R"( + Used by `nix upgrade-nix`, the URL of the file that contains the + store paths of the latest Nix release. + )" + }; }; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 78b05031ae6..7314573547f 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -3,11 +3,10 @@ #include "pool.hh" #include "remote-store.hh" #include "serve-protocol.hh" +#include "serve-protocol-impl.hh" #include "build-result.hh" #include "store-api.hh" #include "path-with-outputs.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" #include "ssh.hh" #include "derivations.hh" #include "callback.hh" @@ -18,10 +17,10 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig { using CommonSSHStoreConfig::CommonSSHStoreConfig; - const Setting remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", + const Setting remoteProgram{this, "nix-store", "remote-program", "Path to the `nix-store` executable on the remote machine."}; - const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", + const Setting maxConnections{this, 1, "max-connections", "Maximum number of concurrent SSH connections."}; const std::string name() override { return "SSH Store"; } @@ -39,49 +38,45 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor // Hack for getting remote build log output. // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in // the documentation - const Setting logFD{(StoreConfig*) this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; + const Setting logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; struct Connection { std::unique_ptr sshConn; FdSink to; FdSource from; - int remoteVersion; + ServeProto::Version remoteVersion; bool good = true; /** - * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the - * factored out worker protocol searlizers with a + * Coercion to `ServeProto::ReadConn`. This makes it easy to use the + * factored out serve protocol searlizers with a * `LegacySSHStore::Connection`. * - * The worker protocol connection types are unidirectional, unlike + * The serve protocol connection types are unidirectional, unlike * this type. - * - * @todo Use server protocol serializers, not worker protocol - * serializers, once we have made that distiction. */ - operator WorkerProto::ReadConn () + operator ServeProto::ReadConn () { - return WorkerProto::ReadConn { + return ServeProto::ReadConn { .from = from, + .version = remoteVersion, }; } /* - * Coercion to `WorkerProto::WriteConn`. This makes it easy to use the - * factored out worker protocol searlizers with a + * Coercion to `ServeProto::WriteConn`. This makes it easy to use the + * factored out serve protocol searlizers with a * `LegacySSHStore::Connection`. * - * The worker protocol connection types are unidirectional, unlike + * The serve protocol connection types are unidirectional, unlike * this type. - * - * @todo Use server protocol serializers, not worker protocol - * serializers, once we have made that distiction. */ - operator WorkerProto::WriteConn () + operator ServeProto::WriteConn () { - return WorkerProto::WriteConn { + return ServeProto::WriteConn { .to = to, + .version = remoteVersion, }; } }; @@ -183,7 +178,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = WorkerProto::Serialise::read(*this, *conn); + info->references = ServeProto::Serialise::read(*this, *conn); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -216,8 +211,8 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << ServeProto::Command::AddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); - WorkerProto::write(*this, *conn, info.references); + << info.narHash.to_string(HashFormat::Base16, false); + ServeProto::write(*this, *conn, info.references); conn->to << info.registrationTime << info.narSize @@ -246,7 +241,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << exportMagic << printStorePath(info.path); - WorkerProto::write(*this, *conn, info.references); + ServeProto::write(*this, *conn, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 @@ -324,20 +319,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to.flush(); - BuildResult status; - status.status = (BuildResult::Status) readInt(conn->from); - conn->from >> status.errorMsg; - - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) - conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { - auto builtOutputs = WorkerProto::Serialise::read(*this, *conn); - for (auto && [output, realisation] : builtOutputs) - status.builtOutputs.insert_or_assign( - std::move(output.outputName), - std::move(realisation)); - } - return status; + return ServeProto::Serialise::read(*this, *conn); } void buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) override @@ -381,7 +363,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor void ensurePath(const StorePath & path) override { unsupported("ensurePath"); } - virtual ref getFSAccessor() override + virtual ref getFSAccessor(bool requireValidPath) override { unsupported("getFSAccessor"); } /** @@ -409,10 +391,10 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << ServeProto::Command::QueryClosure << includeOutputs; - WorkerProto::write(*this, *conn, paths); + ServeProto::write(*this, *conn, paths); conn->to.flush(); - for (auto & i : WorkerProto::Serialise::read(*this, *conn)) + for (auto & i : ServeProto::Serialise::read(*this, *conn)) out.insert(i); } @@ -425,10 +407,10 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << ServeProto::Command::QueryValidPaths << false // lock << maybeSubstitute; - WorkerProto::write(*this, *conn, paths); + ServeProto::write(*this, *conn, paths); conn->to.flush(); - return WorkerProto::Serialise::read(*this, *conn); + return ServeProto::Serialise::read(*this, *conn); } void connect() override diff --git a/src/libstore/length-prefixed-protocol-helper.hh b/src/libstore/length-prefixed-protocol-helper.hh new file mode 100644 index 00000000000..4061b0cd6de --- /dev/null +++ b/src/libstore/length-prefixed-protocol-helper.hh @@ -0,0 +1,162 @@ +#pragma once +/** + * @file Reusable serialisers for serialization container types in a + * length-prefixed manner. + * + * Used by both the Worker and Serve protocols. + */ + +#include "types.hh" + +namespace nix { + +class Store; + +/** + * Reusable serialisers for serialization container types in a + * length-prefixed manner. + * + * @param T The type of the collection being serialised + * + * @param Inner This the most important parameter; this is the "inner" + * protocol. The user of this will substitute `MyProtocol` or similar + * when making a `MyProtocol::Serialiser>`. Note that the + * inside is allowed to call to call `Inner::Serialiser` on different + * types. This is especially important for `std::map` which doesn't have + * a single `T` but one `K` and one `V`. + */ +template +struct LengthPrefixedProtoHelper; + +/*! + * \typedef LengthPrefixedProtoHelper::S + * + * Read this as simply `using S = Inner::Serialise;`. + * + * It would be nice to use that directly, but C++ doesn't seem to allow + * it. The `typename` keyword needed to refer to `Inner` seems to greedy + * (low precedence), and then C++ complains that `Serialise` is not a + * type parameter but a real type. + * + * Making this `S` alias seems to be the only way to avoid these issues. + */ + +#define LENGTH_PREFIXED_PROTO_HELPER(Inner, T) \ + struct LengthPrefixedProtoHelper< Inner, T > \ + { \ + static T read(const Store & store, typename Inner::ReadConn conn); \ + static void write(const Store & store, typename Inner::WriteConn conn, const T & str); \ + private: \ + template using S = typename Inner::template Serialise; \ + } + +template +LENGTH_PREFIXED_PROTO_HELPER(Inner, std::vector); + +template +LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set); + +template +LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple); + +template +#define _X std::map +LENGTH_PREFIXED_PROTO_HELPER(Inner, _X); +#undef _X + +template +std::vector +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + std::vector resSet; + auto size = readNum(conn.from); + while (size--) { + resSet.push_back(S::read(store, conn)); + } + return resSet; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::vector & resSet) +{ + conn.to << resSet.size(); + for (auto & key : resSet) { + S::write(store, conn, key); + } +} + +template +std::set +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + std::set resSet; + auto size = readNum(conn.from); + while (size--) { + resSet.insert(S::read(store, conn)); + } + return resSet; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::set & resSet) +{ + conn.to << resSet.size(); + for (auto & key : resSet) { + S::write(store, conn, key); + } +} + +template +std::map +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + std::map resMap; + auto size = readNum(conn.from); + while (size--) { + auto k = S::read(store, conn); + auto v = S::read(store, conn); + resMap.insert_or_assign(std::move(k), std::move(v)); + } + return resMap; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::map & resMap) +{ + conn.to << resMap.size(); + for (auto & i : resMap) { + S::write(store, conn, i.first); + S::write(store, conn, i.second); + } +} + +template +std::tuple +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + return std::tuple { + S::read(store, conn)..., + }; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::tuple & res) +{ + std::apply([&](const Us &... args) { + (S::write(store, conn, args), ...); + }, res); +} + +} diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index b224fc3e989..953f3a264eb 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -1,5 +1,5 @@ #include "archive.hh" -#include "fs-accessor.hh" +#include "posix-source-accessor.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" @@ -13,69 +13,53 @@ LocalFSStore::LocalFSStore(const Params & params) { } -struct LocalStoreAccessor : public FSAccessor +struct LocalStoreAccessor : PosixSourceAccessor { ref store; + bool requireValidPath; - LocalStoreAccessor(ref store) : store(store) { } + LocalStoreAccessor(ref store, bool requireValidPath) + : store(store) + , requireValidPath(requireValidPath) + { } - Path toRealPath(const Path & path, bool requireValidPath = true) + CanonPath toRealPath(const CanonPath & path) { - auto storePath = store->toStorePath(path).first; + auto [storePath, rest] = store->toStorePath(path.abs()); if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); - return store->getRealStoreDir() + std::string(path, store->storeDir.size()); + return CanonPath(store->getRealStoreDir()) + storePath.to_string() + CanonPath(rest); } - FSAccessor::Stat stat(const Path & path) override + std::optional maybeLstat(const CanonPath & path) override { - auto realPath = toRealPath(path); - - struct stat st; - if (lstat(realPath.c_str(), &st)) { - if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; - throw SysError("getting status of '%1%'", path); - } - - if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) - throw Error("file '%1%' has unsupported type", path); - - return { - S_ISREG(st.st_mode) ? Type::tRegular : - S_ISLNK(st.st_mode) ? Type::tSymlink : - Type::tDirectory, - S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0, - S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}; + return PosixSourceAccessor::maybeLstat(toRealPath(path)); } - StringSet readDirectory(const Path & path) override + DirEntries readDirectory(const CanonPath & path) override { - auto realPath = toRealPath(path); - - auto entries = nix::readDirectory(realPath); - - StringSet res; - for (auto & entry : entries) - res.insert(entry.name); - - return res; + return PosixSourceAccessor::readDirectory(toRealPath(path)); } - std::string readFile(const Path & path, bool requireValidPath = true) override + void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) override { - return nix::readFile(toRealPath(path, requireValidPath)); + return PosixSourceAccessor::readFile(toRealPath(path), sink, sizeCallback); } - std::string readLink(const Path & path) override + std::string readLink(const CanonPath & path) override { - return nix::readLink(toRealPath(path)); + return PosixSourceAccessor::readLink(toRealPath(path)); } }; -ref LocalFSStore::getFSAccessor() +ref LocalFSStore::getFSAccessor(bool requireValidPath) { return make_ref(ref( - std::dynamic_pointer_cast(shared_from_this()))); + std::dynamic_pointer_cast(shared_from_this())), + requireValidPath); } void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 48810950113..bf855b67ece 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -11,25 +11,21 @@ struct LocalFSStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes - // it to omit the call to the Setting constructor. Clang works fine - // either way. - - const OptionalPathSetting rootDir{(StoreConfig*) this, std::nullopt, + const OptionalPathSetting rootDir{this, std::nullopt, "root", "Directory prefixed to all other paths."}; - const PathSetting stateDir{(StoreConfig*) this, + const PathSetting stateDir{this, rootDir.get() ? *rootDir.get() + "/nix/var/nix" : settings.nixStateDir, "state", "Directory where Nix will store state."}; - const PathSetting logDir{(StoreConfig*) this, + const PathSetting logDir{this, rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : settings.nixLogDir, "log", "directory where Nix will store log files."}; - const PathSetting realStoreDir{(StoreConfig*) this, + const PathSetting realStoreDir{this, rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real", "Physical path of the Nix store."}; }; @@ -47,7 +43,7 @@ public: LocalFSStore(const Params & params); void narFromPath(const StorePath & path, Sink & sink) override; - ref getFSAccessor() override; + ref getFSAccessor(bool requireValidPath) override; /** * Creates symlink from the `gcRoot` to the `storePath` and diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 17b4ecc7312..2a3582ad8ea 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -10,6 +10,7 @@ #include "topo-sort.hh" #include "finally.hh" #include "compression.hh" +#include "signals.hh" #include #include @@ -826,7 +827,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmts->RegisterValidPath.use() (printStorePath(info.path)) - (info.narHash.to_string(Base16, true)) + (info.narHash.to_string(HashFormat::Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) @@ -933,7 +934,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmts->UpdatePathInfo.use() (info.narSize, info.narSize != 0) - (info.narHash.to_string(Base16, true)) + (info.narHash.to_string(HashFormat::Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (renderContentAddress(info.ca), (bool) info.ca) @@ -1200,7 +1201,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, bool narRead = false; Finally cleanup = [&]() { if (!narRead) { - ParseSink sink; + NullParseSink sink; parseDump(sink, source); } }; @@ -1236,7 +1237,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); + printStorePath(info.path), info.narHash.to_string(HashFormat::Base32, true), hashResult.first.to_string(HashFormat::Base32, true)); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s", @@ -1252,8 +1253,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (specified.hash != actualHash.hash) { throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", printStorePath(info.path), - specified.hash.to_string(Base32, true), - actualHash.hash.to_string(Base32, true)); + specified.hash.to_string(HashFormat::Base32, true), + actualHash.hash.to_string(HashFormat::Base32, true)); } } @@ -1545,7 +1546,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) for (auto & link : readDirectory(linksDir)) { printMsg(lvlTalkative, "checking contents of '%s'", link.name); Path linkPath = linksDir + "/" + link.name; - std::string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); + std::string hash = hashPath(htSHA256, linkPath).first.to_string(HashFormat::Base32, false); if (hash != link.name) { printError("link '%s' was modified! expected hash '%s', got '%s'", linkPath, link.name, hash); @@ -1578,7 +1579,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) if (info->narHash != nullHash && info->narHash != current.first) { printError("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)); + printStorePath(i), info->narHash.to_string(HashFormat::Base32, true), current.first.to_string(HashFormat::Base32, true)); if (repair) repairPath(i); else errors = true; } else { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index e97195f5b88..6d589bee5b1 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -7,7 +7,6 @@ #include "store-api.hh" #include "indirect-root-store.hh" #include "sync.hh" -#include "util.hh" #include #include @@ -40,12 +39,12 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; - Setting requireSigs{(StoreConfig*) this, + Setting requireSigs{this, settings.requireSigs, "require-sigs", "Whether store paths copied into this store should have a trusted signature."}; - Setting readOnly{(StoreConfig*) this, + Setting readOnly{this, false, "read-only", R"( diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 7202a64b3c5..87f55ce496b 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -1,4 +1,5 @@ #include "lock.hh" +#include "file-system.hh" #include "globals.hh" #include "pathlocks.hh" @@ -7,6 +8,31 @@ namespace nix { +#if __linux__ + +static std::vector get_group_list(const char *username, gid_t group_id) +{ + std::vector gids; + gids.resize(32); // Initial guess + + auto getgroupl_failed {[&] { + int ngroups = gids.size(); + int err = getgrouplist(username, group_id, gids.data(), &ngroups); + gids.resize(ngroups); + return err == -1; + }}; + + // The first error means that the vector was not big enough. + // If it happens again, there is some different problem. + if (getgroupl_failed() && getgroupl_failed()) { + throw SysError("failed to get list of supplementary groups for '%s'", username); + } + + return gids; +} +#endif + + struct SimpleUserLock : UserLock { AutoCloseFD fdUserLock; @@ -67,37 +93,14 @@ struct SimpleUserLock : UserLock throw Error("the Nix user should not be a member of '%s'", settings.buildUsersGroup); #if __linux__ - /* Get the list of supplementary groups of this build - user. This is usually either empty or contains a - group such as "kvm". */ - int ngroups = 32; // arbitrary initial guess - std::vector gids; - gids.resize(ngroups); - - int err = getgrouplist( - pw->pw_name, pw->pw_gid, - gids.data(), - &ngroups); - - /* Our initial size of 32 wasn't sufficient, the - correct size has been stored in ngroups, so we try - again. */ - if (err == -1) { - gids.resize(ngroups); - err = getgrouplist( - pw->pw_name, pw->pw_gid, - gids.data(), - &ngroups); - } - - // If it failed once more, then something must be broken. - if (err == -1) - throw Error("failed to get list of supplementary groups for '%s'", pw->pw_name); + /* Get the list of supplementary groups of this user. This is + * usually either empty or contains a group such as "kvm". */ // Finally, trim back the GID list to its real size. - for (auto i = 0; i < ngroups; i++) - if (gids[i] != lock->gid) - lock->supplementaryGIDs.push_back(gids[i]); + for (auto gid : get_group_list(pw->pw_name, pw->pw_gid)) { + if (gid != lock->gid) + lock->supplementaryGIDs.push_back(gid); + } #endif return lock; diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index e87f469800d..512115893ed 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -1,5 +1,4 @@ #include "machines.hh" -#include "util.hh" #include "globals.hh" #include "store-api.hh" diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index f0dfcb19b77..15b05fe25fe 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -11,13 +11,7 @@ namespace nix { struct NarMember { - FSAccessor::Type type = FSAccessor::Type::tMissing; - - bool isExecutable = false; - - /* If this is a regular file, position of the contents of this - file in the NAR. */ - uint64_t start = 0, size = 0; + SourceAccessor::Stat stat; std::string target; @@ -25,7 +19,7 @@ struct NarMember std::map children; }; -struct NarAccessor : public FSAccessor +struct NarAccessor : public SourceAccessor { std::optional nar; @@ -57,7 +51,7 @@ struct NarAccessor : public FSAccessor acc.root = std::move(member); parents.push(&acc.root); } else { - if (parents.top()->type != FSAccessor::Type::tDirectory) + if (parents.top()->stat.type != Type::tDirectory) throw Error("NAR file missing parent directory of path '%s'", path); auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member)); parents.push(&result.first->second); @@ -66,12 +60,22 @@ struct NarAccessor : public FSAccessor void createDirectory(const Path & path) override { - createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0}); + createMember(path, NarMember{ .stat = { + .type = Type::tDirectory, + .fileSize = 0, + .isExecutable = false, + .narOffset = 0 + } }); } void createRegularFile(const Path & path) override { - createMember(path, {FSAccessor::Type::tRegular, false, 0, 0}); + createMember(path, NarMember{ .stat = { + .type = Type::tRegular, + .fileSize = 0, + .isExecutable = false, + .narOffset = 0 + } }); } void closeRegularFile() override @@ -79,14 +83,14 @@ struct NarAccessor : public FSAccessor void isExecutable() override { - parents.top()->isExecutable = true; + parents.top()->stat.isExecutable = true; } void preallocateContents(uint64_t size) override { - assert(size <= std::numeric_limits::max()); - parents.top()->size = (uint64_t) size; - parents.top()->start = pos; + auto & st = parents.top()->stat; + st.fileSize = size; + st.narOffset = pos; } void receiveContents(std::string_view data) override @@ -95,7 +99,9 @@ struct NarAccessor : public FSAccessor void createSymlink(const Path & path, const std::string & target) override { createMember(path, - NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); + NarMember{ + .stat = {.type = Type::tSymlink}, + .target = target}); } size_t read(char * data, size_t len) override @@ -130,18 +136,19 @@ struct NarAccessor : public FSAccessor std::string type = v["type"]; if (type == "directory") { - member.type = FSAccessor::Type::tDirectory; - for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) { - std::string name = i.key(); - recurse(member.children[name], i.value()); + member.stat = {.type = Type::tDirectory}; + for (const auto &[name, function] : v["entries"].items()) { + recurse(member.children[name], function); } } else if (type == "regular") { - member.type = FSAccessor::Type::tRegular; - member.size = v["size"]; - member.isExecutable = v.value("executable", false); - member.start = v["narOffset"]; + member.stat = { + .type = Type::tRegular, + .fileSize = v["size"], + .isExecutable = v.value("executable", false), + .narOffset = v["narOffset"] + }; } else if (type == "symlink") { - member.type = FSAccessor::Type::tSymlink; + member.stat = {.type = Type::tSymlink}; member.target = v.value("target", ""); } else return; }; @@ -150,134 +157,122 @@ struct NarAccessor : public FSAccessor recurse(root, v); } - NarMember * find(const Path & path) + NarMember * find(const CanonPath & path) { - Path canon = path == "" ? "" : canonPath(path); NarMember * current = &root; - auto end = path.end(); - for (auto it = path.begin(); it != end; ) { - // because it != end, the remaining component is non-empty so we need - // a directory - if (current->type != FSAccessor::Type::tDirectory) return nullptr; - - // skip slash (canonPath above ensures that this is always a slash) - assert(*it == '/'); - it += 1; - - // lookup current component - auto next = std::find(it, end, '/'); - auto child = current->children.find(std::string(it, next)); + + for (const auto & i : path) { + if (current->stat.type != Type::tDirectory) return nullptr; + auto child = current->children.find(std::string(i)); if (child == current->children.end()) return nullptr; current = &child->second; - - it = next; } return current; } - NarMember & get(const Path & path) { + NarMember & get(const CanonPath & path) { auto result = find(path); - if (result == nullptr) + if (!result) throw Error("NAR file does not contain path '%1%'", path); return *result; } - Stat stat(const Path & path) override + std::optional maybeLstat(const CanonPath & path) override { auto i = find(path); - if (i == nullptr) - return {FSAccessor::Type::tMissing, 0, false}; - return {i->type, i->size, i->isExecutable, i->start}; + if (!i) + return std::nullopt; + return i->stat; } - StringSet readDirectory(const Path & path) override + DirEntries readDirectory(const CanonPath & path) override { auto i = get(path); - if (i.type != FSAccessor::Type::tDirectory) + if (i.stat.type != Type::tDirectory) throw Error("path '%1%' inside NAR file is not a directory", path); - StringSet res; - for (auto & child : i.children) - res.insert(child.first); + DirEntries res; + for (const auto & child : i.children) + res.insert_or_assign(child.first, std::nullopt); return res; } - std::string readFile(const Path & path, bool requireValidPath = true) override + std::string readFile(const CanonPath & path) override { auto i = get(path); - if (i.type != FSAccessor::Type::tRegular) + if (i.stat.type != Type::tRegular) throw Error("path '%1%' inside NAR file is not a regular file", path); - if (getNarBytes) return getNarBytes(i.start, i.size); + if (getNarBytes) return getNarBytes(*i.stat.narOffset, *i.stat.fileSize); assert(nar); - return std::string(*nar, i.start, i.size); + return std::string(*nar, *i.stat.narOffset, *i.stat.fileSize); } - std::string readLink(const Path & path) override + std::string readLink(const CanonPath & path) override { auto i = get(path); - if (i.type != FSAccessor::Type::tSymlink) + if (i.stat.type != Type::tSymlink) throw Error("path '%1%' inside NAR file is not a symlink", path); return i.target; } }; -ref makeNarAccessor(std::string && nar) +ref makeNarAccessor(std::string && nar) { return make_ref(std::move(nar)); } -ref makeNarAccessor(Source & source) +ref makeNarAccessor(Source & source) { return make_ref(source); } -ref makeLazyNarAccessor(const std::string & listing, +ref makeLazyNarAccessor(const std::string & listing, GetNarBytes getNarBytes) { return make_ref(listing, getNarBytes); } using nlohmann::json; -json listNar(ref accessor, const Path & path, bool recurse) +json listNar(ref accessor, const CanonPath & path, bool recurse) { - auto st = accessor->stat(path); + auto st = accessor->lstat(path); json obj = json::object(); switch (st.type) { - case FSAccessor::Type::tRegular: + case SourceAccessor::Type::tRegular: obj["type"] = "regular"; - obj["size"] = st.fileSize; + if (st.fileSize) + obj["size"] = *st.fileSize; if (st.isExecutable) obj["executable"] = true; - if (st.narOffset) - obj["narOffset"] = st.narOffset; + if (st.narOffset && *st.narOffset) + obj["narOffset"] = *st.narOffset; break; - case FSAccessor::Type::tDirectory: + case SourceAccessor::Type::tDirectory: obj["type"] = "directory"; { obj["entries"] = json::object(); json &res2 = obj["entries"]; - for (auto & name : accessor->readDirectory(path)) { + for (const auto & [name, type] : accessor->readDirectory(path)) { if (recurse) { - res2[name] = listNar(accessor, path + "/" + name, true); + res2[name] = listNar(accessor, path + name, true); } else res2[name] = json::object(); } } break; - case FSAccessor::Type::tSymlink: + case SourceAccessor::Type::tSymlink: obj["type"] = "symlink"; obj["target"] = accessor->readLink(path); break; - case FSAccessor::Type::tMissing: - default: - throw Error("path '%s' does not exist in NAR", path); + case SourceAccessor::Type::tMisc: + assert(false); // cannot happen for NARs } return obj; } diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 5e19bd3c755..0043897c658 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -1,10 +1,11 @@ #pragma once ///@file +#include "source-accessor.hh" + #include #include -#include "fs-accessor.hh" namespace nix { @@ -14,9 +15,9 @@ struct Source; * Return an object that provides access to the contents of a NAR * file. */ -ref makeNarAccessor(std::string && nar); +ref makeNarAccessor(std::string && nar); -ref makeNarAccessor(Source & source); +ref makeNarAccessor(Source & source); /** * Create a NAR accessor from a NAR listing (in the format produced by @@ -24,9 +25,9 @@ ref makeNarAccessor(Source & source); * readFile() method of the accessor to get the contents of files * inside the NAR. */ -typedef std::function GetNarBytes; +using GetNarBytes = std::function; -ref makeLazyNarAccessor( +ref makeLazyNarAccessor( const std::string & listing, GetNarBytes getNarBytes); @@ -34,6 +35,6 @@ ref makeLazyNarAccessor( * Write a JSON representation of the contents of a NAR (except file * contents). */ -nlohmann::json listNar(ref accessor, const Path & path, bool recurse); +nlohmann::json listNar(ref accessor, const CanonPath & path, bool recurse); } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index c7176d30f7a..e50c15939dc 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -1,4 +1,5 @@ #include "nar-info-disk-cache.hh" +#include "users.hh" #include "sync.hh" #include "sqlite.hh" #include "globals.hh" @@ -332,9 +333,9 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash) + (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(HashFormat::Base32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash.to_string(Base32, true)) + (info->narHash.to_string(HashFormat::Base32, true)) (info->narSize) (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index d17253741a6..1060a6c8ba8 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -4,6 +4,15 @@ namespace nix { +GENERATE_CMP_EXT( + , + NarInfo, + me->url, + me->compression, + me->fileHash, + me->fileSize, + static_cast(*me)); + NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) : ValidPathInfo(StorePath(StorePath::dummy), Hash(Hash::dummy)) // FIXME: hack { @@ -29,12 +38,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & while (pos < s.size()) { size_t colon = s.find(':', pos); - if (colon == std::string::npos) throw corrupt("expecting ':'"); + if (colon == s.npos) throw corrupt("expecting ':'"); std::string name(s, pos, colon - pos); size_t eol = s.find('\n', colon + 2); - if (eol == std::string::npos) throw corrupt("expecting '\\n'"); + if (eol == s.npos) throw corrupt("expecting '\\n'"); std::string value(s, colon + 2, eol - colon - 2); @@ -105,10 +114,10 @@ std::string NarInfo::to_string(const Store & store) const assert(compression != ""); res += "Compression: " + compression + "\n"; assert(fileHash && fileHash->type == htSHA256); - res += "FileHash: " + fileHash->to_string(Base32, true) + "\n"; + res += "FileHash: " + fileHash->to_string(HashFormat::Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; assert(narHash.type == htSHA256); - res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; + res += "NarHash: " + narHash.to_string(HashFormat::Base32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; @@ -125,4 +134,59 @@ std::string NarInfo::to_string(const Store & store) const return res; } +nlohmann::json NarInfo::toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const +{ + using nlohmann::json; + + auto jsonObject = ValidPathInfo::toJSON(store, includeImpureInfo, hashFormat); + + if (includeImpureInfo) { + if (!url.empty()) + jsonObject["url"] = url; + if (!compression.empty()) + jsonObject["compression"] = compression; + if (fileHash) + jsonObject["downloadHash"] = fileHash->to_string(hashFormat, true); + if (fileSize) + jsonObject["downloadSize"] = fileSize; + } + + return jsonObject; +} + +NarInfo NarInfo::fromJSON( + const Store & store, + const StorePath & path, + const nlohmann::json & json) +{ + using nlohmann::detail::value_t; + + NarInfo res { + ValidPathInfo { + path, + UnkeyedValidPathInfo::fromJSON(store, json), + } + }; + + if (json.contains("url")) + res.url = ensureType(valueAt(json, "url"), value_t::string); + + if (json.contains("compression")) + res.compression = ensureType(valueAt(json, "compression"), value_t::string); + + if (json.contains("downloadHash")) + res.fileHash = Hash::parseAny( + static_cast( + ensureType(valueAt(json, "downloadHash"), value_t::string)), + std::nullopt); + + if (json.contains("downloadSize")) + res.fileSize = ensureType(valueAt(json, "downloadSize"), value_t::number_integer); + + return res; +} + } diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 5dbdafac3eb..fd538a7cd9f 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -17,14 +17,25 @@ struct NarInfo : ValidPathInfo uint64_t fileSize = 0; NarInfo() = delete; - NarInfo(const Store & store, std::string && name, ContentAddressWithReferences && ca, Hash narHash) + NarInfo(const Store & store, std::string name, ContentAddressWithReferences ca, Hash narHash) : ValidPathInfo(store, std::move(name), std::move(ca), narHash) { } - NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } + NarInfo(StorePath path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); + DECLARE_CMP(NarInfo); + std::string to_string(const Store & store) const; + + nlohmann::json toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const override; + static NarInfo fromJSON( + const Store & store, + const StorePath & path, + const nlohmann::json & json); }; } diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 4a79cf4a197..a4ac413b343 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -1,6 +1,6 @@ -#include "util.hh" #include "local-store.hh" #include "globals.hh" +#include "signals.hh" #include #include @@ -146,10 +146,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ Hash hash = hashPath(htSHA256, path).first; - debug("'%1%' has hash '%2%'", path, hash.to_string(Base32, true)); + debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Base32, true)); /* Check if this is a known hash. */ - Path linkPath = linksDir + "/" + hash.to_string(Base32, false); + Path linkPath = linksDir + "/" + hash.to_string(HashFormat::Base32, false); /* Maybe delete the link, if it has been corrupted. */ if (pathExists(linkPath)) { diff --git a/src/libstore/outputs-spec.cc b/src/libstore/outputs-spec.cc index d943bc11156..21c06922379 100644 --- a/src/libstore/outputs-spec.cc +++ b/src/libstore/outputs-spec.cc @@ -63,7 +63,7 @@ std::optional> ExtendedOutputsS auto specOpt = OutputsSpec::parseOpt(s.substr(found + 1)); if (!specOpt) return std::nullopt; - return std::pair { s.substr(0, found), ExtendedOutputsSpec::Explicit { *std::move(specOpt) } }; + return std::pair { s.substr(0, found), ExtendedOutputsSpec::Explicit { std::move(*specOpt) } }; } diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index cc4a94fab8d..73e55a96ca5 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -122,7 +122,7 @@ bool ParsedDerivation::willBuildLocally(Store & localStore) const bool ParsedDerivation::substitutesAllowed() const { - return getBoolAttr("allowSubstitutes", true); + return settings.alwaysAllowSubstitutes ? true : getBoolAttr("allowSubstitutes", true); } bool ParsedDerivation::useUidRange() const @@ -132,6 +132,41 @@ bool ParsedDerivation::useUidRange() const static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); +/** + * Write a JSON representation of store object metadata, such as the + * hash and the references. + */ +static nlohmann::json pathInfoToJSON( + Store & store, + const StorePathSet & storePaths) +{ + nlohmann::json::array_t jsonList = nlohmann::json::array(); + + for (auto & storePath : storePaths) { + auto info = store.queryPathInfo(storePath); + + auto & jsonPath = jsonList.emplace_back( + info->toJSON(store, false, HashFormat::Base32)); + + // Add the path to the object whose metadata we are including. + jsonPath["path"] = store.printStorePath(storePath); + + jsonPath["valid"] = true; + + jsonPath["closureSize"] = ({ + uint64_t totalNarSize = 0; + StorePathSet closure; + store.computeFSClosure(info->path, closure, false, false); + for (auto & p : closure) { + auto info = store.queryPathInfo(p); + totalNarSize += info->narSize; + } + totalNarSize; + }); + } + return jsonList; +} + std::optional ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths) { auto structuredAttrs = getStructuredAttrs(); @@ -152,8 +187,8 @@ std::optional ParsedDerivation::prepareStructuredAttrs(Store & s StorePathSet storePaths; for (auto & p : *i) storePaths.insert(store.parseStorePath(p.get())); - json[i.key()] = store.pathInfoToJSON( - store.exportReferences(storePaths, inputPaths), false, true); + json[i.key()] = pathInfoToJSON(store, + store.exportReferences(storePaths, inputPaths)); } } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index ccb57104f91..2d7dc972f17 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -1,10 +1,30 @@ +#include + #include "path-info.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" #include "store-api.hh" +#include "json-utils.hh" namespace nix { +GENERATE_CMP_EXT( + , + UnkeyedValidPathInfo, + me->deriver, + me->narHash, + me->references, + me->registrationTime, + me->narSize, + //me->id, + me->ultimate, + me->sigs, + me->ca); + +GENERATE_CMP_EXT( + , + ValidPathInfo, + me->path, + static_cast(*me)); + std::string ValidPathInfo::fingerprint(const Store & store) const { if (narSize == 0) @@ -12,7 +32,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const store.printStorePath(path)); return "1;" + store.printStorePath(path) + ";" - + narHash.to_string(Base32, true) + ";" + + narHash.to_string(HashFormat::Base32, true) + ";" + std::to_string(narSize) + ";" + concatStringsSep(",", store.printStorePathSet(references)); } @@ -99,14 +119,13 @@ Strings ValidPathInfo::shortRefs() const return refs; } - ValidPathInfo::ValidPathInfo( const Store & store, std::string_view name, ContentAddressWithReferences && ca, Hash narHash) - : path(store.makeFixedOutputPathFromCA(name, ca)) - , narHash(narHash) + : UnkeyedValidPathInfo(narHash) + , path(store.makeFixedOutputPathFromCA(name, ca)) { std::visit(overloaded { [this](TextInfo && ti) { @@ -129,48 +148,93 @@ ValidPathInfo::ValidPathInfo( } -ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) +nlohmann::json UnkeyedValidPathInfo::toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const { - return read(source, store, format, store.parseStorePath(readString(source))); -} + using nlohmann::json; -ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format, StorePath && path) -{ - auto deriver = readString(source); - auto narHash = Hash::parseAny(readString(source), htSHA256); - ValidPathInfo info(path, narHash); - if (deriver != "") info.deriver = store.parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(store, - WorkerProto::ReadConn { .from = source }); - source >> info.registrationTime >> info.narSize; - if (format >= 16) { - source >> info.ultimate; - info.sigs = readStrings(source); - info.ca = ContentAddress::parseOpt(readString(source)); + auto jsonObject = json::object(); + + jsonObject["narHash"] = narHash.to_string(hashFormat, true); + jsonObject["narSize"] = narSize; + + { + auto& jsonRefs = (jsonObject["references"] = json::array()); + for (auto & ref : references) + jsonRefs.emplace_back(store.printStorePath(ref)); } - return info; -} + if (ca) + jsonObject["ca"] = renderContentAddress(ca); + + if (includeImpureInfo) { + if (deriver) + jsonObject["deriver"] = store.printStorePath(*deriver); + + if (registrationTime) + jsonObject["registrationTime"] = registrationTime; + + if (ultimate) + jsonObject["ultimate"] = ultimate; -void ValidPathInfo::write( - Sink & sink, + if (!sigs.empty()) { + for (auto & sig : sigs) + jsonObject["signatures"].push_back(sig); + } + } + + return jsonObject; +} + +UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON( const Store & store, - unsigned int format, - bool includePath) const + const nlohmann::json & json) { - if (includePath) - sink << store.printStorePath(path); - sink << (deriver ? store.printStorePath(*deriver) : "") - << narHash.to_string(Base16, false); - WorkerProto::write(store, - WorkerProto::WriteConn { .to = sink }, - references); - sink << registrationTime << narSize; - if (format >= 16) { - sink << ultimate - << sigs - << renderContentAddress(ca); + using nlohmann::detail::value_t; + + UnkeyedValidPathInfo res { + Hash(Hash::dummy), + }; + + ensureType(json, value_t::object); + res.narHash = Hash::parseAny( + static_cast( + ensureType(valueAt(json, "narHash"), value_t::string)), + std::nullopt); + res.narSize = ensureType(valueAt(json, "narSize"), value_t::number_integer); + + try { + auto & references = ensureType(valueAt(json, "references"), value_t::array); + for (auto & input : references) + res.references.insert(store.parseStorePath(static_cast +(input))); + } catch (Error & e) { + e.addTrace({}, "while reading key 'references'"); + throw; } + + if (json.contains("ca")) + res.ca = ContentAddress::parse( + static_cast( + ensureType(valueAt(json, "ca"), value_t::string))); + + if (json.contains("deriver")) + res.deriver = store.parseStorePath( + static_cast( + ensureType(valueAt(json, "deriver"), value_t::string))); + + if (json.contains("registrationTime")) + res.registrationTime = ensureType(valueAt(json, "registrationTime"), value_t::number_integer); + + if (json.contains("ultimate")) + res.ultimate = ensureType(valueAt(json, "ultimate"), value_t::boolean); + + if (json.contains("signatures")) + res.sigs = valueAt(json, "signatures"); + + return res; } } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 22152362206..077abc7e15b 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -29,12 +29,11 @@ struct SubstitutablePathInfo uint64_t narSize; }; -typedef std::map SubstitutablePathInfos; +using SubstitutablePathInfos = std::map; -struct ValidPathInfo +struct UnkeyedValidPathInfo { - StorePath path; std::optional deriver; /** * \todo document this @@ -43,7 +42,7 @@ struct ValidPathInfo StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown - uint64_t id; // internal use only + uint64_t id = 0; // internal use only /** * Whether the path is ultimately trusted, that is, it's a @@ -72,13 +71,31 @@ struct ValidPathInfo */ std::optional ca; - bool operator == (const ValidPathInfo & i) const - { - return - path == i.path - && narHash == i.narHash - && references == i.references; - } + UnkeyedValidPathInfo(const UnkeyedValidPathInfo & other) = default; + + UnkeyedValidPathInfo(Hash narHash) : narHash(narHash) { }; + + DECLARE_CMP(UnkeyedValidPathInfo); + + virtual ~UnkeyedValidPathInfo() { } + + /** + * @param includeImpureInfo If true, variable elements such as the + * registration time are included. + */ + virtual nlohmann::json toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const; + static UnkeyedValidPathInfo fromJSON( + const Store & store, + const nlohmann::json & json); +}; + +struct ValidPathInfo : UnkeyedValidPathInfo { + StorePath path; + + DECLARE_CMP(ValidPathInfo); /** * Return a fingerprint of the store path to be used in binary @@ -92,11 +109,11 @@ struct ValidPathInfo void sign(const Store & store, const SecretKey & secretKey); - /** - * @return The `ContentAddressWithReferences` that determines the - * store path for a content-addressed store object, `std::nullopt` - * for an input-addressed store object. - */ + /** + * @return The `ContentAddressWithReferences` that determines the + * store path for a content-addressed store object, `std::nullopt` + * for an input-addressed store object. + */ std::optional contentAddressWithReferences() const; /** @@ -122,20 +139,15 @@ struct ValidPathInfo ValidPathInfo(const ValidPathInfo & other) = default; - ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; - ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; + ValidPathInfo(StorePath && path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info), path(std::move(path)) { }; + ValidPathInfo(const StorePath & path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info), path(path) { }; ValidPathInfo(const Store & store, std::string_view name, ContentAddressWithReferences && ca, Hash narHash); virtual ~ValidPathInfo() { } - - static ValidPathInfo read(Source & source, const Store & store, unsigned int format); - static ValidPathInfo read(Source & source, const Store & store, unsigned int format, StorePath && path); - - void write(Sink & sink, const Store & store, unsigned int format, bool includePath = true) const; }; -typedef std::map ValidPathInfos; +using ValidPathInfos = std::map; } diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc index 33cf66ce366..274b596c00e 100644 --- a/src/libstore/path-references.cc +++ b/src/libstore/path-references.cc @@ -1,6 +1,5 @@ #include "path-references.hh" #include "hash.hh" -#include "util.hh" #include "archive.hh" #include diff --git a/src/libstore/path-references.hh b/src/libstore/path-references.hh index 7b44e32617f..0553003f83a 100644 --- a/src/libstore/path-references.hh +++ b/src/libstore/path-references.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "references.hh" #include "path.hh" diff --git a/src/libstore/path-regex.hh b/src/libstore/path-regex.hh index 4f8dc4c1faa..a44e6a2eb52 100644 --- a/src/libstore/path-regex.hh +++ b/src/libstore/path-regex.hh @@ -3,6 +3,6 @@ namespace nix { -static constexpr std::string_view nameRegexStr = R"([0-9a-zA-Z\+\-\._\?=]+)"; +static constexpr std::string_view nameRegexStr = R"([0-9a-zA-Z\+\-_\?=][0-9a-zA-Z\+\-\._\?=]*)"; } diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 552e831146a..ec3e53232b3 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -11,6 +11,8 @@ static void checkName(std::string_view path, std::string_view name) if (name.size() > StorePath::MaxPathLen) throw BadStorePath("store path '%s' has a name longer than %d characters", path, StorePath::MaxPathLen); + if (name[0] == '.') + throw BadStorePath("store path '%s' starts with illegal character '.'", path); // See nameRegexStr for the definition for (auto c : name) if (!((c >= '0' && c <= '9') @@ -33,7 +35,7 @@ StorePath::StorePath(std::string_view _baseName) } StorePath::StorePath(const Hash & hash, std::string_view _name) - : baseName((hash.to_string(Base32, false) + "-").append(std::string(_name))) + : baseName((hash.to_string(HashFormat::Base32, false) + "-").append(std::string(_name))) { checkName(baseName, name()); } diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index adc763e6a3c..2b5b8dfe735 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -1,6 +1,7 @@ #include "pathlocks.hh" #include "util.hh" #include "sync.hh" +#include "signals.hh" #include #include diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 4921df352fc..7fcfa2e4087 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "util.hh" +#include "file-descriptor.hh" namespace nix { diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 4e99559481c..e8b88693d26 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -1,7 +1,7 @@ #include "profiles.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "util.hh" +#include "users.hh" #include #include @@ -183,7 +183,7 @@ void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bo iterDropUntil(gens, i, [&](auto & g) { return g.number == curGen; }); // Skip over `max` generations, preserving them - for (auto keep = 0; i != gens.rend() && keep < max; ++i, ++keep); + for (GenerationNumber keep = 0; i != gens.rend() && keep < max; ++i, ++keep); // Delete the rest for (; i != gens.rend(); ++i) diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 559483ce3b4..4ba2123d81c 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -39,7 +39,7 @@ struct DrvOutput { std::string to_string() const; std::string strHash() const - { return drvHash.to_string(Base16, true); } + { return drvHash.to_string(HashFormat::Base16, true); } static DrvOutput parse(const std::string &); diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index fcfb527f50e..03e57a565d0 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -8,8 +8,9 @@ namespace nix { -RemoteFSAccessor::RemoteFSAccessor(ref store, const Path & cacheDir) +RemoteFSAccessor::RemoteFSAccessor(ref store, bool requireValidPath, const Path & cacheDir) : store(store) + , requireValidPath(requireValidPath) , cacheDir(cacheDir) { if (cacheDir != "") @@ -22,7 +23,7 @@ Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::strin return fmt("%s/%s.%s", cacheDir, hashPart, ext); } -ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar) +ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar) { if (cacheDir != "") { try { @@ -38,7 +39,7 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::str if (cacheDir != "") { try { - nlohmann::json j = listNar(narAccessor, "", true); + nlohmann::json j = listNar(narAccessor, CanonPath::root, true); writeFile(makeCacheFile(hashPart, "ls"), j.dump()); } catch (...) { ignoreException(); @@ -48,11 +49,10 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::str return narAccessor; } -std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, bool requireValidPath) +std::pair, CanonPath> RemoteFSAccessor::fetch(const CanonPath & path) { - auto path = canonPath(path_); - - auto [storePath, restPath] = store->toStorePath(path); + auto [storePath, restPath_] = store->toStorePath(path.abs()); + auto restPath = CanonPath(restPath_); if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); @@ -63,7 +63,7 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, boo std::string listing; Path cacheFile; - if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) { + if (cacheDir != "" && nix::pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) { try { listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls")); @@ -101,25 +101,25 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, boo return {addToCache(storePath.hashPart(), std::move(sink.s)), restPath}; } -FSAccessor::Stat RemoteFSAccessor::stat(const Path & path) +std::optional RemoteFSAccessor::maybeLstat(const CanonPath & path) { auto res = fetch(path); - return res.first->stat(res.second); + return res.first->maybeLstat(res.second); } -StringSet RemoteFSAccessor::readDirectory(const Path & path) +SourceAccessor::DirEntries RemoteFSAccessor::readDirectory(const CanonPath & path) { auto res = fetch(path); return res.first->readDirectory(res.second); } -std::string RemoteFSAccessor::readFile(const Path & path, bool requireValidPath) +std::string RemoteFSAccessor::readFile(const CanonPath & path) { - auto res = fetch(path, requireValidPath); + auto res = fetch(path); return res.first->readFile(res.second); } -std::string RemoteFSAccessor::readLink(const Path & path) +std::string RemoteFSAccessor::readLink(const CanonPath & path) { auto res = fetch(path); return res.first->readLink(res.second); diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index e2673b6f6d9..d09762a53c4 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -1,40 +1,43 @@ #pragma once ///@file -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "ref.hh" #include "store-api.hh" namespace nix { -class RemoteFSAccessor : public FSAccessor +class RemoteFSAccessor : public SourceAccessor { ref store; - std::map> nars; + std::map> nars; + + bool requireValidPath; Path cacheDir; - std::pair, Path> fetch(const Path & path_, bool requireValidPath = true); + std::pair, CanonPath> fetch(const CanonPath & path); friend class BinaryCacheStore; Path makeCacheFile(std::string_view hashPart, const std::string & ext); - ref addToCache(std::string_view hashPart, std::string && nar); + ref addToCache(std::string_view hashPart, std::string && nar); public: RemoteFSAccessor(ref store, + bool requireValidPath = true, const /* FIXME: use std::optional */ Path & cacheDir = ""); - Stat stat(const Path & path) override; + std::optional maybeLstat(const CanonPath & path) override; - StringSet readDirectory(const Path & path) override; + DirEntries readDirectory(const CanonPath & path) override; - std::string readFile(const Path & path, bool requireValidPath = true) override; + std::string readFile(const CanonPath & path) override; - std::string readLink(const Path & path) override; + std::string readLink(const CanonPath & path) override; }; } diff --git a/src/libstore/remote-store-connection.hh b/src/libstore/remote-store-connection.hh index ce4740a9c7a..44328b06b55 100644 --- a/src/libstore/remote-store-connection.hh +++ b/src/libstore/remote-store-connection.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "remote-store.hh" #include "worker-protocol.hh" #include "pool.hh" @@ -30,7 +33,7 @@ struct RemoteStore::Connection * sides support. (If the maximum doesn't exist, we would fail to * establish a connection and produce a value of this type.) */ - unsigned int daemonVersion; + WorkerProto::Version daemonVersion; /** * Whether the remote side trusts us or not. @@ -70,6 +73,7 @@ struct RemoteStore::Connection { return WorkerProto::ReadConn { .from = from, + .version = daemonVersion, }; } @@ -85,6 +89,7 @@ struct RemoteStore::Connection { return WorkerProto::WriteConn { .to = to, + .version = daemonVersion, }; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index a639346d1ad..77dc97e3929 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -16,6 +16,8 @@ #include "logging.hh" #include "callback.hh" #include "filetransfer.hh" +#include "signals.hh" + #include namespace nix { @@ -332,7 +334,8 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); } info = std::make_shared( - ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion), StorePath{path})); + StorePath{path}, + WorkerProto::Serialise::read(*this, *conn)); } callback(std::move(info)); } catch (...) { callback.rethrow(); } @@ -445,7 +448,7 @@ ref RemoteStore::addCAToStore( } return make_ref( - ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion))); + WorkerProto::Serialise::read(*this, *conn)); } else { if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); @@ -541,7 +544,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn->to << WorkerProto::Op::AddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash.to_string(HashFormat::Base16, false); WorkerProto::write(*this, *conn, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) @@ -570,7 +573,12 @@ void RemoteStore::addMultipleToStore( auto source = sinkToSource([&](Sink & sink) { sink << pathsToCopy.size(); for (auto & [pathInfo, pathSource] : pathsToCopy) { - pathInfo.write(sink, *this, 16); + WorkerProto::Serialise::write(*this, + WorkerProto::WriteConn { + .to = sink, + .version = 16, + }, + pathInfo); pathSource->drainInto(sink); } }); @@ -655,33 +663,6 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, } catch (...) { return callback.rethrow(); } } -static void writeDerivedPaths(RemoteStore & store, RemoteStore::Connection & conn, const std::vector & reqs) -{ - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 30) { - WorkerProto::write(store, conn, reqs); - } else { - Strings ss; - for (auto & p : reqs) { - auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); - std::visit(overloaded { - [&](const StorePathWithOutputs & s) { - ss.push_back(s.to_string(store)); - }, - [&](const StorePath & drvPath) { - throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", - store.printStorePath(drvPath), - GET_PROTOCOL_MAJOR(conn.daemonVersion), - GET_PROTOCOL_MINOR(conn.daemonVersion)); - }, - [&](std::monostate) { - throw Error("wanted to build a derivation that is itself a build product, but the legacy 'ssh://' protocol doesn't support that. Try using 'ssh-ng://'"); - }, - }, sOrDrvPath); - } - conn.to << ss; - } -} - void RemoteStore::copyDrvsFromEvalStore( const std::vector & paths, std::shared_ptr evalStore) @@ -711,7 +692,7 @@ void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMod auto conn(getConnection()); conn->to << WorkerProto::Op::BuildPaths; assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); - writeDerivedPaths(*this, *conn, drvPaths); + WorkerProto::write(*this, *conn, drvPaths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) conn->to << buildMode; else @@ -735,7 +716,7 @@ std::vector RemoteStore::buildPathsWithResults( if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 34) { conn->to << WorkerProto::Op::BuildPathsWithResults; - writeDerivedPaths(*this, *conn, paths); + WorkerProto::write(*this, *conn, paths); conn->to << buildMode; conn.processStderr(); return WorkerProto::Serialise>::read(*this, *conn); @@ -815,20 +796,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD writeDerivation(conn->to, *this, drv); conn->to << buildMode; conn.processStderr(); - BuildResult res; - res.status = (BuildResult::Status) readInt(conn->from); - conn->from >> res.errorMsg; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { - conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime; - } - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) { - auto builtOutputs = WorkerProto::Serialise::read(*this, *conn); - for (auto && [output, realisation] : builtOutputs) - res.builtOutputs.insert_or_assign( - std::move(output.outputName), - std::move(realisation)); - } - return res; + return WorkerProto::Serialise::read(*this, *conn); } @@ -929,7 +897,7 @@ void RemoteStore::queryMissing(const std::vector & targets, // to prevent a deadlock. goto fallback; conn->to << WorkerProto::Op::QueryMissing; - writeDerivedPaths(*this, *conn, targets); + WorkerProto::write(*this, *conn, targets); conn.processStderr(); willBuild = WorkerProto::Serialise::read(*this, *conn); willSubstitute = WorkerProto::Serialise::read(*this, *conn); @@ -1004,7 +972,7 @@ void RemoteStore::narFromPath(const StorePath & path, Sink & sink) copyNAR(conn->from, sink); } -ref RemoteStore::getFSAccessor() +ref RemoteStore::getFSAccessor(bool requireValidPath) { return make_ref(ref(shared_from_this())); } @@ -1105,6 +1073,7 @@ void RemoteStore::ConnectionHandle::withFramedSink(std::function maxConnections{(StoreConfig*) this, 1, "max-connections", + const Setting maxConnections{this, 1, "max-connections", "Maximum number of concurrent connections to the Nix daemon."}; - const Setting maxConnectionAge{(StoreConfig*) this, + const Setting maxConnectionAge{this, std::numeric_limits::max(), "max-connection-age", "Maximum age of a connection before it is closed."}; @@ -185,7 +185,7 @@ protected: friend struct ConnectionHandle; - virtual ref getFSAccessor() override; + virtual ref getFSAccessor(bool requireValidPath) override; virtual void narFromPath(const StorePath & path, Sink & sink) override; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index d2fc6abafe4..1a62d92d44a 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -193,20 +193,20 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - const Setting profile{(StoreConfig*) this, "", "profile", + const Setting profile{this, "", "profile", R"( The name of the AWS configuration profile to use. By default Nix will use the `default` profile. )"}; - const Setting region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", + const Setting region{this, Aws::Region::US_EAST_1, "region", R"( The region of the S3 bucket. If your bucket is not in `us–east-1`, you should always explicitly specify the region parameter. )"}; - const Setting scheme{(StoreConfig*) this, "", "scheme", + const Setting scheme{this, "", "scheme", R"( The scheme used for S3 requests, `https` (default) or `http`. This option allows you to disable HTTPS for binary caches which don't @@ -218,7 +218,7 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig > information. )"}; - const Setting endpoint{(StoreConfig*) this, "", "endpoint", + const Setting endpoint{this, "", "endpoint", R"( The URL of the endpoint of an S3-compatible service such as MinIO. Do not specify this setting if you're using Amazon S3. @@ -229,13 +229,13 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig > addressing instead of virtual host based addressing. )"}; - const Setting narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", + const Setting narinfoCompression{this, "", "narinfo-compression", "Compression method for `.narinfo` files."}; - const Setting lsCompression{(StoreConfig*) this, "", "ls-compression", + const Setting lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."}; - const Setting logCompression{(StoreConfig*) this, "", "log-compression", + const Setting logCompression{this, "", "log-compression", R"( Compression method for `log/*` files. It is recommended to use a compression method supported by most web browsers @@ -243,11 +243,11 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig )"}; const Setting multipartUpload{ - (StoreConfig*) this, false, "multipart-upload", + this, false, "multipart-upload", "Whether to use multi-part uploads."}; const Setting bufferSize{ - (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", + this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."}; const std::string name() override { return "S3 Binary Cache Store"; } diff --git a/src/libstore/s3-binary-cache-store.md b/src/libstore/s3-binary-cache-store.md index 70fe0eb0966..675470261ec 100644 --- a/src/libstore/s3-binary-cache-store.md +++ b/src/libstore/s3-binary-cache-store.md @@ -2,7 +2,103 @@ R"( **Store URL format**: `s3://`*bucket-name* -This store allows reading and writing a binary cache stored in an AWS -S3 bucket. +This store allows reading and writing a binary cache stored in an AWS S3 (or S3-compatible service) bucket. +This store shares many idioms with the [HTTP Binary Cache Store](#http-binary-cache-store). + +For AWS S3, the binary cache URL for a bucket named `example-nix-cache` will be exactly . +For S3 compatible binary caches, consult that cache's documentation. + +### Anonymous reads to your S3-compatible binary cache + +> If your binary cache is publicly accessible and does not require authentication, +> it is simplest to use the [HTTP Binary Cache Store] rather than S3 Binary Cache Store with +> instead of . + +Your bucket will need a +[bucket policy](https://docs.aws.amazon.com/AmazonS3/v1/userguide/bucket-policies.html) +like the following to be accessible: + +```json +{ + "Id": "DirectReads", + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowDirectReads", + "Action": [ + "s3:GetObject", + "s3:GetBucketLocation" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::example-nix-cache", + "arn:aws:s3:::example-nix-cache/*" + ], + "Principal": "*" + } + ] +} +``` + +### Authentication + +Nix will use the +[default credential provider chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html) +for authenticating requests to Amazon S3. + +Note that this means Nix will read environment variables and files with different idioms than with Nix's own settings, as implemented by the AWS SDK. +Consult the documentation linked above for further details. + +### Authenticated reads to your S3 binary cache + +Your bucket will need a bucket policy allowing the desired users to perform the `s3:GetObject` and `s3:GetBucketLocation` action on all objects in the bucket. +The [anonymous policy given above](#anonymous-reads-to-your-s3-compatible-binary-cache) can be updated to have a restricted `Principal` to support this. + +### Authenticated writes to your S3-compatible binary cache + +Your account will need an IAM policy to support uploading to the bucket: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "UploadToCache", + "Effect": "Allow", + "Action": [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:ListMultipartUploadParts", + "s3:PutObject" + ], + "Resource": [ + "arn:aws:s3:::example-nix-cache", + "arn:aws:s3:::example-nix-cache/*" + ] + } + ] +} +``` + +### Examples + +With bucket policies and authentication set up as described above, uploading works via [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md) (experimental). + +- To upload with a specific credential profile for Amazon S3: + + ```console + $ nix copy nixpkgs.hello \ + --to 's3://example-nix-cache?profile=cache-upload®ion=eu-west-2' + ``` + +- To upload to an S3-compatible binary cache: + + ```console + $ nix copy nixpkgs.hello --to \ + 's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com' + ``` )" diff --git a/src/libstore/serve-protocol-impl.hh b/src/libstore/serve-protocol-impl.hh new file mode 100644 index 00000000000..a3ce8102609 --- /dev/null +++ b/src/libstore/serve-protocol-impl.hh @@ -0,0 +1,59 @@ +#pragma once +/** + * @file + * + * Template implementations (as opposed to mere declarations). + * + * This file is an exmample of the "impl.hh" pattern. See the + * contributing guide. + */ + +#include "serve-protocol.hh" +#include "length-prefixed-protocol-helper.hh" + +namespace nix { + +/* protocol-agnostic templates */ + +#define SERVE_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ + TEMPLATE T ServeProto::Serialise< T >::read(const Store & store, ServeProto::ReadConn conn) \ + { \ + return LengthPrefixedProtoHelper::read(store, conn); \ + } \ + TEMPLATE void ServeProto::Serialise< T >::write(const Store & store, ServeProto::WriteConn conn, const T & t) \ + { \ + LengthPrefixedProtoHelper::write(store, conn, t); \ + } + +SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) + +#define COMMA_ , +SERVE_USE_LENGTH_PREFIX_SERIALISER( + template, + std::map) +#undef COMMA_ + +/** + * Use `CommonProto` where possible. + */ +template +struct ServeProto::Serialise +{ + static T read(const Store & store, ServeProto::ReadConn conn) + { + return CommonProto::Serialise::read(store, + CommonProto::ReadConn { .from = conn.from }); + } + static void write(const Store & store, ServeProto::WriteConn conn, const T & t) + { + CommonProto::Serialise::write(store, + CommonProto::WriteConn { .to = conn.to }, + t); + } +}; + +/* protocol-specific templates */ + +} diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc new file mode 100644 index 00000000000..9bfcc279cba --- /dev/null +++ b/src/libstore/serve-protocol.cc @@ -0,0 +1,57 @@ +#include "serialise.hh" +#include "path-with-outputs.hh" +#include "store-api.hh" +#include "build-result.hh" +#include "serve-protocol.hh" +#include "serve-protocol-impl.hh" +#include "archive.hh" + +#include + +namespace nix { + +/* protocol-specific definitions */ + +BuildResult ServeProto::Serialise::read(const Store & store, ServeProto::ReadConn conn) +{ + BuildResult status; + status.status = (BuildResult::Status) readInt(conn.from); + conn.from >> status.errorMsg; + + if (GET_PROTOCOL_MINOR(conn.version) >= 3) + conn.from + >> status.timesBuilt + >> status.isNonDeterministic + >> status.startTime + >> status.stopTime; + if (GET_PROTOCOL_MINOR(conn.version) >= 6) { + auto builtOutputs = ServeProto::Serialise::read(store, conn); + for (auto && [output, realisation] : builtOutputs) + status.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); + } + return status; +} + +void ServeProto::Serialise::write(const Store & store, ServeProto::WriteConn conn, const BuildResult & status) +{ + conn.to + << status.status + << status.errorMsg; + + if (GET_PROTOCOL_MINOR(conn.version) >= 3) + conn.to + << status.timesBuilt + << status.isNonDeterministic + << status.startTime + << status.stopTime; + if (GET_PROTOCOL_MINOR(conn.version) >= 6) { + DrvOutputs builtOutputs; + for (auto & [output, realisation] : status.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + ServeProto::write(store, conn, builtOutputs); + } +} + +} diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index 7e43b396991..ba159f6e9ab 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include "common-protocol.hh" + namespace nix { #define SERVE_MAGIC_1 0x390c9deb @@ -10,6 +12,14 @@ namespace nix { #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) + +class Store; +struct Source; + +// items being serialised +struct BuildResult; + + /** * The "serve protocol", used by ssh:// stores. * @@ -22,6 +32,60 @@ struct ServeProto * Enumeration of all the request types for the protocol. */ enum struct Command : uint64_t; + + /** + * Version type for the protocol. + * + * @todo Convert to struct with separate major vs minor fields. + */ + using Version = unsigned int; + + /** + * A unidirectional read connection, to be used by the read half of the + * canonical serializers below. + */ + struct ReadConn { + Source & from; + Version version; + }; + + /** + * A unidirectional write connection, to be used by the write half of the + * canonical serializers below. + */ + struct WriteConn { + Sink & to; + Version version; + }; + + /** + * Data type for canonical pairs of serialisers for the serve protocol. + * + * See https://en.cppreference.com/w/cpp/language/adl for the broader + * concept of what is going on here. + */ + template + struct Serialise; + // This is the definition of `Serialise` we *want* to put here, but + // do not do so. + // + // See `worker-protocol.hh` for a longer explanation. +#if 0 + { + static T read(const Store & store, ReadConn conn); + static void write(const Store & store, WriteConn conn, const T & t); + }; +#endif + + /** + * Wrapper function around `ServeProto::Serialise::write` that allows us to + * infer the type instead of having to write it down explicitly. + */ + template + static void write(const Store & store, WriteConn conn, const T & t) + { + ServeProto::Serialise::write(store, conn, t); + } }; enum struct ServeProto::Command : uint64_t @@ -58,4 +122,36 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op) return s << (uint64_t) op; } +/** + * Declare a canonical serialiser pair for the worker protocol. + * + * We specialise the struct merely to indicate that we are implementing + * the function for the given type. + * + * Some sort of `template<...>` must be used with the caller for this to + * be legal specialization syntax. See below for what that looks like in + * practice. + */ +#define DECLARE_SERVE_SERIALISER(T) \ + struct ServeProto::Serialise< T > \ + { \ + static T read(const Store & store, ServeProto::ReadConn conn); \ + static void write(const Store & store, ServeProto::WriteConn conn, const T & t); \ + }; + +template<> +DECLARE_SERVE_SERIALISER(BuildResult); + +template +DECLARE_SERVE_SERIALISER(std::vector); +template +DECLARE_SERVE_SERIALISER(std::set); +template +DECLARE_SERVE_SERIALISER(std::tuple); + +#define COMMA_ , +template +DECLARE_SERVE_SERIALISER(std::map); +#undef COMMA_ + } diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 7c8decb7430..d7432a3059e 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -2,6 +2,7 @@ #include "globals.hh" #include "util.hh" #include "url.hh" +#include "signals.hh" #include diff --git a/src/libstore/ssh-store-config.hh b/src/libstore/ssh-store-config.hh index c27a5d00fc9..bf55d20cf4e 100644 --- a/src/libstore/ssh-store-config.hh +++ b/src/libstore/ssh-store-config.hh @@ -9,16 +9,16 @@ struct CommonSSHStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting sshKey{(StoreConfig*) this, "", "ssh-key", + const Setting sshKey{this, "", "ssh-key", "Path to the SSH private key used to authenticate to the remote machine."}; - const Setting sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", + const Setting sshPublicHostKey{this, "", "base64-ssh-public-host-key", "The public host key of the remote machine."}; - const Setting compress{(StoreConfig*) this, false, "compress", + const Setting compress{this, false, "compress", "Whether to enable SSH compression."}; - const Setting remoteStore{(StoreConfig*) this, "", "remote-store", + const Setting remoteStore{this, "", "remote-store", R"( [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) to be used on the remote machine. The default is `auto` diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 9c6c42ef41f..4a6aad44979 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -16,7 +16,7 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig using RemoteStoreConfig::RemoteStoreConfig; using CommonSSHStoreConfig::CommonSSHStoreConfig; - const Setting remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", + const Setting remoteProgram{this, "nix-daemon", "remote-program", "Path to the `nix-daemon` executable on the remote machine."}; const std::string name() override { return "Experimental SSH Store"; } diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index da32f1b79f1..5c8d6a5042b 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -1,5 +1,8 @@ #include "ssh.hh" #include "finally.hh" +#include "current-process.hh" +#include "environment-variables.hh" +#include "util.hh" namespace nix { @@ -111,8 +114,10 @@ std::unique_ptr SSHMaster::startCommand(const std::string reply = readLine(out.readSide.get()); } catch (EndOfFile & e) { } - if (reply != "started") + if (reply != "started") { + printTalkative("SSH stdout first line: %s", reply); throw Error("failed to start SSH connection to '%s'", host); + } } conn->out = std::move(out.readSide); @@ -129,7 +134,6 @@ Path SSHMaster::startMaster() if (state->sshMaster != -1) return state->socketPath; - state->socketPath = (Path) *state->tmpDir + "/ssh.sock"; Pipe out; @@ -141,7 +145,8 @@ Path SSHMaster::startMaster() logger->pause(); Finally cleanup = [&]() { logger->resume(); }; - bool wasMasterRunning = isMasterRunning(); + if (isMasterRunning()) + return state->socketPath; state->sshMaster = startProcess([&]() { restoreProcessContext(); @@ -162,14 +167,14 @@ Path SSHMaster::startMaster() out.writeSide = -1; - if (!wasMasterRunning) { - std::string reply; - try { - reply = readLine(out.readSide.get()); - } catch (EndOfFile & e) { } + std::string reply; + try { + reply = readLine(out.readSide.get()); + } catch (EndOfFile & e) { } - if (reply != "started") - throw Error("failed to start SSH master connection to '%s'", host); + if (reply != "started") { + printTalkative("SSH master stdout first line: %s", reply); + throw Error("failed to start SSH master connection to '%s'", host); } return state->socketPath; diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index 94b952af9c0..bfcd6f21cea 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -1,8 +1,9 @@ #pragma once ///@file -#include "util.hh" #include "sync.hh" +#include "processes.hh" +#include "file-system.hh" namespace nix { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 28689e100e2..fe1903ec680 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,5 +1,5 @@ #include "crypto.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "globals.hh" #include "derivations.hh" #include "store-api.hh" @@ -11,6 +11,11 @@ #include "archive.hh" #include "callback.hh" #include "remote-store.hh" +// FIXME this should not be here, see TODO below on +// `addMultipleToStore`. +#include "worker-protocol.hh" +#include "signals.hh" +#include "users.hh" #include #include @@ -154,7 +159,7 @@ StorePath Store::makeStorePath(std::string_view type, StorePath Store::makeStorePath(std::string_view type, const Hash & hash, std::string_view name) const { - return makeStorePath(type, hash.to_string(Base16, true), name); + return makeStorePath(type, hash.to_string(HashFormat::Base16, true), name); } @@ -192,7 +197,7 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf hashString(htSHA256, "fixed:out:" + makeFileIngestionPrefix(info.method) - + info.hash.to_string(Base16, true) + ":"), + + info.hash.to_string(HashFormat::Base16, true) + ":"), name); } } @@ -225,12 +230,16 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA } -std::pair Store::computeStorePathForPath(std::string_view name, - const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const +std::pair Store::computeStorePathFromDump( + Source & dump, + std::string_view name, + FileIngestionMethod method, + HashType hashAlgo, + const StorePathSet & references) const { - Hash h = method == FileIngestionMethod::Recursive - ? hashPath(hashAlgo, srcPath, filter).first - : hashFile(hashAlgo, srcPath); + HashSink sink(hashAlgo); + dump.drainInto(sink); + auto h = sink.finish().first; FixedOutputInfo caInfo { .method = method, .hash = h, @@ -357,7 +366,13 @@ void Store::addMultipleToStore( { auto expected = readNum(source); for (uint64_t i = 0; i < expected; ++i) { - auto info = ValidPathInfo::read(source, *this, 16); + // FIXME we should not be using the worker protocol here, let + // alone the worker protocol with a hard-coded version! + auto info = WorkerProto::Serialise::read(*this, + WorkerProto::ReadConn { + .from = source, + .version = 16, + }); info.ultimate = false; addToStore(info, source, repair, checkSigs); } @@ -397,7 +412,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, /* Note that fileSink and unusualHashTee must be mutually exclusive, since they both write to caHashSink. Note that that requisite is currently true because the former is only used in the flat case. */ - RetrieveRegularNARSink fileSink { caHashSink }; + RegularFileSink fileSink { caHashSink }; TeeSink unusualHashTee { narHashSink, caHashSink }; auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256 @@ -415,10 +430,10 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, information to narSink. */ TeeSource tapped { *fileSource, narSink }; - ParseSink blank; + NullParseSink blank; auto & parseSink = method == FileIngestionMethod::Flat - ? fileSink - : blank; + ? (ParseSink &) fileSink + : (ParseSink &) blank; /* The information that flows from tapped (besides being replicated in narSink), is now put in parseSink. */ @@ -530,8 +545,8 @@ std::map> Store::queryPartialDerivationOut return outputs; } -OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) { - auto resp = queryPartialDerivationOutputMap(path); +OutputPathMap Store::queryDerivationOutputMap(const StorePath & path, Store * evalStore) { + auto resp = queryPartialDerivationOutputMap(path, evalStore); OutputPathMap result; for (auto & [outName, optOutPath] : resp) { if (!optOutPath) @@ -806,7 +821,7 @@ void Store::substitutePaths(const StorePathSet & paths) std::vector paths2; for (auto & path : paths) if (!path.isDerivation()) - paths2.push_back(DerivedPath::Opaque{path}); + paths2.emplace_back(DerivedPath::Opaque{path}); uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; queryMissing(paths2, @@ -884,7 +899,7 @@ std::string Store::makeValidityRegistration(const StorePathSet & paths, auto info = queryPathInfo(i); if (showHash) { - s += info->narHash.to_string(Base16, false) + "\n"; + s += info->narHash.to_string(HashFormat::Base16, false) + "\n"; s += fmt("%1%\n", info->narSize); } @@ -936,96 +951,6 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor return paths; } -json Store::pathInfoToJSON(const StorePathSet & storePaths, - bool includeImpureInfo, bool showClosureSize, - Base hashBase, - AllowInvalidFlag allowInvalid) -{ - json::array_t jsonList = json::array(); - - for (auto & storePath : storePaths) { - auto& jsonPath = jsonList.emplace_back(json::object()); - - try { - auto info = queryPathInfo(storePath); - - jsonPath["path"] = printStorePath(info->path); - jsonPath["valid"] = true; - jsonPath["narHash"] = info->narHash.to_string(hashBase, true); - jsonPath["narSize"] = info->narSize; - - { - auto& jsonRefs = (jsonPath["references"] = json::array()); - for (auto & ref : info->references) - jsonRefs.emplace_back(printStorePath(ref)); - } - - if (info->ca) - jsonPath["ca"] = renderContentAddress(info->ca); - - std::pair closureSizes; - - if (showClosureSize) { - closureSizes = getClosureSize(info->path); - jsonPath["closureSize"] = closureSizes.first; - } - - if (includeImpureInfo) { - - if (info->deriver) - jsonPath["deriver"] = printStorePath(*info->deriver); - - if (info->registrationTime) - jsonPath["registrationTime"] = info->registrationTime; - - if (info->ultimate) - jsonPath["ultimate"] = info->ultimate; - - if (!info->sigs.empty()) { - for (auto & sig : info->sigs) - jsonPath["signatures"].push_back(sig); - } - - auto narInfo = std::dynamic_pointer_cast( - std::shared_ptr(info)); - - if (narInfo) { - if (!narInfo->url.empty()) - jsonPath["url"] = narInfo->url; - if (narInfo->fileHash) - jsonPath["downloadHash"] = narInfo->fileHash->to_string(hashBase, true); - if (narInfo->fileSize) - jsonPath["downloadSize"] = narInfo->fileSize; - if (showClosureSize) - jsonPath["closureDownloadSize"] = closureSizes.second; - } - } - - } catch (InvalidPath &) { - jsonPath["path"] = printStorePath(storePath); - jsonPath["valid"] = false; - } - } - return jsonList; -} - - -std::pair Store::getClosureSize(const StorePath & storePath) -{ - uint64_t totalNarSize = 0, totalDownloadSize = 0; - StorePathSet closure; - computeFSClosure(storePath, closure, false, false); - for (auto & p : closure) { - auto info = queryPathInfo(p); - totalNarSize += info->narSize; - auto narInfo = std::dynamic_pointer_cast( - std::shared_ptr(info)); - if (narInfo) - totalDownloadSize += narInfo->fileSize; - } - return {totalNarSize, totalDownloadSize}; -} - const Store::Stats & Store::getStats() { @@ -1325,12 +1250,12 @@ Derivation Store::derivationFromPath(const StorePath & drvPath) return readDerivation(drvPath); } -Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool requireValidPath) +static Derivation readDerivationCommon(Store & store, const StorePath & drvPath, bool requireValidPath) { - auto accessor = store.getFSAccessor(); + auto accessor = store.getFSAccessor(requireValidPath); try { return parseDerivation(store, - accessor->readFile(store.printStorePath(drvPath), requireValidPath), + accessor->readFile(CanonPath(store.printStorePath(drvPath))), Derivation::nameFromPath(drvPath)); } catch (FormatError & e) { throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg()); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index da1a3eefbbc..a58dd15354c 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -70,7 +70,7 @@ MakeError(InvalidStoreURI, Error); struct BasicDerivation; struct Derivation; -class FSAccessor; +struct SourceAccessor; class NarInfoDiskCache; class Store; @@ -80,7 +80,6 @@ typedef std::map OutputPathMap; enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; -enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; /** * Magic header of exportPath() output (obsolete). @@ -153,19 +152,22 @@ struct StoreConfig : public Config Setting priority{this, 0, "priority", R"( - Priority of this store when used as a substituter. A lower value means a higher priority. + Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). + A lower value means a higher priority. )"}; Setting wantMassQuery{this, false, "want-mass-query", R"( - Whether this store (when used as a substituter) can be - queried efficiently for path validity. + Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). )"}; Setting systemFeatures{this, getDefaultSystemFeatures(), "system-features", - "Optional features that the system this store builds on implements (like \"kvm\")."}; + R"( + Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. + Example: `"kvm"` + )" }; }; class Store : public std::enable_shared_from_this, public virtual StoreConfig @@ -289,14 +291,15 @@ public: StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; /** - * Preparatory part of addToStore(). - * - * @return the store path to which srcPath is to be copied - * and the cryptographic hash of the contents of srcPath. + * Read-only variant of addToStoreFromDump(). It returns the store + * path to which a NAR or flat file would be written. */ - std::pair computeStorePathForPath(std::string_view name, - const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, - HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; + std::pair computeStorePathFromDump( + Source & dump, + std::string_view name, + FileIngestionMethod method = FileIngestionMethod::Recursive, + HashType hashAlgo = htSHA256, + const StorePathSet & references = {}) const; /** * Preparatory part of addTextToStore(). @@ -457,7 +460,7 @@ public: * Query the mapping outputName=>outputPath for the given derivation. * Assume every output has a mapping and throw an exception otherwise. */ - OutputPathMap queryDerivationOutputMap(const StorePath & path); + OutputPathMap queryDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr); /** * Query the full store path given the hash part of a valid store @@ -661,28 +664,6 @@ public: std::string makeValidityRegistration(const StorePathSet & paths, bool showDerivers, bool showHash); - /** - * Write a JSON representation of store path metadata, such as the - * hash and the references. - * - * @param includeImpureInfo If true, variable elements such as the - * registration time are included. - * - * @param showClosureSize If true, the closure size of each path is - * included. - */ - nlohmann::json pathInfoToJSON(const StorePathSet & storePaths, - bool includeImpureInfo, bool showClosureSize, - Base hashBase = Base32, - AllowInvalidFlag allowInvalid = DisallowInvalid); - - /** - * @return the size of the closure of the specified path, that is, - * the sum of the size of the NAR serialisation of each path in the - * closure. - */ - std::pair getClosureSize(const StorePath & storePath); - /** * Optimise the disk space usage of the Nix store by hard-linking files * with the same contents. @@ -699,7 +680,7 @@ public: /** * @return An object to access files in the Nix store. */ - virtual ref getFSAccessor() = 0; + virtual ref getFSAccessor(bool requireValidPath = true) = 0; /** * Repair the contents of the given path by redownloading it using diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc deleted file mode 100644 index c360c9707f4..00000000000 --- a/src/libstore/tests/derivation.cc +++ /dev/null @@ -1,369 +0,0 @@ -#include -#include - -#include "experimental-features.hh" -#include "derivations.hh" - -#include "tests/libstore.hh" - -namespace nix { - -class DerivationTest : public LibStoreTest -{ -public: - /** - * We set these in tests rather than the regular globals so we don't have - * to worry about race conditions if the tests run concurrently. - */ - ExperimentalFeatureSettings mockXpSettings; -}; - -class CaDerivationTest : public DerivationTest -{ - void SetUp() override - { - mockXpSettings.set("experimental-features", "ca-derivations"); - } -}; - -class DynDerivationTest : public DerivationTest -{ - void SetUp() override - { - mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); - } -}; - -class ImpureDerivationTest : public DerivationTest -{ - void SetUp() override - { - mockXpSettings.set("experimental-features", "impure-derivations"); - } -}; - -TEST_F(DerivationTest, BadATerm_version) { - ASSERT_THROW( - parseDerivation( - *store, - R"(DrvWithVersion("invalid-version",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", - "whatever", - mockXpSettings), - FormatError); -} - -TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) { - ASSERT_THROW( - parseDerivation( - *store, - R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", - "dyn-dep-derivation", - mockXpSettings), - FormatError); -} - -#define TEST_JSON(FIXTURE, NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \ - TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - STR ## _json, \ - (DerivationOutput { VAL }).toJSON( \ - *store, \ - DRV_NAME, \ - OUTPUT_NAME)); \ - } \ - \ - TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - DerivationOutput { VAL }, \ - DerivationOutput::fromJSON( \ - *store, \ - DRV_NAME, \ - OUTPUT_NAME, \ - STR ## _json, \ - mockXpSettings)); \ - } - -TEST_JSON(DerivationTest, inputAddressed, - R"({ - "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" - })", - (DerivationOutput::InputAddressed { - .path = store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"), - }), - "drv-name", "output-name") - -TEST_JSON(DerivationTest, caFixedFlat, - R"({ - "hashAlgo": "sha256", - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name" - })", - (DerivationOutput::CAFixed { - .ca = { - .method = FileIngestionMethod::Flat, - .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), - }, - }), - "drv-name", "output-name") - -TEST_JSON(DerivationTest, caFixedNAR, - R"({ - "hashAlgo": "r:sha256", - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" - })", - (DerivationOutput::CAFixed { - .ca = { - .method = FileIngestionMethod::Recursive, - .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), - }, - }), - "drv-name", "output-name") - -TEST_JSON(DynDerivationTest, caFixedText, - R"({ - "hashAlgo": "text:sha256", - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" - })", - (DerivationOutput::CAFixed { - .ca = { - .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), - }, - }), - "drv-name", "output-name") - -TEST_JSON(CaDerivationTest, caFloating, - R"({ - "hashAlgo": "r:sha256" - })", - (DerivationOutput::CAFloating { - .method = FileIngestionMethod::Recursive, - .hashType = htSHA256, - }), - "drv-name", "output-name") - -TEST_JSON(DerivationTest, deferred, - R"({ })", - DerivationOutput::Deferred { }, - "drv-name", "output-name") - -TEST_JSON(ImpureDerivationTest, impure, - R"({ - "hashAlgo": "r:sha256", - "impure": true - })", - (DerivationOutput::Impure { - .method = FileIngestionMethod::Recursive, - .hashType = htSHA256, - }), - "drv-name", "output-name") - -#undef TEST_JSON - -#define TEST_JSON(FIXTURE, NAME, STR, VAL) \ - TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - STR ## _json, \ - (VAL).toJSON(*store)); \ - } \ - \ - TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - (VAL), \ - Derivation::fromJSON( \ - *store, \ - STR ## _json, \ - mockXpSettings)); \ - } - -#define TEST_ATERM(FIXTURE, NAME, STR, VAL, DRV_NAME) \ - TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) { \ - ASSERT_EQ( \ - STR, \ - (VAL).unparse(*store, false)); \ - } \ - \ - TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) { \ - auto parsed = parseDerivation( \ - *store, \ - STR, \ - DRV_NAME, \ - mockXpSettings); \ - ASSERT_EQ( \ - (VAL).toJSON(*store), \ - parsed.toJSON(*store)); \ - ASSERT_EQ( \ - (VAL), \ - parsed); \ - } - -Derivation makeSimpleDrv(const Store & store) { - Derivation drv; - drv.name = "simple-derivation"; - drv.inputSrcs = { - store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), - }; - drv.inputDrvs = { - .map = { - { - store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), - { - .value = { - "cat", - "dog", - }, - }, - }, - }, - }; - drv.platform = "wasm-sel4"; - drv.builder = "foo"; - drv.args = { - "bar", - "baz", - }; - drv.env = { - { - "BIG_BAD", - "WOLF", - }, - }; - return drv; -} - -TEST_JSON(DerivationTest, simple, - R"({ - "name": "simple-derivation", - "inputSrcs": [ - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" - ], - "inputDrvs": { - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { - "dynamicOutputs": {}, - "outputs": [ - "cat", - "dog" - ] - } - }, - "system": "wasm-sel4", - "builder": "foo", - "args": [ - "bar", - "baz" - ], - "env": { - "BIG_BAD": "WOLF" - }, - "outputs": {} - })", - makeSimpleDrv(*store)) - -TEST_ATERM(DerivationTest, simple, - R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", - makeSimpleDrv(*store), - "simple-derivation") - -Derivation makeDynDepDerivation(const Store & store) { - Derivation drv; - drv.name = "dyn-dep-derivation"; - drv.inputSrcs = { - store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), - }; - drv.inputDrvs = { - .map = { - { - store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), - DerivedPathMap::ChildNode { - .value = { - "cat", - "dog", - }, - .childMap = { - { - "cat", - DerivedPathMap::ChildNode { - .value = { - "kitten", - }, - }, - }, - { - "goose", - DerivedPathMap::ChildNode { - .value = { - "gosling", - }, - }, - }, - }, - }, - }, - }, - }; - drv.platform = "wasm-sel4"; - drv.builder = "foo"; - drv.args = { - "bar", - "baz", - }; - drv.env = { - { - "BIG_BAD", - "WOLF", - }, - }; - return drv; -} - -TEST_JSON(DynDerivationTest, dynDerivationDeps, - R"({ - "name": "dyn-dep-derivation", - "inputSrcs": [ - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" - ], - "inputDrvs": { - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { - "dynamicOutputs": { - "cat": { - "dynamicOutputs": {}, - "outputs": ["kitten"] - }, - "goose": { - "dynamicOutputs": {}, - "outputs": ["gosling"] - } - }, - "outputs": [ - "cat", - "dog" - ] - } - }, - "system": "wasm-sel4", - "builder": "foo", - "args": [ - "bar", - "baz" - ], - "env": { - "BIG_BAD": "WOLF" - }, - "outputs": {} - })", - makeDynDepDerivation(*store)) - -TEST_ATERM(DynDerivationTest, dynDerivationDeps, - R"(DrvWithVersion("xp-dyn-drv",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", - makeDynDepDerivation(*store), - "dyn-dep-derivation") - -#undef TEST_JSON -#undef TEST_ATERM - -} diff --git a/src/libstore/tests/local.mk b/src/libstore/tests/local.mk deleted file mode 100644 index 03becc7d11b..00000000000 --- a/src/libstore/tests/local.mk +++ /dev/null @@ -1,29 +0,0 @@ -check: libstore-tests-exe_RUN - -programs += libstore-tests-exe - -libstore-tests-exe_NAME = libnixstore-tests - -libstore-tests-exe_DIR := $(d) - -libstore-tests-exe_INSTALL_DIR := - -libstore-tests-exe_LIBS = libstore-tests - -libstore-tests-exe_LDFLAGS := $(GTEST_LIBS) - -libraries += libstore-tests - -libstore-tests_NAME = libnixstore-tests - -libstore-tests_DIR := $(d) - -libstore-tests_INSTALL_DIR := - -libstore-tests_SOURCES := $(wildcard $(d)/*.cc) - -libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil - -libstore-tests_LIBS = libutil-tests libstore libutil - -libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 99589f8b2d9..226cdf7175c 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -1,4 +1,5 @@ #include "uds-remote-store.hh" +#include "unix-domain-socket.hh" #include "worker-protocol.hh" #include diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index cdb28a001cf..a5ac9080ad1 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -35,8 +35,8 @@ public: static std::set uriSchemes() { return {"unix"}; } - ref getFSAccessor() override - { return LocalFSStore::getFSAccessor(); } + ref getFSAccessor(bool requireValidPath) override + { return LocalFSStore::getFSAccessor(requireValidPath); } void narFromPath(const StorePath & path, Sink & sink) override { LocalFSStore::narFromPath(path, sink); } diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh index d3d2792ff52..c043588d677 100644 --- a/src/libstore/worker-protocol-impl.hh +++ b/src/libstore/worker-protocol-impl.hh @@ -9,70 +9,51 @@ */ #include "worker-protocol.hh" +#include "length-prefixed-protocol-helper.hh" namespace nix { -template -std::vector WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - std::vector resSet; - auto size = readNum(conn.from); - while (size--) { - resSet.push_back(WorkerProto::Serialise::read(store, conn)); - } - return resSet; -} +/* protocol-agnostic templates */ -template -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::vector & resSet) -{ - conn.to << resSet.size(); - for (auto & key : resSet) { - WorkerProto::Serialise::write(store, conn, key); +#define WORKER_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ + TEMPLATE T WorkerProto::Serialise< T >::read(const Store & store, WorkerProto::ReadConn conn) \ + { \ + return LengthPrefixedProtoHelper::read(store, conn); \ + } \ + TEMPLATE void WorkerProto::Serialise< T >::write(const Store & store, WorkerProto::WriteConn conn, const T & t) \ + { \ + LengthPrefixedProtoHelper::write(store, conn, t); \ } -} -template -std::set WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - std::set resSet; - auto size = readNum(conn.from); - while (size--) { - resSet.insert(WorkerProto::Serialise::read(store, conn)); - } - return resSet; -} +WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) +#define COMMA_ , +WORKER_USE_LENGTH_PREFIX_SERIALISER( + template, + std::map) +#undef COMMA_ + +/** + * Use `CommonProto` where possible. + */ template -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::set & resSet) +struct WorkerProto::Serialise { - conn.to << resSet.size(); - for (auto & key : resSet) { - WorkerProto::Serialise::write(store, conn, key); + static T read(const Store & store, WorkerProto::ReadConn conn) + { + return CommonProto::Serialise::read(store, + CommonProto::ReadConn { .from = conn.from }); } -} - -template -std::map WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - std::map resMap; - auto size = readNum(conn.from); - while (size--) { - auto k = WorkerProto::Serialise::read(store, conn); - auto v = WorkerProto::Serialise::read(store, conn); - resMap.insert_or_assign(std::move(k), std::move(v)); + static void write(const Store & store, WorkerProto::WriteConn conn, const T & t) + { + CommonProto::Serialise::write(store, + CommonProto::WriteConn { .to = conn.to }, + t); } - return resMap; -} +}; -template -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::map & resMap) -{ - conn.to << resMap.size(); - for (auto & i : resMap) { - WorkerProto::Serialise::write(store, conn, i.first); - WorkerProto::Serialise::write(store, conn, i.second); - } -} +/* protocol-specific templates */ } diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index a23130743ee..7118558b1ab 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -1,38 +1,17 @@ #include "serialise.hh" -#include "util.hh" #include "path-with-outputs.hh" #include "store-api.hh" #include "build-result.hh" #include "worker-protocol.hh" #include "worker-protocol-impl.hh" #include "archive.hh" -#include "derivations.hh" +#include "path-info.hh" #include namespace nix { -std::string WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return readString(conn.from); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const std::string & str) -{ - conn.to << str; -} - - -StorePath WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return store.parseStorePath(readString(conn.from)); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const StorePath & storePath) -{ - conn.to << store.printStorePath(storePath); -} - +/* protocol-specific definitions */ std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) { @@ -52,14 +31,14 @@ std::optional WorkerProto::Serialise>::r void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional & optTrusted) { if (!optTrusted) - conn.to << (uint8_t)0; + conn.to << uint8_t{0}; else { switch (*optTrusted) { case Trusted: - conn.to << (uint8_t)1; + conn.to << uint8_t{1}; break; case NotTrusted: - conn.to << (uint8_t)2; + conn.to << uint8_t{2}; break; default: assert(false); @@ -68,52 +47,37 @@ void WorkerProto::Serialise>::write(const Store & sto } -ContentAddress WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return ContentAddress::parse(readString(conn.from)); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const ContentAddress & ca) -{ - conn.to << renderContentAddress(ca); -} - - DerivedPath WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { auto s = readString(conn.from); - return DerivedPath::parseLegacy(store, s); + if (GET_PROTOCOL_MINOR(conn.version) >= 30) { + return DerivedPath::parseLegacy(store, s); + } else { + return parsePathWithOutputs(store, s).toDerivedPath(); + } } void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const DerivedPath & req) { - conn.to << req.to_string_legacy(store); -} - - -Realisation WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - std::string rawInput = readString(conn.from); - return Realisation::fromJSON( - nlohmann::json::parse(rawInput), - "remote-protocol" - ); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const Realisation & realisation) -{ - conn.to << realisation.toJSON().dump(); -} - - -DrvOutput WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return DrvOutput::parse(readString(conn.from)); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const DrvOutput & drvOutput) -{ - conn.to << drvOutput.to_string(); + if (GET_PROTOCOL_MINOR(conn.version) >= 30) { + conn.to << req.to_string_legacy(store); + } else { + auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(req); + std::visit(overloaded { + [&](const StorePathWithOutputs & s) { + conn.to << s.to_string(store); + }, + [&](const StorePath & drvPath) { + throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", + store.printStorePath(drvPath), + GET_PROTOCOL_MAJOR(conn.version), + GET_PROTOCOL_MINOR(conn.version)); + }, + [&](std::monostate) { + throw Error("wanted to build a derivation that is itself a build product, but protocols do not support that. Try upgrading the Nix on the other end of this connection"); + }, + }, sOrDrvPath); + } } @@ -137,18 +101,22 @@ void WorkerProto::Serialise::write(const Store & store, Worker BuildResult WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { BuildResult res; - res.status = (BuildResult::Status) readInt(conn.from); - conn.from - >> res.errorMsg - >> res.timesBuilt - >> res.isNonDeterministic - >> res.startTime - >> res.stopTime; - auto builtOutputs = WorkerProto::Serialise::read(store, conn); - for (auto && [output, realisation] : builtOutputs) - res.builtOutputs.insert_or_assign( - std::move(output.outputName), - std::move(realisation)); + res.status = static_cast(readInt(conn.from)); + conn.from >> res.errorMsg; + if (GET_PROTOCOL_MINOR(conn.version) >= 29) { + conn.from + >> res.timesBuilt + >> res.isNonDeterministic + >> res.startTime + >> res.stopTime; + } + if (GET_PROTOCOL_MINOR(conn.version) >= 28) { + auto builtOutputs = WorkerProto::Serialise::read(store, conn); + for (auto && [output, realisation] : builtOutputs) + res.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); + } return res; } @@ -156,38 +124,68 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto { conn.to << res.status - << res.errorMsg - << res.timesBuilt - << res.isNonDeterministic - << res.startTime - << res.stopTime; - DrvOutputs builtOutputs; - for (auto & [output, realisation] : res.builtOutputs) - builtOutputs.insert_or_assign(realisation.id, realisation); - WorkerProto::write(store, conn, builtOutputs); + << res.errorMsg; + if (GET_PROTOCOL_MINOR(conn.version) >= 29) { + conn.to + << res.timesBuilt + << res.isNonDeterministic + << res.startTime + << res.stopTime; + } + if (GET_PROTOCOL_MINOR(conn.version) >= 28) { + DrvOutputs builtOutputs; + for (auto & [output, realisation] : res.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + WorkerProto::write(store, conn, builtOutputs); + } } -std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) +ValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn) { - auto s = readString(conn.from); - return s == "" ? std::optional {} : store.parseStorePath(s); + auto path = WorkerProto::Serialise::read(store, conn); + return ValidPathInfo { + std::move(path), + WorkerProto::Serialise::read(store, conn), + }; } -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional & storePathOpt) +void WorkerProto::Serialise::write(const Store & store, WriteConn conn, const ValidPathInfo & pathInfo) { - conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); + WorkerProto::write(store, conn, pathInfo.path); + WorkerProto::write(store, conn, static_cast(pathInfo)); } -std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) +UnkeyedValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn) { - return ContentAddress::parseOpt(readString(conn.from)); + auto deriver = readString(conn.from); + auto narHash = Hash::parseAny(readString(conn.from), htSHA256); + UnkeyedValidPathInfo info(narHash); + if (deriver != "") info.deriver = store.parseStorePath(deriver); + info.references = WorkerProto::Serialise::read(store, conn); + conn.from >> info.registrationTime >> info.narSize; + if (GET_PROTOCOL_MINOR(conn.version) >= 16) { + conn.from >> info.ultimate; + info.sigs = readStrings(conn.from); + info.ca = ContentAddress::parseOpt(readString(conn.from)); + } + return info; } -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional & caOpt) +void WorkerProto::Serialise::write(const Store & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo) { - conn.to << (caOpt ? renderContentAddress(*caOpt) : ""); + conn.to + << (pathInfo.deriver ? store.printStorePath(*pathInfo.deriver) : "") + << pathInfo.narHash.to_string(HashFormat::Base16, false); + WorkerProto::write(store, conn, pathInfo.references); + conn.to << pathInfo.registrationTime << pathInfo.narSize; + if (GET_PROTOCOL_MINOR(conn.version) >= 16) { + conn.to + << pathInfo.ultimate + << pathInfo.sigs + << renderContentAddress(pathInfo.ca); + } } } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index ff762c924ab..25d544ba72d 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "serialise.hh" +#include "common-protocol.hh" namespace nix { @@ -29,10 +29,10 @@ struct Source; // items being serialised struct DerivedPath; -struct DrvOutput; -struct Realisation; struct BuildResult; struct KeyedBuildResult; +struct ValidPathInfo; +struct UnkeyedValidPathInfo; enum TrustedFlag : bool; @@ -49,26 +49,29 @@ struct WorkerProto */ enum struct Op : uint64_t; + /** + * Version type for the protocol. + * + * @todo Convert to struct with separate major vs minor fields. + */ + using Version = unsigned int; + /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. - * - * This currently is just a `Source &`, but more fields will be added - * later. */ struct ReadConn { Source & from; + Version version; }; /** * A unidirectional write connection, to be used by the write half of the * canonical serializers below. - * - * This currently is just a `Sink &`, but more fields will be added - * later. */ struct WriteConn { Sink & to; + Version version; }; /** @@ -168,7 +171,7 @@ enum struct WorkerProto::Op : uint64_t */ inline Sink & operator << (Sink & sink, WorkerProto::Op op) { - return sink << (uint64_t) op; + return sink << static_cast(op); } /** @@ -178,7 +181,7 @@ inline Sink & operator << (Sink & sink, WorkerProto::Op op) */ inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op) { - return s << (uint64_t) op; + return s << static_cast(op); } /** @@ -191,58 +194,36 @@ inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op) * be legal specialization syntax. See below for what that looks like in * practice. */ -#define MAKE_WORKER_PROTO(T) \ - struct WorkerProto::Serialise< T > { \ +#define DECLARE_WORKER_SERIALISER(T) \ + struct WorkerProto::Serialise< T > \ + { \ static T read(const Store & store, WorkerProto::ReadConn conn); \ static void write(const Store & store, WorkerProto::WriteConn conn, const T & t); \ }; template<> -MAKE_WORKER_PROTO(std::string); -template<> -MAKE_WORKER_PROTO(StorePath); -template<> -MAKE_WORKER_PROTO(ContentAddress); +DECLARE_WORKER_SERIALISER(DerivedPath); template<> -MAKE_WORKER_PROTO(DerivedPath); +DECLARE_WORKER_SERIALISER(BuildResult); template<> -MAKE_WORKER_PROTO(Realisation); +DECLARE_WORKER_SERIALISER(KeyedBuildResult); template<> -MAKE_WORKER_PROTO(DrvOutput); +DECLARE_WORKER_SERIALISER(ValidPathInfo); template<> -MAKE_WORKER_PROTO(BuildResult); +DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo); template<> -MAKE_WORKER_PROTO(KeyedBuildResult); -template<> -MAKE_WORKER_PROTO(std::optional); +DECLARE_WORKER_SERIALISER(std::optional); template -MAKE_WORKER_PROTO(std::vector); +DECLARE_WORKER_SERIALISER(std::vector); template -MAKE_WORKER_PROTO(std::set); +DECLARE_WORKER_SERIALISER(std::set); +template +DECLARE_WORKER_SERIALISER(std::tuple); +#define COMMA_ , template -#define X_ std::map -MAKE_WORKER_PROTO(X_); -#undef X_ - -/** - * These use the empty string for the null case, relying on the fact - * that the underlying types never serialise to the empty string. - * - * We do this instead of a generic std::optional instance because - * ordinal tags (0 or 1, here) are a bit of a compatability hazard. For - * the same reason, we don't have a std::variant instances (ordinal - * tags 0...n). - * - * We could the generic instances and then these as specializations for - * compatability, but that's proven a bit finnicky, and also makes the - * worker protocol harder to implement in other languages where such - * specializations may not be allowed. - */ -template<> -MAKE_WORKER_PROTO(std::optional); -template<> -MAKE_WORKER_PROTO(std::optional); +DECLARE_WORKER_SERIALISER(std::map); +#undef COMMA_ } diff --git a/src/libutil/abstract-setting-to-json.hh b/src/libutil/abstract-setting-to-json.hh index d506dfb743f..eea687d8a4a 100644 --- a/src/libutil/abstract-setting-to-json.hh +++ b/src/libutil/abstract-setting-to-json.hh @@ -7,7 +7,7 @@ namespace nix { template -std::map BaseSetting::toJSONObject() +std::map BaseSetting::toJSONObject() const { auto obj = AbstractSetting::toJSONObject(); obj.emplace("value", value); diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 268a798d900..465df2073d2 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -5,15 +5,11 @@ #include // for strcasecmp -#include -#include -#include -#include -#include - #include "archive.hh" -#include "util.hh" #include "config.hh" +#include "posix-source-accessor.hh" +#include "file-system.hh" +#include "signals.hh" namespace nix { @@ -27,8 +23,6 @@ struct ArchiveSettings : Config #endif "use-case-hack", "Whether to enable a Darwin-specific hack for dealing with file name collisions."}; - Setting preallocateContents{this, false, "preallocate-contents", - "Whether to preallocate files when writing objects with known size."}; }; static ArchiveSettings archiveSettings; @@ -38,91 +32,87 @@ static GlobalConfig::Register rArchiveSettings(&archiveSettings); PathFilter defaultPathFilter = [](const Path &) { return true; }; -static void dumpContents(const Path & path, off_t size, - Sink & sink) +void SourceAccessor::dumpPath( + const CanonPath & path, + Sink & sink, + PathFilter & filter) { - sink << "contents" << size; - - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) throw SysError("opening file '%1%'", path); - - std::vector buf(65536); - size_t left = size; - - while (left > 0) { - auto n = std::min(left, buf.size()); - readFull(fd.get(), buf.data(), n); - left -= n; - sink({buf.data(), n}); - } + auto dumpContents = [&](const CanonPath & path) + { + sink << "contents"; + std::optional size; + readFile(path, sink, [&](uint64_t _size) + { + size = _size; + sink << _size; + }); + assert(size); + writePadding(*size, sink); + }; - writePadding(size, sink); -} + std::function dump; + dump = [&](const CanonPath & path) { + checkInterrupt(); -static time_t dump(const Path & path, Sink & sink, PathFilter & filter) -{ - checkInterrupt(); + auto st = lstat(path); - auto st = lstat(path); - time_t result = st.st_mtime; + sink << "("; - sink << "("; + if (st.type == tRegular) { + sink << "type" << "regular"; + if (st.isExecutable) + sink << "executable" << ""; + dumpContents(path); + } - if (S_ISREG(st.st_mode)) { - sink << "type" << "regular"; - if (st.st_mode & S_IXUSR) - sink << "executable" << ""; - dumpContents(path, st.st_size, sink); - } + else if (st.type == tDirectory) { + sink << "type" << "directory"; + + /* If we're on a case-insensitive system like macOS, undo + the case hack applied by restorePath(). */ + std::map unhacked; + for (auto & i : readDirectory(path)) + if (archiveSettings.useCaseHack) { + std::string name(i.first); + size_t pos = i.first.find(caseHackSuffix); + if (pos != std::string::npos) { + debug("removing case hack suffix from '%s'", path + i.first); + name.erase(pos); + } + if (!unhacked.emplace(name, i.first).second) + throw Error("file name collision in between '%s' and '%s'", + (path + unhacked[name]), + (path + i.first)); + } else + unhacked.emplace(i.first, i.first); - else if (S_ISDIR(st.st_mode)) { - sink << "type" << "directory"; - - /* If we're on a case-insensitive system like macOS, undo - the case hack applied by restorePath(). */ - std::map unhacked; - for (auto & i : readDirectory(path)) - if (archiveSettings.useCaseHack) { - std::string name(i.name); - size_t pos = i.name.find(caseHackSuffix); - if (pos != std::string::npos) { - debug("removing case hack suffix from '%1%'", path + "/" + i.name); - name.erase(pos); + for (auto & i : unhacked) + if (filter((path + i.first).abs())) { + sink << "entry" << "(" << "name" << i.first << "node"; + dump(path + i.second); + sink << ")"; } - if (!unhacked.emplace(name, i.name).second) - throw Error("file name collision in between '%1%' and '%2%'", - (path + "/" + unhacked[name]), - (path + "/" + i.name)); - } else - unhacked.emplace(i.name, i.name); - - for (auto & i : unhacked) - if (filter(path + "/" + i.first)) { - sink << "entry" << "(" << "name" << i.first << "node"; - auto tmp_mtime = dump(path + "/" + i.second, sink, filter); - if (tmp_mtime > result) { - result = tmp_mtime; - } - sink << ")"; - } - } + } - else if (S_ISLNK(st.st_mode)) - sink << "type" << "symlink" << "target" << readLink(path); + else if (st.type == tSymlink) + sink << "type" << "symlink" << "target" << readLink(path); - else throw Error("file '%1%' has an unsupported type", path); + else throw Error("file '%s' has an unsupported type", path); - sink << ")"; + sink << ")"; + }; - return result; + sink << narVersionMagic1; + dump(path); } time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter) { - sink << narVersionMagic1; - return dump(path, sink, filter); + PosixSourceAccessor accessor; + accessor.dumpPath(CanonPath::fromCwd(path), sink, filter); + return accessor.mtime; } void dumpPath(const Path & path, Sink & sink, PathFilter & filter) @@ -143,17 +133,6 @@ static SerialisationError badArchive(const std::string & s) } -#if 0 -static void skipGeneric(Source & source) -{ - if (readString(source) == "(") { - while (readString(source) != ")") - skipGeneric(source); - } -} -#endif - - static void parseContents(ParseSink & sink, Source & source, const Path & path) { uint64_t size = readLongLong(source); @@ -302,71 +281,6 @@ void parseDump(ParseSink & sink, Source & source) } -struct RestoreSink : ParseSink -{ - Path dstPath; - AutoCloseFD fd; - - void createDirectory(const Path & path) override - { - Path p = dstPath + path; - if (mkdir(p.c_str(), 0777) == -1) - throw SysError("creating directory '%1%'", p); - }; - - void createRegularFile(const Path & path) override - { - Path p = dstPath + path; - fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); - if (!fd) throw SysError("creating file '%1%'", p); - } - - void closeRegularFile() override - { - /* Call close explicitly to make sure the error is checked */ - fd.close(); - } - - void isExecutable() override - { - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("fstat"); - if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) - throw SysError("fchmod"); - } - - void preallocateContents(uint64_t len) override - { - if (!archiveSettings.preallocateContents) - return; - -#if HAVE_POSIX_FALLOCATE - if (len) { - errno = posix_fallocate(fd.get(), 0, len); - /* Note that EINVAL may indicate that the underlying - filesystem doesn't support preallocation (e.g. on - OpenSolaris). Since preallocation is just an - optimisation, ignore it. */ - if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) - throw SysError("preallocating file of %1% bytes", len); - } -#endif - } - - void receiveContents(std::string_view data) override - { - writeFull(fd.get(), data); - } - - void createSymlink(const Path & path, const std::string & target) override - { - Path p = dstPath + path; - nix::createSymlink(target, p); - } -}; - - void restorePath(const Path & path, Source & source) { RestoreSink sink; @@ -380,7 +294,7 @@ void copyNAR(Source & source, Sink & sink) // FIXME: if 'source' is the output of dumpPath() followed by EOF, // we should just forward all data directly without parsing. - ParseSink parseSink; /* null sink; just parse the NAR */ + NullParseSink parseSink; /* just parse the NAR */ TeeSource wrapper { source, sink }; diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 2cf164a417d..2cf8ee89118 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "serialise.hh" +#include "fs-sink.hh" namespace nix { @@ -72,49 +73,6 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink, */ void dumpString(std::string_view s, Sink & sink); -/** - * \todo Fix this API, it sucks. - */ -struct ParseSink -{ - virtual void createDirectory(const Path & path) { }; - - virtual void createRegularFile(const Path & path) { }; - virtual void closeRegularFile() { }; - virtual void isExecutable() { }; - virtual void preallocateContents(uint64_t size) { }; - virtual void receiveContents(std::string_view data) { }; - - virtual void createSymlink(const Path & path, const std::string & target) { }; -}; - -/** - * If the NAR archive contains a single file at top-level, then save - * the contents of the file to `s`. Otherwise barf. - */ -struct RetrieveRegularNARSink : ParseSink -{ - bool regular = true; - Sink & sink; - - RetrieveRegularNARSink(Sink & sink) : sink(sink) { } - - void createDirectory(const Path & path) override - { - regular = false; - } - - void receiveContents(std::string_view data) override - { - sink(data); - } - - void createSymlink(const Path & path, const std::string & target) override - { - regular = false; - } -}; - void parseDump(ParseSink & sink, Source & source); void restorePath(const Path & path, Source & source); diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 00281ce6f88..4480a03f5c6 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -1,7 +1,14 @@ #include "args.hh" +#include "args/root.hh" #include "hash.hh" +#include "environment-variables.hh" +#include "signals.hh" +#include "users.hh" #include "json-utils.hh" +#include +#include +#include #include namespace nix { @@ -26,6 +33,11 @@ void Args::removeFlag(const std::string & longName) longFlags.erase(flag); } +void Completions::setType(AddCompletions::Type t) +{ + type = t; +} + void Completions::add(std::string completion, std::string description) { description = trim(description); @@ -37,7 +49,7 @@ void Completions::add(std::string completion, std::string description) if (needs_ellipsis) description.append(" [...]"); } - insert(Completion { + completions.insert(Completion { .completion = completion, .description = description }); @@ -46,12 +58,20 @@ void Completions::add(std::string completion, std::string description) bool Completion::operator<(const Completion & other) const { return completion < other.completion || (completion == other.completion && description < other.description); } -CompletionType completionType = ctNormal; -std::shared_ptr completions; - std::string completionMarker = "___COMPLETE___"; -static std::optional needsCompletion(std::string_view s) +RootArgs & Args::getRoot() +{ + Args * p = this; + while (p->parent) + p = p->parent; + + auto * res = dynamic_cast(p); + assert(res); + return *res; +} + +std::optional RootArgs::needsCompletion(std::string_view s) { if (!completions) return {}; auto i = s.find(completionMarker); @@ -60,7 +80,178 @@ static std::optional needsCompletion(std::string_view s) return {}; } -void Args::parseCmdline(const Strings & _cmdline) +/** + * Basically this is `typedef std::optional Parser(std::string_view s, Strings & r);` + * + * Except we can't recursively reference the Parser typedef, so we have to write a class. + */ +struct Parser { + std::string_view remaining; + + /** + * @brief Parse the next character(s) + * + * @param r + * @return std::shared_ptr + */ + virtual void operator()(std::shared_ptr & state, Strings & r) = 0; + + Parser(std::string_view s) : remaining(s) {}; + + virtual ~Parser() { }; +}; + +struct ParseQuoted : public Parser { + /** + * @brief Accumulated string + * + * Parsed argument up to this point. + */ + std::string acc; + + ParseQuoted(std::string_view s) : Parser(s) {}; + + virtual void operator()(std::shared_ptr & state, Strings & r) override; +}; + + +struct ParseUnquoted : public Parser { + /** + * @brief Accumulated string + * + * Parsed argument up to this point. Empty string is not representable in + * unquoted syntax, so we use it for the initial state. + */ + std::string acc; + + ParseUnquoted(std::string_view s) : Parser(s) {}; + + virtual void operator()(std::shared_ptr & state, Strings & r) override { + if (remaining.empty()) { + if (!acc.empty()) + r.push_back(acc); + state = nullptr; // done + return; + } + switch (remaining[0]) { + case ' ': case '\t': case '\n': case '\r': + if (!acc.empty()) + r.push_back(acc); + state = std::make_shared(ParseUnquoted(remaining.substr(1))); + return; + case '`': + if (remaining.size() > 1 && remaining[1] == '`') { + state = std::make_shared(ParseQuoted(remaining.substr(2))); + return; + } + else + throw Error("single backtick is not a supported syntax in the nix shebang."); + + // reserved characters + // meaning to be determined, or may be reserved indefinitely so that + // #!nix syntax looks unambiguous + case '$': + case '*': + case '~': + case '<': + case '>': + case '|': + case ';': + case '(': + case ')': + case '[': + case ']': + case '{': + case '}': + case '\'': + case '"': + case '\\': + throw Error("unsupported unquoted character in nix shebang: " + std::string(1, remaining[0]) + ". Use double backticks to escape?"); + + case '#': + if (acc.empty()) { + throw Error ("unquoted nix shebang argument cannot start with #. Use double backticks to escape?"); + } else { + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + + default: + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + assert(false); + } +}; + +void ParseQuoted::operator()(std::shared_ptr &state, Strings & r) { + if (remaining.empty()) { + throw Error("unterminated quoted string in nix shebang"); + } + switch (remaining[0]) { + case ' ': + if ((remaining.size() == 3 && remaining[1] == '`' && remaining[2] == '`') + || (remaining.size() > 3 && remaining[1] == '`' && remaining[2] == '`' && remaining[3] != '`')) { + // exactly two backticks mark the end of a quoted string, but a preceding space is ignored if present. + state = std::make_shared(ParseUnquoted(remaining.substr(3))); + r.push_back(acc); + return; + } + else { + // just a normal space + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + case '`': + // exactly two backticks mark the end of a quoted string + if ((remaining.size() == 2 && remaining[1] == '`') + || (remaining.size() > 2 && remaining[1] == '`' && remaining[2] != '`')) { + state = std::make_shared(ParseUnquoted(remaining.substr(2))); + r.push_back(acc); + return; + } + + // a sequence of at least 3 backticks is one escape-backtick which is ignored, followed by any number of backticks, which are verbatim + else if (remaining.size() >= 3 && remaining[1] == '`' && remaining[2] == '`') { + // ignore "escape" backtick + remaining = remaining.substr(1); + // add the rest + while (remaining.size() > 0 && remaining[0] == '`') { + acc += '`'; + remaining = remaining.substr(1); + } + return; + } + else { + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + default: + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + assert(false); +} + +Strings parseShebangContent(std::string_view s) { + Strings result; + std::shared_ptr parserState(std::make_shared(ParseUnquoted(s))); + + // trampoline == iterated strategy pattern + while (parserState) { + auto currentState = parserState; + (*currentState)(parserState, result); + } + + return result; +} + +void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang) { Strings pendingArgs; bool dashDash = false; @@ -71,11 +262,50 @@ void Args::parseCmdline(const Strings & _cmdline) size_t n = std::stoi(*s); assert(n > 0 && n <= cmdline.size()); *std::next(cmdline.begin(), n - 1) += completionMarker; - completions = std::make_shared(); + completions = std::make_shared(); verbosity = lvlError; } bool argsSeen = false; + + // Heuristic to see if we're invoked as a shebang script, namely, + // if we have at least one argument, it's the name of an + // executable file, and it starts with "#!". + Strings savedArgs; + if (allowShebang){ + auto script = *cmdline.begin(); + try { + std::ifstream stream(script); + char shebang[3]={0,0,0}; + stream.get(shebang,3); + if (strncmp(shebang,"#!",2) == 0){ + for (auto pos = std::next(cmdline.begin()); pos != cmdline.end();pos++) + savedArgs.push_back(*pos); + cmdline.clear(); + + std::string line; + std::getline(stream,line); + static const std::string commentChars("#/\\%@*-"); + std::string shebangContent; + while (std::getline(stream,line) && !line.empty() && commentChars.find(line[0]) != std::string::npos){ + line = chomp(line); + + std::smatch match; + // We match one space after `nix` so that we preserve indentation. + // No space is necessary for an empty line. An empty line has basically no effect. + if (std::regex_match(line, match, std::regex("^#!\\s*nix(:? |$)(.*)$"))) + shebangContent += match[2].str() + "\n"; + } + for (const auto & word : parseShebangContent(shebangContent)) { + cmdline.push_back(word); + } + cmdline.push_back(script); + commandBaseDir = dirOf(script); + for (auto pos = savedArgs.begin(); pos != savedArgs.end();pos++) + cmdline.push_back(*pos); + } + } catch (SysError &) { } + } for (auto pos = cmdline.begin(); pos != cmdline.end(); ) { auto arg = *pos; @@ -125,17 +355,34 @@ void Args::parseCmdline(const Strings & _cmdline) for (auto & f : flagExperimentalFeatures) experimentalFeatureSettings.require(f); + /* Now that all the other args are processed, run the deferred completions. + */ + for (auto d : deferredCompletions) + d.completer(*completions, d.n, d.prefix); +} + +Path Args::getCommandBaseDir() const +{ + assert(parent); + return parent->getCommandBaseDir(); +} + +Path RootArgs::getCommandBaseDir() const +{ + return commandBaseDir; } bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) { assert(pos != end); + auto & rootArgs = getRoot(); + auto process = [&](const std::string & name, const Flag & flag) -> bool { ++pos; if (auto & f = flag.experimentalFeature) - flagExperimentalFeatures.insert(*f); + rootArgs.flagExperimentalFeatures.insert(*f); std::vector args; bool anyCompleted = false; @@ -146,10 +393,15 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) "flag '%s' requires %d argument(s), but only %d were given", name, flag.handler.arity, n); } - if (auto prefix = needsCompletion(*pos)) { + if (auto prefix = rootArgs.needsCompletion(*pos)) { anyCompleted = true; - if (flag.completer) - flag.completer(n, *prefix); + if (flag.completer) { + rootArgs.deferredCompletions.push_back({ + .completer = flag.completer, + .n = n, + .prefix = *prefix, + }); + } } args.push_back(*pos++); } @@ -159,14 +411,14 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) }; if (std::string(*pos, 0, 2) == "--") { - if (auto prefix = needsCompletion(*pos)) { + if (auto prefix = rootArgs.needsCompletion(*pos)) { for (auto & [name, flag] : longFlags) { if (!hiddenCategories.count(flag->category) && hasPrefix(name, std::string(*prefix, 2))) { if (auto & f = flag->experimentalFeature) - flagExperimentalFeatures.insert(*f); - completions->add("--" + name, flag->description); + rootArgs.flagExperimentalFeatures.insert(*f); + rootArgs.completions->add("--" + name, flag->description); } } return false; @@ -183,12 +435,12 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) return process(std::string("-") + c, *i->second); } - if (auto prefix = needsCompletion(*pos)) { + if (auto prefix = rootArgs.needsCompletion(*pos)) { if (prefix == "-") { - completions->add("--"); + rootArgs.completions->add("--"); for (auto & [flagName, flag] : shortFlags) if (experimentalFeatureSettings.isEnabled(flag->experimentalFeature)) - completions->add(std::string("-") + flagName, flag->description); + rootArgs.completions->add(std::string("-") + flagName, flag->description); } } @@ -203,6 +455,8 @@ bool Args::processArgs(const Strings & args, bool finish) return true; } + auto & rootArgs = getRoot(); + auto & exp = expectedArgs.front(); bool res = false; @@ -211,16 +465,35 @@ bool Args::processArgs(const Strings & args, bool finish) (exp.handler.arity != ArityAny && args.size() == exp.handler.arity)) { std::vector ss; + bool anyCompleted = false; for (const auto &[n, s] : enumerate(args)) { - if (auto prefix = needsCompletion(s)) { + if (auto prefix = rootArgs.needsCompletion(s)) { + anyCompleted = true; ss.push_back(*prefix); - if (exp.completer) - exp.completer(n, *prefix); + if (exp.completer) { + rootArgs.deferredCompletions.push_back({ + .completer = exp.completer, + .n = n, + .prefix = *prefix, + }); + } } else ss.push_back(s); } - exp.handler.fun(ss); - expectedArgs.pop_front(); + if (!anyCompleted) + exp.handler.fun(ss); + + /* Move the list element to the processedArgs. This is almost the same as + `processedArgs.push_back(expectedArgs.front()); expectedArgs.pop_front()`, + except that it will only adjust the next and prev pointers of the list + elements, meaning the actual contents don't move in memory. This is + critical to prevent invalidating internal pointers! */ + processedArgs.splice( + processedArgs.end(), + expectedArgs, + expectedArgs.begin(), + ++expectedArgs.begin()); + res = true; } @@ -236,6 +509,7 @@ nlohmann::json Args::toJSON() for (auto & [name, flag] : longFlags) { auto j = nlohmann::json::object(); + j["hiddenCategory"] = hiddenCategories.count(flag->category) > 0; if (flag->aliases.count(name)) continue; if (flag->shortName) j["shortName"] = std::string(1, flag->shortName); @@ -270,11 +544,11 @@ nlohmann::json Args::toJSON() return res; } -static void hashTypeCompleter(size_t index, std::string_view prefix) +static void hashTypeCompleter(AddCompletions & completions, size_t index, std::string_view prefix) { for (auto & type : hashTypes) if (hasPrefix(type, prefix)) - completions->add(type); + completions.add(type); } Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) @@ -286,7 +560,7 @@ Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) .handler = {[ht](std::string s) { *ht = parseHashType(s); }}, - .completer = hashTypeCompleter + .completer = hashTypeCompleter, }; } @@ -299,13 +573,13 @@ Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional< .handler = {[oht](std::string s) { *oht = std::optional { parseHashType(s) }; }}, - .completer = hashTypeCompleter + .completer = hashTypeCompleter, }; } -static void _completePath(std::string_view prefix, bool onlyDirs) +static void _completePath(AddCompletions & completions, std::string_view prefix, bool onlyDirs) { - completionType = ctFilenames; + completions.setType(Completions::Type::Filenames); glob_t globbuf; int flags = GLOB_NOESCAPE; #ifdef GLOB_ONLYDIR @@ -319,20 +593,20 @@ static void _completePath(std::string_view prefix, bool onlyDirs) auto st = stat(globbuf.gl_pathv[i]); if (!S_ISDIR(st.st_mode)) continue; } - completions->add(globbuf.gl_pathv[i]); + completions.add(globbuf.gl_pathv[i]); } } globfree(&globbuf); } -void completePath(size_t, std::string_view prefix) +void Args::completePath(AddCompletions & completions, size_t, std::string_view prefix) { - _completePath(prefix, false); + _completePath(completions, prefix, false); } -void completeDir(size_t, std::string_view prefix) +void Args::completeDir(AddCompletions & completions, size_t, std::string_view prefix) { - _completePath(prefix, true); + _completePath(completions, prefix, true); } Strings argvToStrings(int argc, char * * argv) @@ -367,10 +641,10 @@ MultiCommand::MultiCommand(const Commands & commands_) command = {s, i->second()}; command->second->parent = this; }}, - .completer = {[&](size_t, std::string_view prefix) { + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { for (auto & [name, command] : commands) if (hasPrefix(name, prefix)) - completions->add(name); + completions.add(name); }} }); @@ -392,14 +666,6 @@ bool MultiCommand::processArgs(const Strings & args, bool finish) return Args::processArgs(args, finish); } -void MultiCommand::completionHook() -{ - if (command) - return command->second->completionHook(); - else - return Args::completionHook(); -} - nlohmann::json MultiCommand::toJSON() { auto cmds = nlohmann::json::object(); diff --git a/src/libutil/args.hh b/src/libutil/args.hh index d90129796ff..7af82b17884 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -2,12 +2,15 @@ ///@file #include +#include #include #include +#include #include -#include "util.hh" +#include "types.hh" +#include "experimental-features.hh" namespace nix { @@ -15,19 +18,18 @@ enum HashType : char; class MultiCommand; +class RootArgs; + +class AddCompletions; + class Args { -public: - /** - * Parse the command line, throwing a UsageError if something goes - * wrong. - */ - void parseCmdline(const Strings & cmdline); +public: /** * Return a short one-line description of the command. - */ + */ virtual std::string description() { return ""; } virtual bool forceImpureByDefault() { return false; } @@ -37,16 +39,39 @@ public: */ virtual std::string doc() { return ""; } + /** + * @brief Get the base directory for the command. + * + * @return Generally the working directory, but in case of a shebang + * interpreter, returns the directory of the script. + * + * This only returns the correct value after parseCmdline() has run. + */ + virtual Path getCommandBaseDir() const; + protected: + /** + * The largest `size_t` is used to indicate the "any" arity, for + * handlers/flags/arguments that accept an arbitrary number of + * arguments. + */ static const size_t ArityAny = std::numeric_limits::max(); + /** + * Arguments (flags/options and positional) have a "handler" which is + * caused when the argument is parsed. The handler has an arbitrary side + * effect, including possible affect further command-line parsing. + * + * There are many constructors in order to support many shorthand + * initializations, and this is used a lot. + */ struct Handler { std::function)> fun; size_t arity; - Handler() {} + Handler() = default; Handler(std::function)> && fun) : fun(std::move(fun)) @@ -73,29 +98,29 @@ protected: { } Handler(std::vector * dest) - : fun([=](std::vector ss) { *dest = ss; }) + : fun([dest](std::vector ss) { *dest = ss; }) , arity(ArityAny) { } Handler(std::string * dest) - : fun([=](std::vector ss) { *dest = ss[0]; }) + : fun([dest](std::vector ss) { *dest = ss[0]; }) , arity(1) { } Handler(std::optional * dest) - : fun([=](std::vector ss) { *dest = ss[0]; }) + : fun([dest](std::vector ss) { *dest = ss[0]; }) , arity(1) { } template Handler(T * dest, const T & val) - : fun([=](std::vector ss) { *dest = val; }) + : fun([dest, val](std::vector ss) { *dest = val; }) , arity(0) { } template Handler(I * dest) - : fun([=](std::vector ss) { + : fun([dest](std::vector ss) { *dest = string2IntWithUnitPrefix(ss[0]); }) , arity(1) @@ -103,17 +128,41 @@ protected: template Handler(std::optional * dest) - : fun([=](std::vector ss) { + : fun([dest](std::vector ss) { *dest = string2IntWithUnitPrefix(ss[0]); }) , arity(1) { } }; - /* Options. */ + /** + * The basic function type of the completion callback. + * + * Used to define `CompleterClosure` and some common case completers + * that individual flags/arguments can use. + * + * The `AddCompletions` that is passed is an interface to the state + * stored as part of the root command + */ + using CompleterFun = void(AddCompletions &, size_t, std::string_view); + + /** + * The closure type of the completion callback. + * + * This is what is actually stored as part of each Flag / Expected + * Arg. + */ + using CompleterClosure = std::function; + + /** + * Description of flags / options + * + * These are arguments like `-s` or `--long` that can (mostly) + * appear in any order. + */ struct Flag { - typedef std::shared_ptr ptr; + using ptr = std::shared_ptr; std::string longName; std::set aliases; @@ -122,7 +171,7 @@ protected: std::string category; Strings labels; Handler handler; - std::function completer; + CompleterClosure completer; std::optional experimentalFeature; @@ -130,22 +179,68 @@ protected: static Flag mkHashTypeOptFlag(std::string && longName, std::optional * oht); }; + /** + * Index of all registered "long" flag descriptions (flags like + * `--long`). + */ std::map longFlags; + + /** + * Index of all registered "short" flag descriptions (flags like + * `-s`). + */ std::map shortFlags; + /** + * Process a single flag and its arguments, pulling from an iterator + * of raw CLI args as needed. + */ virtual bool processFlag(Strings::iterator & pos, Strings::iterator end); - /* Positional arguments. */ + /** + * Description of positional arguments + * + * These are arguments that do not start with a `-`, and for which + * the order does matter. + */ struct ExpectedArg { std::string label; bool optional = false; Handler handler; - std::function completer; + CompleterClosure completer; }; + /** + * Queue of expected positional argument forms. + * + * Positional argument descriptions are inserted on the back. + * + * As positional arguments are passed, these are popped from the + * front, until there are hopefully none left as all args that were + * expected in fact were passed. + */ std::list expectedArgs; + /** + * List of processed positional argument forms. + * + * All items removed from `expectedArgs` are added here. After all + * arguments were processed, this list should be exactly the same as + * `expectedArgs` was before. + * + * This list is used to extend the lifetime of the argument forms. + * If this is not done, some closures that reference the command + * itself will segfault. + */ + std::list processedArgs; + /** + * Process some positional arugments + * + * @param finish: We have parsed everything else, and these are the only + * arguments left. Used because we accumulate some "pending args" we might + * have left over. + */ virtual bool processArgs(const Strings & args, bool finish); virtual Strings::iterator rewriteArgs(Strings & args, Strings::iterator pos) @@ -159,13 +254,6 @@ protected: */ virtual void initialFlagsProcessed() {} - /** - * Called after the command line has been processed if we need to generate - * completions. Useful for commands that need to know the whole command line - * in order to know what completions to generate. - */ - virtual void completionHook() { } - public: void addFlag(Flag && flag); @@ -200,21 +288,30 @@ public: }); } + static CompleterFun completePath; + + static CompleterFun completeDir; + virtual nlohmann::json toJSON(); friend class MultiCommand; + /** + * The parent command, used if this is a subcommand. + * + * Invariant: An Args with a null parent must also be a RootArgs + * + * \todo this would probably be better in the CommandClass. + * getRoot() could be an abstract method that peels off at most one + * layer before recuring. + */ MultiCommand * parent = nullptr; -private: - /** - * Experimental features needed when parsing args. These are checked - * after flag parsing is completed in order to support enabling - * experimental features coming after the flag that needs the - * experimental feature. + * Traverse parent pointers until we find the \ref RootArgs "root + * arguments" object. */ - std::set flagExperimentalFeatures; + RootArgs & getRoot(); }; /** @@ -225,23 +322,23 @@ struct Command : virtual public Args { friend class MultiCommand; - virtual ~Command() { } + virtual ~Command() = default; /** * Entry point to the command */ virtual void run() = 0; - typedef int Category; + using Category = int; static constexpr Category catDefault = 0; - virtual std::optional experimentalFeature (); + virtual std::optional experimentalFeature(); virtual Category category() { return catDefault; } }; -typedef std::map()>> Commands; +using Commands = std::map()>>; /** * An argument parser that supports multiple subcommands, @@ -265,8 +362,6 @@ public: bool processArgs(const Strings & args, bool finish) override; - void completionHook() override; - nlohmann::json toJSON() override; }; @@ -278,21 +373,42 @@ struct Completion { bool operator<(const Completion & other) const; }; -class Completions : public std::set { + +/** + * The abstract interface for completions callbacks + * + * The idea is to restrict the callback so it can only add additional + * completions to the collection, or set the completion type. By making + * it go through this interface, the callback cannot make any other + * changes, or even view the completions / completion type that have + * been set so far. + */ +class AddCompletions +{ public: - void add(std::string completion, std::string description = ""); -}; -extern std::shared_ptr completions; -enum CompletionType { - ctNormal, - ctFilenames, - ctAttrs -}; -extern CompletionType completionType; + /** + * The type of completion we are collecting. + */ + enum class Type { + Normal, + Filenames, + Attrs, + }; + + /** + * Set the type of the completions being collected + * + * \todo it should not be possible to change the type after it has been set. + */ + virtual void setType(Type type) = 0; -void completePath(size_t, std::string_view prefix); + /** + * Add a single completion to the collection + */ + virtual void add(std::string completion, std::string description = "") = 0; +}; -void completeDir(size_t, std::string_view prefix); +Strings parseShebangContent(std::string_view s); } diff --git a/src/libutil/args/root.hh b/src/libutil/args/root.hh new file mode 100644 index 00000000000..5c55c37a55d --- /dev/null +++ b/src/libutil/args/root.hh @@ -0,0 +1,84 @@ +#pragma once + +#include "args.hh" + +namespace nix { + +/** + * The concrete implementation of a collection of completions. + * + * This is exposed so that the main entry point can print out the + * collected completions. + */ +struct Completions final : AddCompletions +{ + std::set completions; + Type type = Type::Normal; + + void setType(Type type) override; + void add(std::string completion, std::string description = "") override; +}; + +/** + * The outermost Args object. This is the one we will actually parse a command + * line with, whereas the inner ones (if they exists) are subcommands (and this + * is also a MultiCommand or something like it). + * + * This Args contains completions state shared between it and all of its + * descendent Args. + */ +class RootArgs : virtual public Args +{ + /** + * @brief The command's "working directory", but only set when top level. + * + * Use getCommandBaseDir() to get the directory regardless of whether this + * is a top-level command or subcommand. + * + * @see getCommandBaseDir() + */ + Path commandBaseDir = "."; + +public: + /** Parse the command line, throwing a UsageError if something goes + * wrong. + */ + void parseCmdline(const Strings & cmdline, bool allowShebang = false); + + std::shared_ptr completions; + + Path getCommandBaseDir() const override; + +protected: + + friend class Args; + + /** + * A pointer to the completion and its two arguments; a thunk; + */ + struct DeferredCompletion { + const CompleterClosure & completer; + size_t n; + std::string prefix; + }; + + /** + * Completions are run after all args and flags are parsed, so completions + * of earlier arguments can benefit from later arguments. + */ + std::vector deferredCompletions; + + /** + * Experimental features needed when parsing args. These are checked + * after flag parsing is completed in order to support enabling + * experimental features coming after the flag that needs the + * experimental feature. + */ + std::set flagExperimentalFeatures; + +private: + + std::optional needsCompletion(std::string_view s); +}; + +} diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc index 040464532b8..f678fae9476 100644 --- a/src/libutil/canon-path.cc +++ b/src/libutil/canon-path.cc @@ -1,5 +1,5 @@ #include "canon-path.hh" -#include "util.hh" +#include "file-system.hh" namespace nix { diff --git a/src/libutil/cgroup.cc b/src/libutil/cgroup.cc index a008481caa9..4c2bf31ffe1 100644 --- a/src/libutil/cgroup.cc +++ b/src/libutil/cgroup.cc @@ -2,6 +2,7 @@ #include "cgroup.hh" #include "util.hh" +#include "file-system.hh" #include "finally.hh" #include diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index ba0847cded2..d06f1f87bd8 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -1,6 +1,6 @@ #include "compression.hh" +#include "signals.hh" #include "tarfile.hh" -#include "util.hh" #include "finally.hh" #include "logging.hh" diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh index b9639e76179..9f69e844417 100644 --- a/src/libutil/config-impl.hh +++ b/src/libutil/config-impl.hh @@ -45,13 +45,13 @@ bool BaseSetting::isAppendable() return trait::appendable; } -template<> void BaseSetting::appendOrSet(Strings && newValue, bool append); -template<> void BaseSetting::appendOrSet(StringSet && newValue, bool append); -template<> void BaseSetting::appendOrSet(StringMap && newValue, bool append); -template<> void BaseSetting>::appendOrSet(std::set && newValue, bool append); +template<> void BaseSetting::appendOrSet(Strings newValue, bool append); +template<> void BaseSetting::appendOrSet(StringSet newValue, bool append); +template<> void BaseSetting::appendOrSet(StringMap newValue, bool append); +template<> void BaseSetting>::appendOrSet(std::set newValue, bool append); template -void BaseSetting::appendOrSet(T && newValue, bool append) +void BaseSetting::appendOrSet(T newValue, bool append) { static_assert( !trait::appendable, diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 38d406e8a3c..96a0a4df87f 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -2,6 +2,8 @@ #include "args.hh" #include "abstract-setting-to-json.hh" #include "experimental-features.hh" +#include "util.hh" +#include "file-system.hh" #include "config-impl.hh" @@ -9,6 +11,10 @@ namespace nix { +Config::Config(StringMap initials) + : AbstractConfig(std::move(initials)) +{ } + bool Config::set(const std::string & name, const std::string & value) { bool append = false; @@ -29,28 +35,26 @@ bool Config::set(const std::string & name, const std::string & value) void Config::addSetting(AbstractSetting * setting) { - _settings.emplace(setting->name, Config::SettingData(false, setting)); - for (auto & alias : setting->aliases) - _settings.emplace(alias, Config::SettingData(true, setting)); + _settings.emplace(setting->name, Config::SettingData{false, setting}); + for (const auto & alias : setting->aliases) + _settings.emplace(alias, Config::SettingData{true, setting}); bool set = false; - auto i = unknownSettings.find(setting->name); - if (i != unknownSettings.end()) { - setting->set(i->second); + if (auto i = unknownSettings.find(setting->name); i != unknownSettings.end()) { + setting->set(std::move(i->second)); setting->overridden = true; unknownSettings.erase(i); set = true; } for (auto & alias : setting->aliases) { - auto i = unknownSettings.find(alias); - if (i != unknownSettings.end()) { + if (auto i = unknownSettings.find(alias); i != unknownSettings.end()) { if (set) warn("setting '%s' is set, but it's an alias of '%s' which is also set", alias, setting->name); else { - setting->set(i->second); + setting->set(std::move(i->second)); setting->overridden = true; unknownSettings.erase(i); set = true; @@ -59,22 +63,27 @@ void Config::addSetting(AbstractSetting * setting) } } +AbstractConfig::AbstractConfig(StringMap initials) + : unknownSettings(std::move(initials)) +{ } + void AbstractConfig::warnUnknownSettings() { - for (auto & s : unknownSettings) + for (const auto & s : unknownSettings) warn("unknown setting '%s'", s.first); } void AbstractConfig::reapplyUnknownSettings() { auto unknownSettings2 = std::move(unknownSettings); + unknownSettings = {}; for (auto & s : unknownSettings2) set(s.first, s.second); } void Config::getSettings(std::map & res, bool overriddenOnly) { - for (auto & opt : _settings) + for (const auto & opt : _settings) if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden)) res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } @@ -90,8 +99,7 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string line += contents[pos++]; pos++; - auto hash = line.find('#'); - if (hash != std::string::npos) + if (auto hash = line.find('#'); hash != line.npos) line = std::string(line, 0, hash); auto tokens = tokenizeString>(line); @@ -124,24 +132,24 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string if (tokens[1] != "=") throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); - std::string name = tokens[0]; + std::string name = std::move(tokens[0]); auto i = tokens.begin(); advance(i, 2); parsedContents.push_back({ - name, + std::move(name), concatStringsSep(" ", Strings(i, tokens.end())), }); }; // First apply experimental-feature related settings - for (auto & [name, value] : parsedContents) + for (const auto & [name, value] : parsedContents) if (name == "experimental-features" || name == "extra-experimental-features") set(name, value); // Then apply other settings - for (auto & [name, value] : parsedContents) + for (const auto & [name, value] : parsedContents) if (name != "experimental-features" && name != "extra-experimental-features") set(name, value); } @@ -163,7 +171,7 @@ void Config::resetOverridden() nlohmann::json Config::toJSON() { auto res = nlohmann::json::object(); - for (auto & s : _settings) + for (const auto & s : _settings) if (!s.second.isAlias) res.emplace(s.first, s.second.setting->toJSON()); return res; @@ -171,8 +179,8 @@ nlohmann::json Config::toJSON() std::string Config::toKeyValue() { - auto res = std::string(); - for (auto & s : _settings) + std::string res; + for (const auto & s : _settings) if (s.second.isAlias) res += fmt("%s = %s\n", s.first, s.second.setting->to_string()); return res; @@ -194,8 +202,15 @@ AbstractSetting::AbstractSetting( : name(name) , description(stripIndentation(description)) , aliases(aliases) - , experimentalFeature(experimentalFeature) + , experimentalFeature(std::move(experimentalFeature)) +{ +} + +AbstractSetting::~AbstractSetting() { + // Check against a gcc miscompilation causing our constructor + // not to run (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431). + assert(created == 123); } nlohmann::json AbstractSetting::toJSON() @@ -203,7 +218,7 @@ nlohmann::json AbstractSetting::toJSON() return nlohmann::json(toJSONObject()); } -std::map AbstractSetting::toJSONObject() +std::map AbstractSetting::toJSONObject() const { std::map obj; obj.emplace("description", description); @@ -219,6 +234,9 @@ void AbstractSetting::convertToArg(Args & args, const std::string & category) { } + +bool AbstractSetting::isOverridden() const { return overridden; } + template<> std::string BaseSetting::parse(const std::string & str) const { return str; @@ -263,14 +281,14 @@ template<> void BaseSetting::convertToArg(Args & args, const std::string & .longName = name, .description = fmt("Enable the `%s` setting.", name), .category = category, - .handler = {[this]() { override(true); }}, + .handler = {[this] { override(true); }}, .experimentalFeature = experimentalFeature, }); args.addFlag({ .longName = "no-" + name, .description = fmt("Disable the `%s` setting.", name), .category = category, - .handler = {[this]() { override(false); }}, + .handler = {[this] { override(false); }}, .experimentalFeature = experimentalFeature, }); } @@ -280,10 +298,11 @@ template<> Strings BaseSetting::parse(const std::string & str) const return tokenizeString(str); } -template<> void BaseSetting::appendOrSet(Strings && newValue, bool append) +template<> void BaseSetting::appendOrSet(Strings newValue, bool append) { if (!append) value.clear(); - for (auto && s : std::move(newValue)) value.push_back(std::move(s)); + value.insert(value.end(), std::make_move_iterator(newValue.begin()), + std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting::to_string() const @@ -296,11 +315,10 @@ template<> StringSet BaseSetting::parse(const std::string & str) cons return tokenizeString(str); } -template<> void BaseSetting::appendOrSet(StringSet && newValue, bool append) +template<> void BaseSetting::appendOrSet(StringSet newValue, bool append) { if (!append) value.clear(); - for (auto && s : std::move(newValue)) - value.insert(s); + value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting::to_string() const @@ -312,26 +330,26 @@ template<> std::set BaseSetting res; for (auto & s : tokenizeString(str)) { - auto thisXpFeature = parseExperimentalFeature(s); - if (thisXpFeature) + if (auto thisXpFeature = parseExperimentalFeature(s); thisXpFeature) { res.insert(thisXpFeature.value()); - else + if (thisXpFeature.value() == Xp::Flakes) + res.insert(Xp::FetchTree); + } else warn("unknown experimental feature '%s'", s); } return res; } -template<> void BaseSetting>::appendOrSet(std::set && newValue, bool append) +template<> void BaseSetting>::appendOrSet(std::set newValue, bool append) { if (!append) value.clear(); - for (auto && s : std::move(newValue)) - value.insert(s); + value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting>::to_string() const { StringSet stringifiedXpFeatures; - for (auto & feature : value) + for (const auto & feature : value) stringifiedXpFeatures.insert(std::string(showExperimentalFeature(feature))); return concatStringsSep(" ", stringifiedXpFeatures); } @@ -339,28 +357,25 @@ template<> std::string BaseSetting>::to_string() c template<> StringMap BaseSetting::parse(const std::string & str) const { StringMap res; - for (auto & s : tokenizeString(str)) { - auto eq = s.find_first_of('='); - if (std::string::npos != eq) + for (const auto & s : tokenizeString(str)) { + if (auto eq = s.find_first_of('='); s.npos != eq) res.emplace(std::string(s, 0, eq), std::string(s, eq + 1)); // else ignored } return res; } -template<> void BaseSetting::appendOrSet(StringMap && newValue, bool append) +template<> void BaseSetting::appendOrSet(StringMap newValue, bool append) { if (!append) value.clear(); - for (auto && [k, v] : std::move(newValue)) - value.emplace(std::move(k), std::move(v)); + value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting::to_string() const { - Strings kvstrs; - std::transform(value.begin(), value.end(), back_inserter(kvstrs), - [&](auto kvpair){ return kvpair.first + "=" + kvpair.second; }); - return concatStringsSep(" ", kvstrs); + return std::transform_reduce(value.cbegin(), value.cend(), std::string{}, + [](const auto & l, const auto &r) { return l + " " + r; }, + [](const auto & kvpair){ return kvpair.first + "=" + kvpair.second; }); } template class BaseSetting; @@ -384,11 +399,33 @@ static Path parsePath(const AbstractSetting & s, const std::string & str) return canonPath(str); } +PathSetting::PathSetting(Config * options, + const Path & def, + const std::string & name, + const std::string & description, + const std::set & aliases) + : BaseSetting(def, true, name, description, aliases) +{ + options->addSetting(this); +} + Path PathSetting::parse(const std::string & str) const { return parsePath(*this, str); } + +OptionalPathSetting::OptionalPathSetting(Config * options, + const std::optional & def, + const std::string & name, + const std::string & description, + const std::set & aliases) + : BaseSetting>(def, true, name, description, aliases) +{ + options->addSetting(this); +} + + std::optional OptionalPathSetting::parse(const std::string & str) const { if (str == "") @@ -397,6 +434,11 @@ std::optional OptionalPathSetting::parse(const std::string & str) const return parsePath(*this, str); } +void OptionalPathSetting::operator =(const std::optional & v) +{ + this->assign(v); +} + bool GlobalConfig::set(const std::string & name, const std::string & value) { for (auto & config : *configRegistrations) @@ -422,7 +464,7 @@ void GlobalConfig::resetOverridden() nlohmann::json GlobalConfig::toJSON() { auto res = nlohmann::json::object(); - for (auto & config : *configRegistrations) + for (const auto & config : *configRegistrations) res.update(config->toJSON()); return res; } @@ -432,7 +474,7 @@ std::string GlobalConfig::toKeyValue() std::string res; std::map settings; globalConfig.getSettings(settings); - for (auto & s : settings) + for (const auto & s : settings) res += fmt("%s = %s\n", s.first, s.second.value); return res; } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index cc8532587cc..5d7bd8e0cdb 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -36,8 +36,8 @@ namespace nix { * * std::map settings; * config.getSettings(settings); - * config["system"].description == "the current system" - * config["system"].value == "x86_64-linux" + * settings["system"].description == "the current system" + * settings["system"].value == "x86_64-linux" * * * The above retrieves all currently known settings from the `Config` object @@ -52,9 +52,7 @@ class AbstractConfig protected: StringMap unknownSettings; - AbstractConfig(const StringMap & initials = {}) - : unknownSettings(initials) - { } + AbstractConfig(StringMap initials = {}); public: @@ -150,12 +148,9 @@ public: { bool isAlias; AbstractSetting * setting; - SettingData(bool isAlias, AbstractSetting * setting) - : isAlias(isAlias), setting(setting) - { } }; - typedef std::map Settings; + using Settings = std::map; private: @@ -163,9 +158,7 @@ private: public: - Config(const StringMap & initials = {}) - : AbstractConfig(initials) - { } + Config(StringMap initials = {}); bool set(const std::string & name, const std::string & value) override; @@ -206,12 +199,7 @@ protected: const std::set & aliases, std::optional experimentalFeature = std::nullopt); - virtual ~AbstractSetting() - { - // Check against a gcc miscompilation causing our constructor - // not to run (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431). - assert(created == 123); - } + virtual ~AbstractSetting(); virtual void set(const std::string & value, bool append = false) = 0; @@ -225,11 +213,11 @@ protected: nlohmann::json toJSON(); - virtual std::map toJSONObject(); + virtual std::map toJSONObject() const; virtual void convertToArg(Args & args, const std::string & category); - bool isOverridden() const { return overridden; } + bool isOverridden() const; }; /** @@ -259,7 +247,7 @@ protected: * * @param append Whether to append or overwrite. */ - virtual void appendOrSet(T && newValue, bool append); + virtual void appendOrSet(T newValue, bool append); public: @@ -318,18 +306,17 @@ public: void convertToArg(Args & args, const std::string & category) override; - std::map toJSONObject() override; + std::map toJSONObject() const override; }; template std::ostream & operator <<(std::ostream & str, const BaseSetting & opt) { - str << (const T &) opt; - return str; + return str << static_cast(opt); } template -bool operator ==(const T & v1, const BaseSetting & v2) { return v1 == (const T &) v2; } +bool operator ==(const T & v1, const BaseSetting & v2) { return v1 == static_cast(v2); } template class Setting : public BaseSetting @@ -342,7 +329,7 @@ public: const std::set & aliases = {}, const bool documentDefault = true, std::optional experimentalFeature = std::nullopt) - : BaseSetting(def, documentDefault, name, description, aliases, experimentalFeature) + : BaseSetting(def, documentDefault, name, description, aliases, std::move(experimentalFeature)) { options->addSetting(this); } @@ -365,11 +352,7 @@ public: const Path & def, const std::string & name, const std::string & description, - const std::set & aliases = {}) - : BaseSetting(def, true, name, description, aliases) - { - options->addSetting(this); - } + const std::set & aliases = {}); Path parse(const std::string & str) const override; @@ -391,15 +374,11 @@ public: const std::optional & def, const std::string & name, const std::string & description, - const std::set & aliases = {}) - : BaseSetting>(def, true, name, description, aliases) - { - options->addSetting(this); - } + const std::set & aliases = {}); std::optional parse(const std::string & str) const override; - void operator =(const std::optional & v) { this->assign(v); } + void operator =(const std::optional & v); }; struct GlobalConfig : public AbstractConfig diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc new file mode 100644 index 00000000000..352a6a0fb36 --- /dev/null +++ b/src/libutil/current-process.cc @@ -0,0 +1,110 @@ +#include "current-process.hh" +#include "namespaces.hh" +#include "util.hh" +#include "finally.hh" +#include "file-system.hh" +#include "processes.hh" +#include "signals.hh" + +#ifdef __APPLE__ +# include +#endif + +#if __linux__ +# include +# include +# include "cgroup.hh" +#endif + +#include + +namespace nix { + +unsigned int getMaxCPU() +{ + #if __linux__ + try { + auto cgroupFS = getCgroupFS(); + if (!cgroupFS) return 0; + + auto cgroups = getCgroups("/proc/self/cgroup"); + auto cgroup = cgroups[""]; + if (cgroup == "") return 0; + + auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max"; + + auto cpuMax = readFile(cpuFile); + auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); + auto quota = cpuMaxParts[0]; + auto period = cpuMaxParts[1]; + if (quota != "max") + return std::ceil(std::stoi(quota) / std::stof(period)); + } catch (Error &) { ignoreException(lvlDebug); } + #endif + + return 0; +} + + +////////////////////////////////////////////////////////////////////// + + +#if __linux__ +rlim_t savedStackSize = 0; +#endif + +void setStackSize(size_t stackSize) +{ + #if __linux__ + struct rlimit limit; + if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { + savedStackSize = limit.rlim_cur; + limit.rlim_cur = stackSize; + setrlimit(RLIMIT_STACK, &limit); + } + #endif +} + +void restoreProcessContext(bool restoreMounts) +{ + restoreSignals(); + if (restoreMounts) { + restoreMountNamespace(); + } + + #if __linux__ + if (savedStackSize) { + struct rlimit limit; + if (getrlimit(RLIMIT_STACK, &limit) == 0) { + limit.rlim_cur = savedStackSize; + setrlimit(RLIMIT_STACK, &limit); + } + } + #endif +} + + +////////////////////////////////////////////////////////////////////// + + +std::optional getSelfExe() +{ + static auto cached = []() -> std::optional + { + #if __linux__ + return readLink("/proc/self/exe"); + #elif __APPLE__ + char buf[1024]; + uint32_t size = sizeof(buf); + if (_NSGetExecutablePath(buf, &size) == 0) + return buf; + else + return std::nullopt; + #else + return std::nullopt; + #endif + }(); + return cached; +} + +} diff --git a/src/libutil/current-process.hh b/src/libutil/current-process.hh new file mode 100644 index 00000000000..826d6fe2006 --- /dev/null +++ b/src/libutil/current-process.hh @@ -0,0 +1,34 @@ +#pragma once +///@file + +#include + +#include "types.hh" + +namespace nix { + +/** + * If cgroups are active, attempt to calculate the number of CPUs available. + * If cgroups are unavailable or if cpu.max is set to "max", return 0. + */ +unsigned int getMaxCPU(); + +/** + * Change the stack size. + */ +void setStackSize(size_t stackSize); + +/** + * Restore the original inherited Unix process context (such as signal + * masks, stack size). + + * See startSignalHandlerThread(), saveSignalMask(). + */ +void restoreProcessContext(bool restoreMounts = true); + +/** + * @return the path of the current executable. + */ +std::optional getSelfExe(); + +} diff --git a/src/libutil/environment-variables.cc b/src/libutil/environment-variables.cc new file mode 100644 index 00000000000..6618d787271 --- /dev/null +++ b/src/libutil/environment-variables.cc @@ -0,0 +1,49 @@ +#include "util.hh" +#include "environment-variables.hh" + +extern char * * environ __attribute__((weak)); + +namespace nix { + +std::optional getEnv(const std::string & key) +{ + char * value = getenv(key.c_str()); + if (!value) return {}; + return std::string(value); +} + +std::optional getEnvNonEmpty(const std::string & key) { + auto value = getEnv(key); + if (value == "") return {}; + return value; +} + +std::map getEnv() +{ + std::map env; + for (size_t i = 0; environ[i]; ++i) { + auto s = environ[i]; + auto eq = strchr(s, '='); + if (!eq) + // invalid env, just keep going + continue; + env.emplace(std::string(s, eq), std::string(eq + 1)); + } + return env; +} + + +void clearEnv() +{ + for (auto & name : getEnv()) + unsetenv(name.first.c_str()); +} + +void replaceEnv(const std::map & newEnv) +{ + clearEnv(); + for (auto & newEnvVar : newEnv) + setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); +} + +} diff --git a/src/libutil/environment-variables.hh b/src/libutil/environment-variables.hh new file mode 100644 index 00000000000..21eb4619b28 --- /dev/null +++ b/src/libutil/environment-variables.hh @@ -0,0 +1,41 @@ +#pragma once +/** + * @file + * + * Utilities for working with the current process's environment + * variables. + */ + +#include + +#include "types.hh" + +namespace nix { + +/** + * @return an environment variable. + */ +std::optional getEnv(const std::string & key); + +/** + * @return a non empty environment variable. Returns nullopt if the env + * variable is set to "" + */ +std::optional getEnvNonEmpty(const std::string & key); + +/** + * Get the entire environment. + */ +std::map getEnv(); + +/** + * Clear the environment. + */ +void clearEnv(); + +/** + * Replace the entire environment with the given one. + */ +void replaceEnv(const std::map & newEnv); + +} diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 4a1b346ef4c..8488e7e219e 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -1,4 +1,7 @@ #include "error.hh" +#include "environment-variables.hh" +#include "signals.hh" +#include "terminal.hh" #include #include @@ -7,8 +10,6 @@ namespace nix { -const std::string nativeSystem = SYSTEM; - void BaseError::addTrace(std::shared_ptr && e, hintformat hint, bool frame) { err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .frame = frame }); @@ -155,6 +156,36 @@ static std::string indent(std::string_view indentFirst, std::string_view indentR return res; } +/** + * A development aid for finding missing positions, to improve error messages. Example use: + * + * NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS=1 _NIX_TEST_ACCEPT=1 make tests/lang.sh.test + * git diff -U20 tests + * + */ +static bool printUnknownLocations = getEnv("_NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS").has_value(); + +/** + * Print a position, if it is known. + * + * @return true if a position was printed. + */ +static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std::shared_ptr & pos) { + bool hasPos = pos && *pos; + if (hasPos) { + oss << "\n" << indent << ANSI_BLUE << "at " ANSI_WARNING << *pos << ANSI_NORMAL << ":"; + + if (auto loc = pos->getCodeLines()) { + oss << "\n"; + printCodeLines(oss, "", *pos, *loc); + oss << "\n"; + } + } else if (printUnknownLocations) { + oss << "\n" << indent << ANSI_BLUE << "at " ANSI_RED << "UNKNOWN LOCATION" << ANSI_NORMAL << "\n"; + } + return hasPos; +} + std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace) { std::string prefix; @@ -203,8 +234,6 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s std::ostringstream oss; - auto noSource = ANSI_ITALIC " (source not available)" ANSI_NORMAL "\n"; - /* * Traces * ------ @@ -320,34 +349,15 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s oss << "\n" << "… " << trace.hint.str() << "\n"; - if (trace.pos) { + if (printPosMaybe(oss, ellipsisIndent, trace.pos)) count++; - - oss << "\n" << ellipsisIndent << ANSI_BLUE << "at " ANSI_WARNING << *trace.pos << ANSI_NORMAL << ":"; - - if (auto loc = trace.pos->getCodeLines()) { - oss << "\n"; - printCodeLines(oss, "", *trace.pos, *loc); - oss << "\n"; - } else - oss << noSource; - } } oss << "\n" << prefix; } oss << einfo.msg << "\n"; - if (einfo.errPos) { - oss << "\n" << ANSI_BLUE << "at " ANSI_WARNING << *einfo.errPos << ANSI_NORMAL << ":"; - - if (auto loc = einfo.errPos->getCodeLines()) { - oss << "\n"; - printCodeLines(oss, "", *einfo.errPos, *loc); - oss << "\n"; - } else - oss << noSource; - } + printPosMaybe(oss, "", einfo.errPos); auto suggestions = einfo.suggestions.trim(); if (!suggestions.suggestions.empty()) { diff --git a/src/libutil/error.hh b/src/libutil/error.hh index be5c5e252bf..c04dcbd77b2 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -70,6 +70,13 @@ struct AbstractPos uint32_t line = 0; uint32_t column = 0; + /** + * An AbstractPos may be a "null object", representing an unknown position. + * + * Return true if this position is known. + */ + inline operator bool() const { return line != 0; }; + /** * Return the contents of the source file. */ diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 782331283ca..86665cd9d73 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,19 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +/** + * If two different PRs both add an experimental feature, and we just + * used a number for this, we *woudln't* get merge conflict and the + * counter will be incremented once instead of twice, causing a build + * failure. + * + * By instead defining this instead as 1 + the bottom experimental + * feature, we either have no issue at all if few features are not added + * at the end of the list, or a proper merge conflict if they are. + */ +constexpr size_t numXpFeatures = 1 + static_cast(Xp::VerifiedFetches); + +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -62,6 +74,19 @@ constexpr std::array xpFeatureDetails = {{ flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details. )", }, + { + .tag = Xp::FetchTree, + .name = "fetch-tree", + .description = R"( + Enable the use of the [`fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) built-in function in the Nix language. + + `fetchTree` exposes a generic interface for fetching remote file system trees from different types of remote sources. + The [`flakes`](#xp-feature-flakes) feature flag always enables `fetch-tree`. + This built-in was previously guarded by the `flakes` experimental feature because of that overlap. + + Enabling just this feature serves as a "release candidate", allowing users to try it out in isolation. + )", + }, { .tag = Xp::NixCommand, .name = "nix-command", @@ -70,6 +95,14 @@ constexpr std::array xpFeatureDetails = {{ [`nix`](@docroot@/command-ref/new-cli/nix.md) for details. )", }, + { + .tag = Xp::GitHashing, + .name = "git-hashing", + .description = R"( + Allow creating (content-addressed) store objects which are hashed via Git's hashing algorithm. + These store objects will not be understandable by older versions of Nix. + )", + }, { .tag = Xp::RecursiveNix, .name = "recursive-nix", @@ -163,6 +196,8 @@ constexpr std::array xpFeatureDetails = {{ .tag = Xp::ReplFlake, .name = "repl-flake", .description = R"( + *Enabled with [`flakes`](#xp-feature-flakes) since 2.19* + Allow passing [installables](@docroot@/command-ref/new-cli/nix.md#installables) to `nix repl`, making its interface consistent with the other experimental commands. )", }, @@ -171,7 +206,7 @@ constexpr std::array xpFeatureDetails = {{ .name = "auto-allocate-uids", .description = R"( Allows Nix to automatically pick UIDs for builds, rather than creating - `nixbld*` user accounts. See the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting for details. + `nixbld*` user accounts. See the [`auto-allocate-uids`](@docroot@/command-ref/conf-file.md#conf-auto-allocate-uids) setting for details. )", }, { @@ -179,7 +214,7 @@ constexpr std::array xpFeatureDetails = {{ .name = "cgroups", .description = R"( Allows Nix to execute builds inside cgroups. See - the [`use-cgroups`](#conf-use-cgroups) setting for details. + the [`use-cgroups`](@docroot@/command-ref/conf-file.md#conf-use-cgroups) setting for details. )", }, { @@ -219,6 +254,20 @@ constexpr std::array xpFeatureDetails = {{ Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs. )", }, + { + .tag = Xp::ConfigurableImpureEnv, + .name = "configurable-impure-env", + .description = R"( + Allow the use of the [impure-env](@docroot@/command-ref/conf-file.md#conf-impure-env) setting. + )", + }, + { + .tag = Xp::VerifiedFetches, + .name = "verified-fetches", + .description = R"( + Enables verification of git commit signatures through the [`fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) built-in. + )", + }, }}; static_assert( @@ -272,7 +321,7 @@ std::set parseFeatures(const std::set & rawFea } MissingExperimentalFeature::MissingExperimentalFeature(ExperimentalFeature feature) - : Error("experimental Nix feature '%1%' is disabled; use '--extra-experimental-features %1%' to override", showExperimentalFeature(feature)) + : Error("experimental Nix feature '%1%' is disabled; add '--extra-experimental-features %1%' to enable it", showExperimentalFeature(feature)) , missingFeature(feature) {} diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index add592ae624..c355b8081e6 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -20,7 +20,9 @@ enum struct ExperimentalFeature CaDerivations, ImpureDerivations, Flakes, + FetchTree, NixCommand, + GitHashing, RecursiveNix, NoUrlLiterals, FetchClosure, @@ -31,6 +33,8 @@ enum struct ExperimentalFeature DynamicDerivations, ParseTomlTimestamps, ReadOnlyLocalStore, + ConfigurableImpureEnv, + VerifiedFetches, }; /** diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc new file mode 100644 index 00000000000..38dd70c8e4c --- /dev/null +++ b/src/libutil/file-descriptor.cc @@ -0,0 +1,254 @@ +#include "file-system.hh" +#include "signals.hh" +#include "finally.hh" +#include "serialise.hh" + +#include +#include + +namespace nix { + +std::string readFile(int fd) +{ + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError("statting file"); + + return drainFD(fd, true, st.st_size); +} + + +void readFull(int fd, char * buf, size_t count) +{ + while (count) { + checkInterrupt(); + ssize_t res = read(fd, buf, count); + if (res == -1) { + if (errno == EINTR) continue; + throw SysError("reading from file"); + } + if (res == 0) throw EndOfFile("unexpected end-of-file"); + count -= res; + buf += res; + } +} + + +void writeFull(int fd, std::string_view s, bool allowInterrupts) +{ + while (!s.empty()) { + if (allowInterrupts) checkInterrupt(); + ssize_t res = write(fd, s.data(), s.size()); + if (res == -1 && errno != EINTR) + throw SysError("writing to file"); + if (res > 0) + s.remove_prefix(res); + } +} + + +std::string readLine(int fd) +{ + std::string s; + while (1) { + checkInterrupt(); + char ch; + // FIXME: inefficient + ssize_t rd = read(fd, &ch, 1); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading a line"); + } else if (rd == 0) + throw EndOfFile("unexpected EOF reading a line"); + else { + if (ch == '\n') return s; + s += ch; + } + } +} + + +void writeLine(int fd, std::string s) +{ + s += '\n'; + writeFull(fd, s); +} + + +std::string drainFD(int fd, bool block, const size_t reserveSize) +{ + // the parser needs two extra bytes to append terminating characters, other users will + // not care very much about the extra memory. + StringSink sink(reserveSize + 2); + drainFD(fd, sink, block); + return std::move(sink.s); +} + + +void drainFD(int fd, Sink & sink, bool block) +{ + // silence GCC maybe-uninitialized warning in finally + int saved = 0; + + if (!block) { + saved = fcntl(fd, F_GETFL); + if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) + throw SysError("making file descriptor non-blocking"); + } + + Finally finally([&]() { + if (!block) { + if (fcntl(fd, F_SETFL, saved) == -1) + throw SysError("making file descriptor blocking"); + } + }); + + std::vector buf(64 * 1024); + while (1) { + checkInterrupt(); + ssize_t rd = read(fd, buf.data(), buf.size()); + if (rd == -1) { + if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) + break; + if (errno != EINTR) + throw SysError("reading from file"); + } + else if (rd == 0) break; + else sink({(char *) buf.data(), (size_t) rd}); + } +} + +////////////////////////////////////////////////////////////////////// + +AutoCloseFD::AutoCloseFD() : fd{-1} {} + + +AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} + + +AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd} +{ + that.fd = -1; +} + + +AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that) +{ + close(); + fd = that.fd; + that.fd = -1; + return *this; +} + + +AutoCloseFD::~AutoCloseFD() +{ + try { + close(); + } catch (...) { + ignoreException(); + } +} + + +int AutoCloseFD::get() const +{ + return fd; +} + + +void AutoCloseFD::close() +{ + if (fd != -1) { + if (::close(fd) == -1) + /* This should never happen. */ + throw SysError("closing file descriptor %1%", fd); + fd = -1; + } +} + +void AutoCloseFD::fsync() +{ + if (fd != -1) { + int result; +#if __APPLE__ + result = ::fcntl(fd, F_FULLFSYNC); +#else + result = ::fsync(fd); +#endif + if (result == -1) + throw SysError("fsync file descriptor %1%", fd); + } +} + + +AutoCloseFD::operator bool() const +{ + return fd != -1; +} + + +int AutoCloseFD::release() +{ + int oldFD = fd; + fd = -1; + return oldFD; +} + + +void Pipe::create() +{ + int fds[2]; +#if HAVE_PIPE2 + if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); +#else + if (pipe(fds) != 0) throw SysError("creating pipe"); + closeOnExec(fds[0]); + closeOnExec(fds[1]); +#endif + readSide = fds[0]; + writeSide = fds[1]; +} + + +void Pipe::close() +{ + readSide.close(); + writeSide.close(); +} + +////////////////////////////////////////////////////////////////////// + +void closeMostFDs(const std::set & exceptions) +{ +#if __linux__ + try { + for (auto & s : readDirectory("/proc/self/fd")) { + auto fd = std::stoi(s.name); + if (!exceptions.count(fd)) { + debug("closing leaked FD %d", fd); + close(fd); + } + } + return; + } catch (SysError &) { + } +#endif + + int maxFD = 0; + maxFD = sysconf(_SC_OPEN_MAX); + for (int fd = 0; fd < maxFD; ++fd) + if (!exceptions.count(fd)) + close(fd); /* ignore result */ +} + + +void closeOnExec(int fd) +{ + int prev; + if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || + fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) + throw SysError("setting close-on-exec flag"); +} + +} diff --git a/src/libutil/file-descriptor.hh b/src/libutil/file-descriptor.hh new file mode 100644 index 00000000000..80ec8613592 --- /dev/null +++ b/src/libutil/file-descriptor.hh @@ -0,0 +1,84 @@ +#pragma once +///@file + +#include "types.hh" +#include "error.hh" + +namespace nix { + +struct Sink; +struct Source; + +/** + * Read the contents of a resource into a string. + */ +std::string readFile(int fd); + +/** + * Wrappers arount read()/write() that read/write exactly the + * requested number of bytes. + */ +void readFull(int fd, char * buf, size_t count); + +void writeFull(int fd, std::string_view s, bool allowInterrupts = true); + +/** + * Read a line from a file descriptor. + */ +std::string readLine(int fd); + +/** + * Write a line to a file descriptor. + */ +void writeLine(int fd, std::string s); + +/** + * Read a file descriptor until EOF occurs. + */ +std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); + +void drainFD(int fd, Sink & sink, bool block = true); + +/** + * Automatic cleanup of resources. + */ +class AutoCloseFD +{ + int fd; +public: + AutoCloseFD(); + AutoCloseFD(int fd); + AutoCloseFD(const AutoCloseFD & fd) = delete; + AutoCloseFD(AutoCloseFD&& fd); + ~AutoCloseFD(); + AutoCloseFD& operator =(const AutoCloseFD & fd) = delete; + AutoCloseFD& operator =(AutoCloseFD&& fd); + int get() const; + explicit operator bool() const; + int release(); + void close(); + void fsync(); +}; + +class Pipe +{ +public: + AutoCloseFD readSide, writeSide; + void create(); + void close(); +}; + +/** + * Close all file descriptors except those listed in the given set. + * Good practice in child processes. + */ +void closeMostFDs(const std::set & exceptions); + +/** + * Set the close-on-exec flag for the given file descriptor. + */ +void closeOnExec(int fd); + +MakeError(EndOfFile, Error); + +} diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc new file mode 100644 index 00000000000..c96effff919 --- /dev/null +++ b/src/libutil/file-system.cc @@ -0,0 +1,647 @@ +#include "environment-variables.hh" +#include "file-system.hh" +#include "signals.hh" +#include "finally.hh" +#include "serialise.hh" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace nix { + +Path absPath(Path path, std::optional dir, bool resolveSymlinks) +{ + if (path[0] != '/') { + if (!dir) { +#ifdef __GNU__ + /* GNU (aka. GNU/Hurd) doesn't have any limitation on path + lengths and doesn't define `PATH_MAX'. */ + char *buf = getcwd(NULL, 0); + if (buf == NULL) +#else + char buf[PATH_MAX]; + if (!getcwd(buf, sizeof(buf))) +#endif + throw SysError("cannot get cwd"); + path = concatStrings(buf, "/", path); +#ifdef __GNU__ + free(buf); +#endif + } else + path = concatStrings(*dir, "/", path); + } + return canonPath(path, resolveSymlinks); +} + + +Path canonPath(PathView path, bool resolveSymlinks) +{ + assert(path != ""); + + std::string s; + s.reserve(256); + + if (path[0] != '/') + throw Error("not an absolute path: '%1%'", path); + + std::string temp; + + /* Count the number of times we follow a symlink and stop at some + arbitrary (but high) limit to prevent infinite loops. */ + unsigned int followCount = 0, maxFollow = 1024; + + while (1) { + + /* Skip slashes. */ + while (!path.empty() && path[0] == '/') path.remove_prefix(1); + if (path.empty()) break; + + /* Ignore `.'. */ + if (path == "." || path.substr(0, 2) == "./") + path.remove_prefix(1); + + /* If `..', delete the last component. */ + else if (path == ".." || path.substr(0, 3) == "../") + { + if (!s.empty()) s.erase(s.rfind('/')); + path.remove_prefix(2); + } + + /* Normal component; copy it. */ + else { + s += '/'; + if (const auto slash = path.find('/'); slash == std::string::npos) { + s += path; + path = {}; + } else { + s += path.substr(0, slash); + path = path.substr(slash); + } + + /* If s points to a symlink, resolve it and continue from there */ + if (resolveSymlinks && isLink(s)) { + if (++followCount >= maxFollow) + throw Error("infinite symlink recursion in path '%1%'", path); + temp = concatStrings(readLink(s), path); + path = temp; + if (!temp.empty() && temp[0] == '/') { + s.clear(); /* restart for symlinks pointing to absolute path */ + } else { + s = dirOf(s); + if (s == "/") { // we don’t want trailing slashes here, which dirOf only produces if s = / + s.clear(); + } + } + } + } + } + + return s.empty() ? "/" : std::move(s); +} + + +Path dirOf(const PathView path) +{ + Path::size_type pos = path.rfind('/'); + if (pos == std::string::npos) + return "."; + return pos == 0 ? "/" : Path(path, 0, pos); +} + + +std::string_view baseNameOf(std::string_view path) +{ + if (path.empty()) + return ""; + + auto last = path.size() - 1; + if (path[last] == '/' && last > 0) + last -= 1; + + auto pos = path.rfind('/', last); + if (pos == std::string::npos) + pos = 0; + else + pos += 1; + + return path.substr(pos, last - pos + 1); +} + + +bool isInDir(std::string_view path, std::string_view dir) +{ + return path.substr(0, 1) == "/" + && path.substr(0, dir.size()) == dir + && path.size() >= dir.size() + 2 + && path[dir.size()] == '/'; +} + + +bool isDirOrInDir(std::string_view path, std::string_view dir) +{ + return path == dir || isInDir(path, dir); +} + + +struct stat stat(const Path & path) +{ + struct stat st; + if (stat(path.c_str(), &st)) + throw SysError("getting status of '%1%'", path); + return st; +} + + +struct stat lstat(const Path & path) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError("getting status of '%1%'", path); + return st; +} + + +bool pathExists(const Path & path) +{ + int res; + struct stat st; + res = lstat(path.c_str(), &st); + if (!res) return true; + if (errno != ENOENT && errno != ENOTDIR) + throw SysError("getting status of %1%", path); + return false; +} + +bool pathAccessible(const Path & path) +{ + try { + return pathExists(path); + } catch (SysError & e) { + // swallow EPERM + if (e.errNo == EPERM) return false; + throw; + } +} + + +Path readLink(const Path & path) +{ + checkInterrupt(); + std::vector buf; + for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) { + buf.resize(bufSize); + ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize); + if (rlSize == -1) + if (errno == EINVAL) + throw Error("'%1%' is not a symlink", path); + else + throw SysError("reading symbolic link '%1%'", path); + else if (rlSize < bufSize) + return std::string(buf.data(), rlSize); + } +} + + +bool isLink(const Path & path) +{ + struct stat st = lstat(path); + return S_ISLNK(st.st_mode); +} + + +DirEntries readDirectory(DIR *dir, const Path & path) +{ + DirEntries entries; + entries.reserve(64); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir)) { /* sic */ + checkInterrupt(); + std::string name = dirent->d_name; + if (name == "." || name == "..") continue; + entries.emplace_back(name, dirent->d_ino, +#ifdef HAVE_STRUCT_DIRENT_D_TYPE + dirent->d_type +#else + DT_UNKNOWN +#endif + ); + } + if (errno) throw SysError("reading directory '%1%'", path); + + return entries; +} + +DirEntries readDirectory(const Path & path) +{ + AutoCloseDir dir(opendir(path.c_str())); + if (!dir) throw SysError("opening directory '%1%'", path); + + return readDirectory(dir.get(), path); +} + + +unsigned char getFileType(const Path & path) +{ + struct stat st = lstat(path); + if (S_ISDIR(st.st_mode)) return DT_DIR; + if (S_ISLNK(st.st_mode)) return DT_LNK; + if (S_ISREG(st.st_mode)) return DT_REG; + return DT_UNKNOWN; +} + + +std::string readFile(const Path & path) +{ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%1%'", path); + return readFile(fd.get()); +} + + +void readFile(const Path & path, Sink & sink) +{ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%s'", path); + drainFD(fd.get(), sink); +} + + +void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync) +{ + AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); + if (!fd) + throw SysError("opening file '%1%'", path); + try { + writeFull(fd.get(), s); + } catch (Error & e) { + e.addTrace({}, "writing file '%1%'", path); + throw; + } + if (sync) + fd.fsync(); + // Explicitly close to make sure exceptions are propagated. + fd.close(); + if (sync) + syncParent(path); +} + + +void writeFile(const Path & path, Source & source, mode_t mode, bool sync) +{ + AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); + if (!fd) + throw SysError("opening file '%1%'", path); + + std::vector buf(64 * 1024); + + try { + while (true) { + try { + auto n = source.read(buf.data(), buf.size()); + writeFull(fd.get(), {buf.data(), n}); + } catch (EndOfFile &) { break; } + } + } catch (Error & e) { + e.addTrace({}, "writing file '%1%'", path); + throw; + } + if (sync) + fd.fsync(); + // Explicitly close to make sure exceptions are propagated. + fd.close(); + if (sync) + syncParent(path); +} + +void syncParent(const Path & path) +{ + AutoCloseFD fd = open(dirOf(path).c_str(), O_RDONLY, 0); + if (!fd) + throw SysError("opening file '%1%'", path); + fd.fsync(); +} + + +static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) +{ + checkInterrupt(); + + std::string name(baseNameOf(path)); + + struct stat st; + if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { + if (errno == ENOENT) return; + throw SysError("getting status of '%1%'", path); + } + + if (!S_ISDIR(st.st_mode)) { + /* We are about to delete a file. Will it likely free space? */ + + switch (st.st_nlink) { + /* Yes: last link. */ + case 1: + bytesFreed += st.st_size; + break; + /* Maybe: yes, if 'auto-optimise-store' or manual optimisation + was performed. Instead of checking for real let's assume + it's an optimised file and space will be freed. + + In worst case we will double count on freed space for files + with exactly two hardlinks for unoptimised packages. + */ + case 2: + bytesFreed += st.st_size; + break; + /* No: 3+ links. */ + default: + break; + } + } + + if (S_ISDIR(st.st_mode)) { + /* Make the directory accessible. */ + const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; + if ((st.st_mode & PERM_MASK) != PERM_MASK) { + if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) + throw SysError("chmod '%1%'", path); + } + + int fd = openat(parentfd, path.c_str(), O_RDONLY); + if (fd == -1) + throw SysError("opening directory '%1%'", path); + AutoCloseDir dir(fdopendir(fd)); + if (!dir) + throw SysError("opening directory '%1%'", path); + for (auto & i : readDirectory(dir.get(), path)) + _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed); + } + + int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; + if (unlinkat(parentfd, name.c_str(), flags) == -1) { + if (errno == ENOENT) return; + throw SysError("cannot unlink '%1%'", path); + } +} + +static void _deletePath(const Path & path, uint64_t & bytesFreed) +{ + Path dir = dirOf(path); + if (dir == "") + dir = "/"; + + AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)}; + if (!dirfd) { + if (errno == ENOENT) return; + throw SysError("opening directory '%1%'", path); + } + + _deletePath(dirfd.get(), path, bytesFreed); +} + + +void deletePath(const Path & path) +{ + uint64_t dummy; + deletePath(path, dummy); +} + + +Paths createDirs(const Path & path) +{ + Paths created; + if (path == "/") return created; + + struct stat st; + if (lstat(path.c_str(), &st) == -1) { + created = createDirs(dirOf(path)); + if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) + throw SysError("creating directory '%1%'", path); + st = lstat(path); + created.push_back(path); + } + + if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) + throw SysError("statting symlink '%1%'", path); + + if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path); + + return created; +} + + +void deletePath(const Path & path, uint64_t & bytesFreed) +{ + //Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path); + bytesFreed = 0; + _deletePath(path, bytesFreed); +} + + +////////////////////////////////////////////////////////////////////// + +AutoDelete::AutoDelete() : del{false} {} + +AutoDelete::AutoDelete(const std::string & p, bool recursive) : path(p) +{ + del = true; + this->recursive = recursive; +} + +AutoDelete::~AutoDelete() +{ + try { + if (del) { + if (recursive) + deletePath(path); + else { + if (remove(path.c_str()) == -1) + throw SysError("cannot unlink '%1%'", path); + } + } + } catch (...) { + ignoreException(); + } +} + +void AutoDelete::cancel() +{ + del = false; +} + +void AutoDelete::reset(const Path & p, bool recursive) { + path = p; + this->recursive = recursive; + del = true; +} + +////////////////////////////////////////////////////////////////////// + +////////////////////////////////////////////////////////////////////// + +static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, + std::atomic & counter) +{ + tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); + if (includePid) + return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++); + else + return fmt("%1%/%2%-%3%", tmpRoot, prefix, counter++); +} + +Path createTempDir(const Path & tmpRoot, const Path & prefix, + bool includePid, bool useGlobalCounter, mode_t mode) +{ + static std::atomic globalCounter = 0; + std::atomic localCounter = 0; + auto & counter(useGlobalCounter ? globalCounter : localCounter); + + while (1) { + checkInterrupt(); + Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); + if (mkdir(tmpDir.c_str(), mode) == 0) { +#if __FreeBSD__ + /* Explicitly set the group of the directory. This is to + work around around problems caused by BSD's group + ownership semantics (directories inherit the group of + the parent). For instance, the group of /tmp on + FreeBSD is "wheel", so all directories created in /tmp + will be owned by "wheel"; but if the user is not in + "wheel", then "tar" will fail to unpack archives that + have the setgid bit set on directories. */ + if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) + throw SysError("setting group of directory '%1%'", tmpDir); +#endif + return tmpDir; + } + if (errno != EEXIST) + throw SysError("creating directory '%1%'", tmpDir); + } +} + + +std::pair createTempFile(const Path & prefix) +{ + Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); + // Strictly speaking, this is UB, but who cares... + // FIXME: use O_TMPFILE. + AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + if (!fd) + throw SysError("creating temporary file '%s'", tmpl); + closeOnExec(fd.get()); + return {std::move(fd), tmpl}; +} + +void createSymlink(const Path & target, const Path & link) +{ + if (symlink(target.c_str(), link.c_str())) + throw SysError("creating symlink from '%1%' to '%2%'", link, target); +} + +void replaceSymlink(const Path & target, const Path & link) +{ + for (unsigned int n = 0; true; n++) { + Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); + + try { + createSymlink(target, tmp); + } catch (SysError & e) { + if (e.errNo == EEXIST) continue; + throw; + } + + renameFile(tmp, link); + + break; + } +} + +void setWriteTime(const fs::path & p, const struct stat & st) +{ + struct timeval times[2]; + times[0] = { + .tv_sec = st.st_atime, + .tv_usec = 0, + }; + times[1] = { + .tv_sec = st.st_mtime, + .tv_usec = 0, + }; + if (lutimes(p.c_str(), times) != 0) + throw SysError("changing modification time of '%s'", p); +} + +void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) +{ + // TODO: Rewrite the `is_*` to use `symlink_status()` + auto statOfFrom = lstat(from.path().c_str()); + auto fromStatus = from.symlink_status(); + + // Mark the directory as writable so that we can delete its children + if (andDelete && fs::is_directory(fromStatus)) { + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + } + + + if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { + fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing); + } else if (fs::is_directory(fromStatus)) { + fs::create_directory(to); + for (auto & entry : fs::directory_iterator(from.path())) { + copy(entry, to / entry.path().filename(), andDelete); + } + } else { + throw Error("file '%s' has an unsupported type", from.path()); + } + + setWriteTime(to, statOfFrom); + if (andDelete) { + if (!fs::is_symlink(fromStatus)) + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + fs::remove(from.path()); + } +} + +void renameFile(const Path & oldName, const Path & newName) +{ + fs::rename(oldName, newName); +} + +void moveFile(const Path & oldName, const Path & newName) +{ + try { + renameFile(oldName, newName); + } catch (fs::filesystem_error & e) { + auto oldPath = fs::path(oldName); + auto newPath = fs::path(newName); + // For the move to be as atomic as possible, copy to a temporary + // directory + fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp"); + Finally removeTemp = [&]() { fs::remove(temp); }; + auto tempCopyTarget = temp / "copy-target"; + if (e.code().value() == EXDEV) { + fs::remove(newPath); + warn("Can’t rename %s as %s, copying instead", oldName, newName); + copy(fs::directory_entry(oldPath), tempCopyTarget, true); + renameFile(tempCopyTarget, newPath); + } + } +} + +////////////////////////////////////////////////////////////////////// + +} diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh new file mode 100644 index 00000000000..4637507b35b --- /dev/null +++ b/src/libutil/file-system.hh @@ -0,0 +1,238 @@ +#pragma once +/** + * @file + * + * Utiltities for working with the file sytem and file paths. + */ + +#include "types.hh" +#include "error.hh" +#include "logging.hh" +#include "file-descriptor.hh" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +#ifndef HAVE_STRUCT_DIRENT_D_TYPE +#define DT_UNKNOWN 0 +#define DT_REG 1 +#define DT_LNK 2 +#define DT_DIR 3 +#endif + +namespace nix { + +struct Sink; +struct Source; + +/** + * @return An absolutized path, resolving paths relative to the + * specified directory, or the current directory otherwise. The path + * is also canonicalised. + */ +Path absPath(Path path, + std::optional dir = {}, + bool resolveSymlinks = false); + +/** + * Canonicalise a path by removing all `.` or `..` components and + * double or trailing slashes. Optionally resolves all symlink + * components such that each component of the resulting path is *not* + * a symbolic link. + */ +Path canonPath(PathView path, bool resolveSymlinks = false); + +/** + * @return The directory part of the given canonical path, i.e., + * everything before the final `/`. If the path is the root or an + * immediate child thereof (e.g., `/foo`), this means `/` + * is returned. + */ +Path dirOf(const PathView path); + +/** + * @return the base name of the given canonical path, i.e., everything + * following the final `/` (trailing slashes are removed). + */ +std::string_view baseNameOf(std::string_view path); + +/** + * Check whether 'path' is a descendant of 'dir'. Both paths must be + * canonicalized. + */ +bool isInDir(std::string_view path, std::string_view dir); + +/** + * Check whether 'path' is equal to 'dir' or a descendant of + * 'dir'. Both paths must be canonicalized. + */ +bool isDirOrInDir(std::string_view path, std::string_view dir); + +/** + * Get status of `path`. + */ +struct stat stat(const Path & path); +struct stat lstat(const Path & path); + +/** + * @return true iff the given path exists. + */ +bool pathExists(const Path & path); + +/** + * A version of pathExists that returns false on a permission error. + * Useful for inferring default paths across directories that might not + * be readable. + * @return true iff the given path can be accessed and exists + */ +bool pathAccessible(const Path & path); + +/** + * Read the contents (target) of a symbolic link. The result is not + * in any way canonicalised. + */ +Path readLink(const Path & path); + +bool isLink(const Path & path); + +/** + * Read the contents of a directory. The entries `.` and `..` are + * removed. + */ +struct DirEntry +{ + std::string name; + ino_t ino; + /** + * one of DT_* + */ + unsigned char type; + DirEntry(std::string name, ino_t ino, unsigned char type) + : name(std::move(name)), ino(ino), type(type) { } +}; + +typedef std::vector DirEntries; + +DirEntries readDirectory(const Path & path); + +unsigned char getFileType(const Path & path); + +/** + * Read the contents of a file into a string. + */ +std::string readFile(const Path & path); +void readFile(const Path & path, Sink & sink); + +/** + * Write a string to a file. + */ +void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); + +void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); + +/** + * Flush a file's parent directory to disk + */ +void syncParent(const Path & path); + +/** + * Delete a path; i.e., in the case of a directory, it is deleted + * recursively. It's not an error if the path does not exist. The + * second variant returns the number of bytes and blocks freed. + */ +void deletePath(const Path & path); + +void deletePath(const Path & path, uint64_t & bytesFreed); + +/** + * Create a directory and all its parents, if necessary. Returns the + * list of created directories, in order of creation. + */ +Paths createDirs(const Path & path); +inline Paths createDirs(PathView path) +{ + return createDirs(Path(path)); +} + +/** + * Create a symlink. + */ +void createSymlink(const Path & target, const Path & link); + +/** + * Atomically create or replace a symlink. + */ +void replaceSymlink(const Path & target, const Path & link); + +void renameFile(const Path & src, const Path & dst); + +/** + * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` + * are on a different filesystem. + * + * Beware that this might not be atomic because of the copy that happens behind + * the scenes + */ +void moveFile(const Path & src, const Path & dst); + + +/** + * Automatic cleanup of resources. + */ +class AutoDelete +{ + Path path; + bool del; + bool recursive; +public: + AutoDelete(); + AutoDelete(const Path & p, bool recursive = true); + ~AutoDelete(); + void cancel(); + void reset(const Path & p, bool recursive = true); + operator Path() const { return path; } + operator PathView() const { return path; } +}; + + +struct DIRDeleter +{ + void operator()(DIR * dir) const { + closedir(dir); + } +}; + +typedef std::unique_ptr AutoCloseDir; + + +/** + * Create a temporary directory. + */ +Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", + bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); + +/** + * Create a temporary file, returning a file handle and its path. + */ +std::pair createTempFile(const Path & prefix = "nix"); + + +/** + * Used in various places. + */ +typedef std::function PathFilter; + +extern PathFilter defaultPathFilter; + +} diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc deleted file mode 100644 index 11cc0c0e7c8..00000000000 --- a/src/libutil/filesystem.cc +++ /dev/null @@ -1,162 +0,0 @@ -#include -#include -#include - -#include "finally.hh" -#include "util.hh" -#include "types.hh" - -namespace fs = std::filesystem; - -namespace nix { - -static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, - std::atomic & counter) -{ - tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); - if (includePid) - return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++); - else - return fmt("%1%/%2%-%3%", tmpRoot, prefix, counter++); -} - -Path createTempDir(const Path & tmpRoot, const Path & prefix, - bool includePid, bool useGlobalCounter, mode_t mode) -{ - static std::atomic globalCounter = 0; - std::atomic localCounter = 0; - auto & counter(useGlobalCounter ? globalCounter : localCounter); - - while (1) { - checkInterrupt(); - Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); - if (mkdir(tmpDir.c_str(), mode) == 0) { -#if __FreeBSD__ - /* Explicitly set the group of the directory. This is to - work around around problems caused by BSD's group - ownership semantics (directories inherit the group of - the parent). For instance, the group of /tmp on - FreeBSD is "wheel", so all directories created in /tmp - will be owned by "wheel"; but if the user is not in - "wheel", then "tar" will fail to unpack archives that - have the setgid bit set on directories. */ - if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) - throw SysError("setting group of directory '%1%'", tmpDir); -#endif - return tmpDir; - } - if (errno != EEXIST) - throw SysError("creating directory '%1%'", tmpDir); - } -} - - -std::pair createTempFile(const Path & prefix) -{ - Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); - // Strictly speaking, this is UB, but who cares... - // FIXME: use O_TMPFILE. - AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); - if (!fd) - throw SysError("creating temporary file '%s'", tmpl); - closeOnExec(fd.get()); - return {std::move(fd), tmpl}; -} - -void createSymlink(const Path & target, const Path & link) -{ - if (symlink(target.c_str(), link.c_str())) - throw SysError("creating symlink from '%1%' to '%2%'", link, target); -} - -void replaceSymlink(const Path & target, const Path & link) -{ - for (unsigned int n = 0; true; n++) { - Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); - - try { - createSymlink(target, tmp); - } catch (SysError & e) { - if (e.errNo == EEXIST) continue; - throw; - } - - renameFile(tmp, link); - - break; - } -} - -void setWriteTime(const fs::path & p, const struct stat & st) -{ - struct timeval times[2]; - times[0] = { - .tv_sec = st.st_atime, - .tv_usec = 0, - }; - times[1] = { - .tv_sec = st.st_mtime, - .tv_usec = 0, - }; - if (lutimes(p.c_str(), times) != 0) - throw SysError("changing modification time of '%s'", p); -} - -void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) -{ - // TODO: Rewrite the `is_*` to use `symlink_status()` - auto statOfFrom = lstat(from.path().c_str()); - auto fromStatus = from.symlink_status(); - - // Mark the directory as writable so that we can delete its children - if (andDelete && fs::is_directory(fromStatus)) { - fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); - } - - - if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { - fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing); - } else if (fs::is_directory(fromStatus)) { - fs::create_directory(to); - for (auto & entry : fs::directory_iterator(from.path())) { - copy(entry, to / entry.path().filename(), andDelete); - } - } else { - throw Error("file '%s' has an unsupported type", from.path()); - } - - setWriteTime(to, statOfFrom); - if (andDelete) { - if (!fs::is_symlink(fromStatus)) - fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); - fs::remove(from.path()); - } -} - -void renameFile(const Path & oldName, const Path & newName) -{ - fs::rename(oldName, newName); -} - -void moveFile(const Path & oldName, const Path & newName) -{ - try { - renameFile(oldName, newName); - } catch (fs::filesystem_error & e) { - auto oldPath = fs::path(oldName); - auto newPath = fs::path(newName); - // For the move to be as atomic as possible, copy to a temporary - // directory - fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp"); - Finally removeTemp = [&]() { fs::remove(temp); }; - auto tempCopyTarget = temp / "copy-target"; - if (e.code().value() == EXDEV) { - fs::remove(newPath); - warn("Can’t rename %s as %s, copying instead", oldName, newName); - copy(fs::directory_entry(oldPath), tempCopyTarget, true); - renameFile(tempCopyTarget, newPath); - } - } -} - -} diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index 727255b4581..ac72e47fbb3 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -44,6 +44,11 @@ inline std::string fmt(const std::string & s) return s; } +inline std::string fmt(std::string_view s) +{ + return std::string(s); +} + inline std::string fmt(const char * s) { return s; diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc new file mode 100644 index 00000000000..925e6f05dc2 --- /dev/null +++ b/src/libutil/fs-sink.cc @@ -0,0 +1,125 @@ +#include + +#include "config.hh" +#include "fs-sink.hh" + +namespace nix { + +void copyRecursive( + SourceAccessor & accessor, const CanonPath & from, + ParseSink & sink, const Path & to) +{ + auto stat = accessor.lstat(from); + + switch (stat.type) { + case SourceAccessor::tSymlink: + { + sink.createSymlink(to, accessor.readLink(from)); + } + + case SourceAccessor::tRegular: + { + sink.createRegularFile(to); + if (stat.isExecutable) + sink.isExecutable(); + LambdaSink sink2 { + [&](auto d) { + sink.receiveContents(d); + } + }; + accessor.readFile(from, sink2, [&](uint64_t size) { + sink.preallocateContents(size); + }); + break; + } + + case SourceAccessor::tDirectory: + { + sink.createDirectory(to); + for (auto & [name, _] : accessor.readDirectory(from)) { + copyRecursive( + accessor, from + name, + sink, to + "/" + name); + break; + } + } + + case SourceAccessor::tMisc: + throw Error("file '%1%' has an unsupported type", from); + + default: + abort(); + } +} + + +struct RestoreSinkSettings : Config +{ + Setting preallocateContents{this, false, "preallocate-contents", + "Whether to preallocate files when writing objects with known size."}; +}; + +static RestoreSinkSettings restoreSinkSettings; + +static GlobalConfig::Register r1(&restoreSinkSettings); + + +void RestoreSink::createDirectory(const Path & path) +{ + Path p = dstPath + path; + if (mkdir(p.c_str(), 0777) == -1) + throw SysError("creating directory '%1%'", p); +}; + +void RestoreSink::createRegularFile(const Path & path) +{ + Path p = dstPath + path; + fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); + if (!fd) throw SysError("creating file '%1%'", p); +} + +void RestoreSink::closeRegularFile() +{ + /* Call close explicitly to make sure the error is checked */ + fd.close(); +} + +void RestoreSink::isExecutable() +{ + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("fstat"); + if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) + throw SysError("fchmod"); +} + +void RestoreSink::preallocateContents(uint64_t len) +{ + if (!restoreSinkSettings.preallocateContents) + return; + +#if HAVE_POSIX_FALLOCATE + if (len) { + errno = posix_fallocate(fd.get(), 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) + throw SysError("preallocating file of %1% bytes", len); + } +#endif +} + +void RestoreSink::receiveContents(std::string_view data) +{ + writeFull(fd.get(), data); +} + +void RestoreSink::createSymlink(const Path & path, const std::string & target) +{ + Path p = dstPath + path; + nix::createSymlink(target, p); +} + +} diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh new file mode 100644 index 00000000000..bf54b730193 --- /dev/null +++ b/src/libutil/fs-sink.hh @@ -0,0 +1,105 @@ +#pragma once +///@file + +#include "types.hh" +#include "serialise.hh" +#include "source-accessor.hh" +#include "file-system.hh" + +namespace nix { + +/** + * \todo Fix this API, it sucks. + */ +struct ParseSink +{ + virtual void createDirectory(const Path & path) = 0; + + virtual void createRegularFile(const Path & path) = 0; + virtual void receiveContents(std::string_view data) = 0; + virtual void isExecutable() = 0; + virtual void closeRegularFile() = 0; + + virtual void createSymlink(const Path & path, const std::string & target) = 0; + + /** + * An optimization. By default, do nothing. + */ + virtual void preallocateContents(uint64_t size) { }; +}; + +/** + * Recusively copy file system objects from the source into the sink. + */ +void copyRecursive( + SourceAccessor & accessor, const CanonPath & sourcePath, + ParseSink & sink, const Path & destPath); + +/** + * Ignore everything and do nothing + */ +struct NullParseSink : ParseSink +{ + void createDirectory(const Path & path) override { } + void receiveContents(std::string_view data) override { } + void createSymlink(const Path & path, const std::string & target) override { } + void createRegularFile(const Path & path) override { } + void closeRegularFile() override { } + void isExecutable() override { } +}; + +/** + * Write files at the given path + */ +struct RestoreSink : ParseSink +{ + Path dstPath; + + void createDirectory(const Path & path) override; + + void createRegularFile(const Path & path) override; + void receiveContents(std::string_view data) override; + void isExecutable() override; + void closeRegularFile() override; + + void createSymlink(const Path & path, const std::string & target) override; + + void preallocateContents(uint64_t size) override; + +private: + AutoCloseFD fd; +}; + +/** + * Restore a single file at the top level, passing along + * `receiveContents` to the underlying `Sink`. For anything but a single + * file, set `regular = true` so the caller can fail accordingly. + */ +struct RegularFileSink : ParseSink +{ + bool regular = true; + Sink & sink; + + RegularFileSink(Sink & sink) : sink(sink) { } + + void createDirectory(const Path & path) override + { + regular = false; + } + + void receiveContents(std::string_view data) override + { + sink(data); + } + + void createSymlink(const Path & path, const std::string & target) override + { + regular = false; + } + + void createRegularFile(const Path & path) override { } + void closeRegularFile() override { } + void isExecutable() override { } +}; + +} diff --git a/src/libutil/git.cc b/src/libutil/git.cc index f35c2fdb75c..a4bd6009646 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -1,9 +1,263 @@ +#include +#include +#include +#include +#include +#include // for strcasecmp + +#include "signals.hh" +#include "config.hh" +#include "hash.hh" +#include "posix-source-accessor.hh" + #include "git.hh" +#include "serialise.hh" -#include +namespace nix::git { + +using namespace nix; +using namespace std::string_literals; + +std::optional decodeMode(RawMode m) { + switch (m) { + case (RawMode) Mode::Directory: + case (RawMode) Mode::Executable: + case (RawMode) Mode::Regular: + case (RawMode) Mode::Symlink: + return (Mode) m; + default: + return std::nullopt; + } +} + + +static std::string getStringUntil(Source & source, char byte) +{ + std::string s; + char n[1]; + source(std::string_view { n, 1 }); + while (*n != byte) { + s += *n; + source(std::string_view { n, 1 }); + } + return s; +} + + +static std::string getString(Source & source, int n) +{ + std::string v; + v.resize(n); + source(v); + return v; +} + + +void parse( + ParseSink & sink, + const Path & sinkPath, + Source & source, + std::function hook, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::GitHashing); + + auto type = getString(source, 5); + + if (type == "blob ") { + sink.createRegularFile(sinkPath); + + unsigned long long size = std::stoi(getStringUntil(source, 0)); + + sink.preallocateContents(size); + + unsigned long long left = size; + std::string buf; + buf.reserve(65536); + + while (left) { + checkInterrupt(); + buf.resize(std::min((unsigned long long)buf.capacity(), left)); + source(buf); + sink.receiveContents(buf); + left -= buf.size(); + } + } else if (type == "tree ") { + unsigned long long size = std::stoi(getStringUntil(source, 0)); + unsigned long long left = size; + + sink.createDirectory(sinkPath); + + while (left) { + std::string perms = getStringUntil(source, ' '); + left -= perms.size(); + left -= 1; + + RawMode rawMode = std::stoi(perms, 0, 8); + auto modeOpt = decodeMode(rawMode); + if (!modeOpt) + throw Error("Unknown Git permission: %o", perms); + auto mode = std::move(*modeOpt); + + std::string name = getStringUntil(source, '\0'); + left -= name.size(); + left -= 1; + + std::string hashs = getString(source, 20); + left -= 20; + + Hash hash(htSHA1); + std::copy(hashs.begin(), hashs.end(), hash.hash); + + hook(name, TreeEntry { + .mode = mode, + .hash = hash, + }); + + if (mode == Mode::Executable) + sink.isExecutable(); + } + } else throw Error("input doesn't look like a Git object"); +} + + +std::optional convertMode(SourceAccessor::Type type) +{ + switch (type) { + case SourceAccessor::tSymlink: return Mode::Symlink; + case SourceAccessor::tRegular: return Mode::Regular; + case SourceAccessor::tDirectory: return Mode::Directory; + case SourceAccessor::tMisc: return std::nullopt; + default: abort(); + } +} + + +void restore(ParseSink & sink, Source & source, std::function hook) +{ + parse(sink, "", source, [&](Path name, TreeEntry entry) { + auto [accessor, from] = hook(entry.hash); + auto stat = accessor->lstat(from); + auto gotOpt = convertMode(stat.type); + if (!gotOpt) + throw Error("file '%s' (git hash %s) has an unsupported type", + from, + entry.hash.to_string(HashFormat::Base16, false)); + auto & got = *gotOpt; + if (got != entry.mode) + throw Error("git mode of file '%s' (git hash %s) is %o but expected %o", + from, + entry.hash.to_string(HashFormat::Base16, false), + (RawMode) got, + (RawMode) entry.mode); + copyRecursive( + *accessor, from, + sink, name); + }); +} + + +void dumpBlobPrefix( + uint64_t size, Sink & sink, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::GitHashing); + auto s = fmt("blob %d\0"s, std::to_string(size)); + sink(s); +} + + +void dumpTree(const Tree & entries, Sink & sink, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::GitHashing); + + std::string v1; + + for (auto & [name, entry] : entries) { + auto name2 = name; + if (entry.mode == Mode::Directory) { + assert(name2.back() == '/'); + name2.pop_back(); + } + v1 += fmt("%o %s\0"s, static_cast(entry.mode), name2); + std::copy(entry.hash.hash, entry.hash.hash + entry.hash.hashSize, std::back_inserter(v1)); + } + + { + auto s = fmt("tree %d\0"s, v1.size()); + sink(s); + } + + sink(v1); +} + + +Mode dump( + SourceAccessor & accessor, const CanonPath & path, + Sink & sink, + std::function hook, + PathFilter & filter, + const ExperimentalFeatureSettings & xpSettings) +{ + auto st = accessor.lstat(path); + + switch (st.type) { + case SourceAccessor::tRegular: + { + accessor.readFile(path, sink, [&](uint64_t size) { + dumpBlobPrefix(size, sink, xpSettings); + }); + return st.isExecutable + ? Mode::Executable + : Mode::Regular; + } + + case SourceAccessor::tDirectory: + { + Tree entries; + for (auto & [name, _] : accessor.readDirectory(path)) { + auto child = path + name; + if (!filter(child.abs())) continue; + + auto entry = hook(child); + + auto name2 = name; + if (entry.mode == Mode::Directory) + name2 += "/"; + + entries.insert_or_assign(std::move(name2), std::move(entry)); + } + dumpTree(entries, sink, xpSettings); + return Mode::Directory; + } + + case SourceAccessor::tSymlink: + case SourceAccessor::tMisc: + default: + throw Error("file '%1%' has an unsupported type", path); + } +} + + +TreeEntry dumpHash( + HashType ht, + SourceAccessor & accessor, const CanonPath & path, PathFilter & filter) +{ + std::function hook; + hook = [&](const CanonPath & path) -> TreeEntry { + auto hashSink = HashSink(ht); + auto mode = dump(accessor, path, hashSink, hook, filter); + auto hash = hashSink.finish().first; + return { + .mode = mode, + .hash = hash, + }; + }; + + return hook(path); +} -namespace nix { -namespace git { std::optional parseLsRemoteLine(std::string_view line) { @@ -22,4 +276,3 @@ std::optional parseLsRemoteLine(std::string_view line) } } -} diff --git a/src/libutil/git.hh b/src/libutil/git.hh index bf2b9a2869a..30346007280 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -5,9 +5,127 @@ #include #include -namespace nix { +#include "types.hh" +#include "serialise.hh" +#include "hash.hh" +#include "source-accessor.hh" +#include "fs-sink.hh" -namespace git { +namespace nix::git { + +using RawMode = uint32_t; + +enum struct Mode : RawMode { + Directory = 0040000, + Executable = 0100755, + Regular = 0100644, + Symlink = 0120000, +}; + +std::optional decodeMode(RawMode m); + +/** + * An anonymous Git tree object entry (no name part). + */ +struct TreeEntry +{ + Mode mode; + Hash hash; + + GENERATE_CMP(TreeEntry, me->mode, me->hash); +}; + +/** + * A Git tree object, fully decoded and stored in memory. + * + * Directory names must end in a `/` for sake of sorting. See + * https://github.com/mirage/irmin/issues/352 + */ +using Tree = std::map; + +/** + * Callback for processing a child hash with `parse` + * + * The function should + * + * 1. Obtain the file system objects denoted by `gitHash` + * + * 2. Ensure they match `mode` + * + * 3. Feed them into the same sink `parse` was called with + * + * Implementations may seek to memoize resources (bandwidth, storage, + * etc.) for the same Git hash. + */ +using SinkHook = void(const Path & name, TreeEntry entry); + +void parse( + ParseSink & sink, const Path & sinkPath, + Source & source, + std::function hook, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Assists with writing a `SinkHook` step (2). + */ +std::optional convertMode(SourceAccessor::Type type); + +/** + * Simplified version of `SinkHook` for `restore`. + * + * Given a `Hash`, return a `SourceAccessor` and `CanonPath` pointing to + * the file system object with that path. + */ +using RestoreHook = std::pair(Hash); + +/** + * Wrapper around `parse` and `RestoreSink` + */ +void restore(ParseSink & sink, Source & source, std::function hook); + +/** + * Dumps a single file to a sink + * + * @param xpSettings for testing purposes + */ +void dumpBlobPrefix( + uint64_t size, Sink & sink, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Dumps a representation of a git tree to a sink + */ +void dumpTree( + const Tree & entries, Sink & sink, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Callback for processing a child with `dump` + * + * The function should return the Git hash and mode of the file at the + * given path in the accessor passed to `dump`. + * + * Note that if the child is a directory, its child in must also be so + * processed in order to compute this information. + */ +using DumpHook = TreeEntry(const CanonPath & path); + +Mode dump( + SourceAccessor & accessor, const CanonPath & path, + Sink & sink, + std::function hook, + PathFilter & filter = defaultPathFilter, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Recursively dumps path, hashing as we go. + * + * A smaller wrapper around `dump`. + */ +TreeEntry dumpHash( + HashType ht, + SourceAccessor & accessor, const CanonPath & path, + PathFilter & filter = defaultPathFilter); /** * A line from the output of `git ls-remote --symref`. @@ -16,15 +134,17 @@ namespace git { * * - Symbolic references of the form * - * ref: {target} {reference} - * - * where {target} is itself a reference and {reference} is optional + * ``` + * ref: {target} {reference} + * ``` + * where {target} is itself a reference and {reference} is optional * * - Object references of the form * - * {target} {reference} - * - * where {target} is a commit id and {reference} is mandatory + * ``` + * {target} {reference} + * ``` + * where {target} is a commit id and {reference} is mandatory */ struct LsRemoteRefLine { enum struct Kind { @@ -36,8 +156,9 @@ struct LsRemoteRefLine { std::optional reference; }; +/** + * Parse an `LsRemoteRefLine` + */ std::optional parseLsRemoteLine(std::string_view line); } - -} diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 2c36d9d9498..144f7ae7ed4 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -9,7 +9,6 @@ #include "hash.hh" #include "archive.hh" #include "split.hh" -#include "util.hh" #include #include @@ -111,26 +110,26 @@ static std::string printHash32(const Hash & hash) std::string printHash16or32(const Hash & hash) { assert(hash.type); - return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false); + return hash.to_string(hash.type == htMD5 ? HashFormat::Base16 : HashFormat::Base32, false); } -std::string Hash::to_string(Base base, bool includeType) const +std::string Hash::to_string(HashFormat hashFormat, bool includeType) const { std::string s; - if (base == SRI || includeType) { + if (hashFormat == HashFormat::SRI || includeType) { s += printHashType(type); - s += base == SRI ? '-' : ':'; + s += hashFormat == HashFormat::SRI ? '-' : ':'; } - switch (base) { - case Base16: + switch (hashFormat) { + case HashFormat::Base16: s += printHash16(*this); break; - case Base32: + case HashFormat::Base32: s += printHash32(*this); break; - case Base64: - case SRI: + case HashFormat::Base64: + case HashFormat::SRI: s += base64Encode(std::string_view((const char *) hash, hashSize)); break; } @@ -267,7 +266,7 @@ Hash newHashAllowEmpty(std::string_view hashStr, std::optional ht) if (!ht) throw BadHash("empty hash requires explicit hash type"); Hash h(*ht); - warn("found empty hash, assuming '%s'", h.to_string(SRI, true)); + warn("found empty hash, assuming '%s'", h.to_string(HashFormat::SRI, true)); return h; } else return Hash::parseAny(hashStr, ht); @@ -386,13 +385,48 @@ Hash compressHash(const Hash & hash, unsigned int newSize) } +std::optional parseHashFormatOpt(std::string_view hashFormatName) +{ + if (hashFormatName == "base16") return HashFormat::Base16; + if (hashFormatName == "base32") return HashFormat::Base32; + if (hashFormatName == "base64") return HashFormat::Base64; + if (hashFormatName == "sri") return HashFormat::SRI; + return std::nullopt; +} + +HashFormat parseHashFormat(std::string_view hashFormatName) +{ + auto opt_f = parseHashFormatOpt(hashFormatName); + if (opt_f) + return *opt_f; + throw UsageError("unknown hash format '%1%', expect 'base16', 'base32', 'base64', or 'sri'", hashFormatName); +} + +std::string_view printHashFormat(HashFormat HashFormat) +{ + switch (HashFormat) { + case HashFormat::Base64: + return "base64"; + case HashFormat::Base32: + return "base32"; + case HashFormat::Base16: + return "base16"; + case HashFormat::SRI: + return "sri"; + default: + // illegal hash base enum value internally, as opposed to external input + // which should be validated with nice error message. + assert(false); + } +} + std::optional parseHashTypeOpt(std::string_view s) { if (s == "md5") return htMD5; - else if (s == "sha1") return htSHA1; - else if (s == "sha256") return htSHA256; - else if (s == "sha512") return htSHA512; - else return std::optional {}; + if (s == "sha1") return htSHA1; + if (s == "sha256") return htSHA256; + if (s == "sha512") return htSHA512; + return std::nullopt; } HashType parseHashType(std::string_view s) @@ -401,7 +435,7 @@ HashType parseHashType(std::string_view s) if (opt_h) return *opt_h; else - throw UsageError("unknown hash algorithm '%1%'", s); + throw UsageError("unknown hash algorithm '%1%', expect 'md5', 'sha1', 'sha256', or 'sha512'", s); } std::string_view printHashType(HashType ht) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index ae3ee40f4e3..6ade6555c89 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "serialise.hh" +#include "file-system.hh" namespace nix { @@ -23,7 +24,21 @@ extern std::set hashTypes; extern const std::string base32Chars; -enum Base : int { Base64, Base32, Base16, SRI }; +/** + * @brief Enumeration representing the hash formats. + */ +enum struct HashFormat : int { + /// @brief Base 64 encoding. + /// @see [IETF RFC 4648, section 4](https://datatracker.ietf.org/doc/html/rfc4648#section-4). + Base64, + /// @brief Nix-specific base-32 encoding. @see base32Chars + Base32, + /// @brief Lowercase hexadecimal encoding. @see base16Chars + Base16, + /// @brief ":", format of the SRI integrity attribute. + /// @see W3C recommendation [Subresource Intergrity](https://www.w3.org/TR/SRI/). + SRI +}; struct Hash @@ -114,16 +129,16 @@ public: * or base-64. By default, this is prefixed by the hash type * (e.g. "sha256:"). */ - std::string to_string(Base base, bool includeType) const; + std::string to_string(HashFormat hashFormat, bool includeType) const; std::string gitRev() const { - return to_string(Base16, false); + return to_string(HashFormat::Base16, false); } std::string gitShortRev() const { - return std::string(to_string(Base16, false), 0, 7); + return std::string(to_string(HashFormat::Base16, false), 0, 7); } static Hash dummy; @@ -145,13 +160,17 @@ std::string printHash16or32(const Hash & hash); Hash hashString(HashType ht, std::string_view s); /** - * Compute the hash of the given file. + * Compute the hash of the given file, hashing its contents directly. + * + * (Metadata, such as the executable permission bit, is ignored.) */ Hash hashFile(HashType ht, const Path & path); /** - * Compute the hash of the given path. The hash is defined as - * (essentially) hashString(ht, dumpPath(path)). + * Compute the hash of the given path, serializing as a Nix Archive and + * then hashing that. + * + * The hash is defined as (essentially) hashString(ht, dumpPath(path)). */ typedef std::pair HashResult; HashResult hashPath(HashType ht, const Path & path, @@ -163,6 +182,21 @@ HashResult hashPath(HashType ht, const Path & path, */ Hash compressHash(const Hash & hash, unsigned int newSize); +/** + * Parse a string representing a hash format. + */ +HashFormat parseHashFormat(std::string_view hashFormatName); + +/** + * std::optional version of parseHashFormat that doesn't throw error. + */ +std::optional parseHashFormatOpt(std::string_view hashFormatName); + +/** + * The reverse of parseHashFormat. + */ +std::string_view printHashFormat(HashFormat hashFormat); + /** * Parse a string representing a hash type. */ diff --git a/src/libutil/local.mk b/src/libutil/local.mk index f880c0fc562..81efaafeca8 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,8 +6,13 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) +libutil_CXXFLAGS += -I src/libutil + libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context +$(foreach i, $(wildcard $(d)/args/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/args, 0644))) + ifeq ($(HAVE_LIBCPUID), 1) libutil_LDFLAGS += -lcpuid endif diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 9d7a141b37b..60b0865bf2c 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -1,4 +1,7 @@ #include "logging.hh" +#include "file-descriptor.hh" +#include "environment-variables.hh" +#include "terminal.hh" #include "util.hh" #include "config.hh" diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc new file mode 100644 index 00000000000..78a4dd29815 --- /dev/null +++ b/src/libutil/memory-source-accessor.cc @@ -0,0 +1,180 @@ +#include "memory-source-accessor.hh" + +namespace nix { + +MemorySourceAccessor::File * +MemorySourceAccessor::open(const CanonPath & path, std::optional create) +{ + File * cur = &root; + + bool newF = false; + + for (std::string_view name : path) + { + auto * curDirP = std::get_if(&cur->raw); + if (!curDirP) + return nullptr; + auto & curDir = *curDirP; + + auto i = curDir.contents.find(name); + if (i == curDir.contents.end()) { + if (!create) + return nullptr; + else { + newF = true; + i = curDir.contents.insert(i, { + std::string { name }, + File::Directory {}, + }); + } + } + cur = &i->second; + } + + if (newF && create) *cur = std::move(*create); + + return cur; +} + +std::string MemorySourceAccessor::readFile(const CanonPath & path) +{ + auto * f = open(path, std::nullopt); + if (!f) + throw Error("file '%s' does not exist", path); + if (auto * r = std::get_if(&f->raw)) + return r->contents; + else + throw Error("file '%s' is not a regular file", path); +} + +bool MemorySourceAccessor::pathExists(const CanonPath & path) +{ + return open(path, std::nullopt); +} + +MemorySourceAccessor::Stat MemorySourceAccessor::File::lstat() const +{ + return std::visit(overloaded { + [](const Regular & r) { + return Stat { + .type = tRegular, + .fileSize = r.contents.size(), + .isExecutable = r.executable, + }; + }, + [](const Directory &) { + return Stat { + .type = tDirectory, + }; + }, + [](const Symlink &) { + return Stat { + .type = tSymlink, + }; + }, + }, this->raw); +} + +std::optional +MemorySourceAccessor::maybeLstat(const CanonPath & path) +{ + const auto * f = open(path, std::nullopt); + return f ? std::optional { f->lstat() } : std::nullopt; +} + +MemorySourceAccessor::DirEntries MemorySourceAccessor::readDirectory(const CanonPath & path) +{ + auto * f = open(path, std::nullopt); + if (!f) + throw Error("file '%s' does not exist", path); + if (auto * d = std::get_if(&f->raw)) { + DirEntries res; + for (auto & [name, file] : d->contents) + res.insert_or_assign(name, file.lstat().type); + return res; + } else + throw Error("file '%s' is not a directory", path); + return {}; +} + +std::string MemorySourceAccessor::readLink(const CanonPath & path) +{ + auto * f = open(path, std::nullopt); + if (!f) + throw Error("file '%s' does not exist", path); + if (auto * s = std::get_if(&f->raw)) + return s->target; + else + throw Error("file '%s' is not a symbolic link", path); +} + +CanonPath MemorySourceAccessor::addFile(CanonPath path, std::string && contents) +{ + auto * f = open(path, File { File::Regular {} }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + if (auto * r = std::get_if(&f->raw)) + r->contents = std::move(contents); + else + throw Error("file '%s' is not a regular file", path); + + return path; +} + + +using File = MemorySourceAccessor::File; + +void MemorySink::createDirectory(const Path & path) +{ + auto * f = dst.open(CanonPath{path}, File { File::Directory { } }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + + if (!std::holds_alternative(f->raw)) + throw Error("file '%s' is not a directory", path); +}; + +void MemorySink::createRegularFile(const Path & path) +{ + auto * f = dst.open(CanonPath{path}, File { File::Regular {} }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + if (!(r = std::get_if(&f->raw))) + throw Error("file '%s' is not a regular file", path); +} + +void MemorySink::closeRegularFile() +{ + r = nullptr; +} + +void MemorySink::isExecutable() +{ + assert(r); + r->executable = true; +} + +void MemorySink::preallocateContents(uint64_t len) +{ + assert(r); + r->contents.reserve(len); +} + +void MemorySink::receiveContents(std::string_view data) +{ + assert(r); + r->contents += data; +} + +void MemorySink::createSymlink(const Path & path, const std::string & target) +{ + auto * f = dst.open(CanonPath{path}, File { File::Symlink { } }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + if (auto * s = std::get_if(&f->raw)) + s->target = target; + else + throw Error("file '%s' is not a symbolic link", path); +} + +} diff --git a/src/libutil/memory-source-accessor.hh b/src/libutil/memory-source-accessor.hh new file mode 100644 index 00000000000..b908f3713c0 --- /dev/null +++ b/src/libutil/memory-source-accessor.hh @@ -0,0 +1,99 @@ +#include "source-accessor.hh" +#include "fs-sink.hh" +#include "variant-wrapper.hh" + +namespace nix { + +/** + * An source accessor for an in-memory file system. + */ +struct MemorySourceAccessor : virtual SourceAccessor +{ + /** + * In addition to being part of the implementation of + * `MemorySourceAccessor`, this has a side benefit of nicely + * defining what a "file system object" is in Nix. + */ + struct File { + struct Regular { + bool executable = false; + std::string contents; + + GENERATE_CMP(Regular, me->executable, me->contents); + }; + + struct Directory { + using Name = std::string; + + std::map> contents; + + GENERATE_CMP(Directory, me->contents); + }; + + struct Symlink { + std::string target; + + GENERATE_CMP(Symlink, me->target); + }; + + using Raw = std::variant; + Raw raw; + + MAKE_WRAPPER_CONSTRUCTOR(File); + + GENERATE_CMP(File, me->raw); + + Stat lstat() const; + }; + + File root { File::Directory {} }; + + GENERATE_CMP(MemorySourceAccessor, me->root); + + std::string readFile(const CanonPath & path) override; + bool pathExists(const CanonPath & path) override; + std::optional maybeLstat(const CanonPath & path) override; + DirEntries readDirectory(const CanonPath & path) override; + std::string readLink(const CanonPath & path) override; + + /** + * @param create If present, create this file and any parent directories + * that are needed. + * + * Return null if + * + * - `create = false`: File does not exist. + * + * - `create = true`: some parent file was not a dir, so couldn't + * look/create inside. + */ + File * open(const CanonPath & path, std::optional create); + + CanonPath addFile(CanonPath path, std::string && contents); +}; + +/** + * Write to a `MemorySourceAccessor` at the given path + */ +struct MemorySink : ParseSink +{ + MemorySourceAccessor & dst; + + MemorySink(MemorySourceAccessor & dst) : dst(dst) { } + + void createDirectory(const Path & path) override; + + void createRegularFile(const Path & path) override; + void receiveContents(std::string_view data) override; + void isExecutable() override; + void closeRegularFile() override; + + void createSymlink(const Path & path, const std::string & target) override; + + void preallocateContents(uint64_t size) override; + +private: + MemorySourceAccessor::File::Regular * r; +}; + +} diff --git a/src/libutil/monitor-fd.hh b/src/libutil/monitor-fd.hh index 86d0115fc3c..228fb13f853 100644 --- a/src/libutil/monitor-fd.hh +++ b/src/libutil/monitor-fd.hh @@ -10,6 +10,8 @@ #include #include +#include "signals.hh" + namespace nix { diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc index f66accb10ad..a789b321e17 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/namespaces.cc @@ -1,13 +1,22 @@ -#if __linux__ - -#include "namespaces.hh" +#include "current-process.hh" #include "util.hh" #include "finally.hh" +#include "file-system.hh" +#include "processes.hh" +#include "signals.hh" + +#if __linux__ +# include +# include +# include "cgroup.hh" +#endif #include namespace nix { +#if __linux__ + bool userNamespacesSupported() { static auto res = [&]() -> bool @@ -92,6 +101,60 @@ bool mountAndPidNamespacesSupported() return res; } +#endif + + +////////////////////////////////////////////////////////////////////// + +#if __linux__ +static AutoCloseFD fdSavedMountNamespace; +static AutoCloseFD fdSavedRoot; +#endif + +void saveMountNamespace() +{ +#if __linux__ + static std::once_flag done; + std::call_once(done, []() { + fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); + if (!fdSavedMountNamespace) + throw SysError("saving parent mount namespace"); + + fdSavedRoot = open("/proc/self/root", O_RDONLY); + }); +#endif +} + +void restoreMountNamespace() +{ +#if __linux__ + try { + auto savedCwd = absPath("."); + + if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) + throw SysError("restoring parent mount namespace"); + + if (fdSavedRoot) { + if (fchdir(fdSavedRoot.get())) + throw SysError("chdir into saved root"); + if (chroot(".")) + throw SysError("chroot into saved root"); + } + + if (chdir(savedCwd.c_str()) == -1) + throw SysError("restoring cwd"); + } catch (Error & e) { + debug(e.msg()); + } +#endif } +void unshareFilesystem() +{ +#ifdef __linux__ + if (unshare(CLONE_FS) != 0 && errno != EPERM) + throw SysError("unsharing filesystem state in download thread"); #endif +} + +} diff --git a/src/libutil/namespaces.hh b/src/libutil/namespaces.hh index 0b7eeb66cc4..7e4e921a80a 100644 --- a/src/libutil/namespaces.hh +++ b/src/libutil/namespaces.hh @@ -1,8 +1,31 @@ #pragma once ///@file +#include + +#include "types.hh" + namespace nix { +/** + * Save the current mount namespace. Ignored if called more than + * once. + */ +void saveMountNamespace(); + +/** + * Restore the mount namespace saved by saveMountNamespace(). Ignored + * if saveMountNamespace() was never called. + */ +void restoreMountNamespace(); + +/** + * Cause this thread to not share any FS attributes with the main + * thread, because this causes setns() in restoreMountNamespace() to + * fail. + */ +void unshareFilesystem(); + #if __linux__ bool userNamespacesSupported(); diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc new file mode 100644 index 00000000000..dc96f84e513 --- /dev/null +++ b/src/libutil/posix-source-accessor.cc @@ -0,0 +1,92 @@ +#include "posix-source-accessor.hh" +#include "signals.hh" + +namespace nix { + +void PosixSourceAccessor::readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) +{ + // FIXME: add O_NOFOLLOW since symlinks should be resolved by the + // caller? + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%1%'", path); + + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("statting file"); + + sizeCallback(st.st_size); + + off_t left = st.st_size; + + std::vector buf(64 * 1024); + while (left) { + checkInterrupt(); + ssize_t rd = read(fd.get(), buf.data(), (size_t) std::min(left, (off_t) buf.size())); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading from file '%s'", showPath(path)); + } + else if (rd == 0) + throw SysError("unexpected end-of-file reading '%s'", showPath(path)); + else { + assert(rd <= left); + sink({(char *) buf.data(), (size_t) rd}); + left -= rd; + } + } +} + +bool PosixSourceAccessor::pathExists(const CanonPath & path) +{ + return nix::pathExists(path.abs()); +} + +std::optional PosixSourceAccessor::maybeLstat(const CanonPath & path) +{ + struct stat st; + if (::lstat(path.c_str(), &st)) { + if (errno == ENOENT) return std::nullopt; + throw SysError("getting status of '%s'", showPath(path)); + } + mtime = std::max(mtime, st.st_mtime); + return Stat { + .type = + S_ISREG(st.st_mode) ? tRegular : + S_ISDIR(st.st_mode) ? tDirectory : + S_ISLNK(st.st_mode) ? tSymlink : + tMisc, + .fileSize = S_ISREG(st.st_mode) ? std::optional(st.st_size) : std::nullopt, + .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR, + }; +} + +SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path) +{ + DirEntries res; + for (auto & entry : nix::readDirectory(path.abs())) { + std::optional type; + switch (entry.type) { + case DT_REG: type = Type::tRegular; break; + case DT_LNK: type = Type::tSymlink; break; + case DT_DIR: type = Type::tDirectory; break; + } + res.emplace(entry.name, type); + } + return res; +} + +std::string PosixSourceAccessor::readLink(const CanonPath & path) +{ + return nix::readLink(path.abs()); +} + +std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & path) +{ + return path; +} + +} diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh new file mode 100644 index 00000000000..cf087d26e3d --- /dev/null +++ b/src/libutil/posix-source-accessor.hh @@ -0,0 +1,34 @@ +#pragma once + +#include "source-accessor.hh" + +namespace nix { + +/** + * A source accessor that uses the Unix filesystem. + */ +struct PosixSourceAccessor : SourceAccessor +{ + /** + * The most recent mtime seen by lstat(). This is a hack to + * support dumpPathAndGetMtime(). Should remove this eventually. + */ + time_t mtime = 0; + + void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) override; + + bool pathExists(const CanonPath & path) override; + + std::optional maybeLstat(const CanonPath & path) override; + + DirEntries readDirectory(const CanonPath & path) override; + + std::string readLink(const CanonPath & path) override; + + std::optional getPhysicalPath(const CanonPath & path) override; +}; + +} diff --git a/src/libutil/processes.cc b/src/libutil/processes.cc new file mode 100644 index 00000000000..91a0ea66fda --- /dev/null +++ b/src/libutil/processes.cc @@ -0,0 +1,421 @@ +#include "current-process.hh" +#include "environment-variables.hh" +#include "signals.hh" +#include "processes.hh" +#include "finally.hh" +#include "serialise.hh" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#ifdef __APPLE__ +# include +#endif + +#ifdef __linux__ +# include +# include +#endif + + +namespace nix { + +Pid::Pid() +{ +} + + +Pid::Pid(pid_t pid) + : pid(pid) +{ +} + + +Pid::~Pid() +{ + if (pid != -1) kill(); +} + + +void Pid::operator =(pid_t pid) +{ + if (this->pid != -1 && this->pid != pid) kill(); + this->pid = pid; + killSignal = SIGKILL; // reset signal to default +} + + +Pid::operator pid_t() +{ + return pid; +} + + +int Pid::kill() +{ + assert(pid != -1); + + debug("killing process %1%", pid); + + /* Send the requested signal to the child. If it has its own + process group, send the signal to every process in the child + process group (which hopefully includes *all* its children). */ + if (::kill(separatePG ? -pid : pid, killSignal) != 0) { + /* On BSDs, killing a process group will return EPERM if all + processes in the group are zombies (or something like + that). So try to detect and ignore that situation. */ +#if __FreeBSD__ || __APPLE__ + if (errno != EPERM || ::kill(pid, 0) != 0) +#endif + logError(SysError("killing process %d", pid).info()); + } + + return wait(); +} + + +int Pid::wait() +{ + assert(pid != -1); + while (1) { + int status; + int res = waitpid(pid, &status, 0); + if (res == pid) { + pid = -1; + return status; + } + if (errno != EINTR) + throw SysError("cannot get exit status of PID %d", pid); + checkInterrupt(); + } +} + + +void Pid::setSeparatePG(bool separatePG) +{ + this->separatePG = separatePG; +} + + +void Pid::setKillSignal(int signal) +{ + this->killSignal = signal; +} + + +pid_t Pid::release() +{ + pid_t p = pid; + pid = -1; + return p; +} + + +void killUser(uid_t uid) +{ + debug("killing all processes running under uid '%1%'", uid); + + assert(uid != 0); /* just to be safe... */ + + /* The system call kill(-1, sig) sends the signal `sig' to all + users to which the current process can send signals. So we + fork a process, switch to uid, and send a mass kill. */ + + Pid pid = startProcess([&]() { + + if (setuid(uid) == -1) + throw SysError("setting uid"); + + while (true) { +#ifdef __APPLE__ + /* OSX's kill syscall takes a third parameter that, among + other things, determines if kill(-1, signo) affects the + calling process. In the OSX libc, it's set to true, + which means "follow POSIX", which we don't want here + */ + if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; +#else + if (kill(-1, SIGKILL) == 0) break; +#endif + if (errno == ESRCH || errno == EPERM) break; /* no more processes */ + if (errno != EINTR) + throw SysError("cannot kill processes for uid '%1%'", uid); + } + + _exit(0); + }); + + int status = pid.wait(); + if (status != 0) + throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status)); + + /* !!! We should really do some check to make sure that there are + no processes left running under `uid', but there is no portable + way to do so (I think). The most reliable way may be `ps -eo + uid | grep -q $uid'. */ +} + + +////////////////////////////////////////////////////////////////////// + + +/* Wrapper around vfork to prevent the child process from clobbering + the caller's stack frame in the parent. */ +static pid_t doFork(bool allowVfork, std::function fun) __attribute__((noinline)); +static pid_t doFork(bool allowVfork, std::function fun) +{ +#ifdef __linux__ + pid_t pid = allowVfork ? vfork() : fork(); +#else + pid_t pid = fork(); +#endif + if (pid != 0) return pid; + fun(); + abort(); +} + + +#if __linux__ +static int childEntry(void * arg) +{ + auto main = (std::function *) arg; + (*main)(); + return 1; +} +#endif + + +pid_t startProcess(std::function fun, const ProcessOptions & options) +{ + std::function wrapper = [&]() { + if (!options.allowVfork) + logger = makeSimpleLogger(); + try { +#if __linux__ + if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) + throw SysError("setting death signal"); +#endif + fun(); + } catch (std::exception & e) { + try { + std::cerr << options.errorPrefix << e.what() << "\n"; + } catch (...) { } + } catch (...) { } + if (options.runExitHandlers) + exit(1); + else + _exit(1); + }; + + pid_t pid = -1; + + if (options.cloneFlags) { + #ifdef __linux__ + // Not supported, since then we don't know when to free the stack. + assert(!(options.cloneFlags & CLONE_VM)); + + size_t stackSize = 1 * 1024 * 1024; + auto stack = (char *) mmap(0, stackSize, + PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (stack == MAP_FAILED) throw SysError("allocating stack"); + + Finally freeStack([&]() { munmap(stack, stackSize); }); + + pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper); + #else + throw Error("clone flags are only supported on Linux"); + #endif + } else + pid = doFork(options.allowVfork, wrapper); + + if (pid == -1) throw SysError("unable to fork"); + + return pid; +} + + +std::string runProgram(Path program, bool searchPath, const Strings & args, + const std::optional & input, bool isInteractive) +{ + auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive}); + + if (!statusOk(res.first)) + throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); + + return res.second; +} + +// Output = error code + "standard out" output stream +std::pair runProgram(RunOptions && options) +{ + StringSink sink; + options.standardOut = &sink; + + int status = 0; + + try { + runProgram2(options); + } catch (ExecError & e) { + status = e.status; + } + + return {status, std::move(sink.s)}; +} + +void runProgram2(const RunOptions & options) +{ + checkInterrupt(); + + assert(!(options.standardIn && options.input)); + + std::unique_ptr source_; + Source * source = options.standardIn; + + if (options.input) { + source_ = std::make_unique(*options.input); + source = source_.get(); + } + + /* Create a pipe. */ + Pipe out, in; + if (options.standardOut) out.create(); + if (source) in.create(); + + ProcessOptions processOptions; + // vfork implies that the environment of the main process and the fork will + // be shared (technically this is undefined, but in practice that's the + // case), so we can't use it if we alter the environment + processOptions.allowVfork = !options.environment; + + std::optional>> resumeLoggerDefer; + if (options.isInteractive) { + logger->pause(); + resumeLoggerDefer.emplace( + []() { + logger->resume(); + } + ); + } + + /* Fork. */ + Pid pid = startProcess([&]() { + if (options.environment) + replaceEnv(*options.environment); + if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + if (options.mergeStderrToStdout) + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); + if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) + throw SysError("dupping stdin"); + + if (options.chdir && chdir((*options.chdir).c_str()) == -1) + throw SysError("chdir failed"); + if (options.gid && setgid(*options.gid) == -1) + throw SysError("setgid failed"); + /* Drop all other groups if we're setgid. */ + if (options.gid && setgroups(0, 0) == -1) + throw SysError("setgroups failed"); + if (options.uid && setuid(*options.uid) == -1) + throw SysError("setuid failed"); + + Strings args_(options.args); + args_.push_front(options.program); + + restoreProcessContext(); + + if (options.searchPath) + execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); + // This allows you to refer to a program with a pathname relative + // to the PATH variable. + else + execv(options.program.c_str(), stringsToCharPtrs(args_).data()); + + throw SysError("executing '%1%'", options.program); + }, processOptions); + + out.writeSide.close(); + + std::thread writerThread; + + std::promise promise; + + Finally doJoin([&]() { + if (writerThread.joinable()) + writerThread.join(); + }); + + + if (source) { + in.readSide.close(); + writerThread = std::thread([&]() { + try { + std::vector buf(8 * 1024); + while (true) { + size_t n; + try { + n = source->read(buf.data(), buf.size()); + } catch (EndOfFile &) { + break; + } + writeFull(in.writeSide.get(), {buf.data(), n}); + } + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + in.writeSide.close(); + }); + } + + if (options.standardOut) + drainFD(out.readSide.get(), *options.standardOut); + + /* Wait for the child to finish. */ + int status = pid.wait(); + + /* Wait for the writer thread to finish. */ + if (source) promise.get_future().get(); + + if (status) + throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); +} + +////////////////////////////////////////////////////////////////////// + +std::string statusToString(int status) +{ + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + if (WIFEXITED(status)) + return fmt("failed with exit code %1%", WEXITSTATUS(status)); + else if (WIFSIGNALED(status)) { + int sig = WTERMSIG(status); +#if HAVE_STRSIGNAL + const char * description = strsignal(sig); + return fmt("failed due to signal %1% (%2%)", sig, description); +#else + return fmt("failed due to signal %1%", sig); +#endif + } + else + return "died abnormally"; + } else return "succeeded"; +} + + +bool statusOk(int status) +{ + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + +} diff --git a/src/libutil/processes.hh b/src/libutil/processes.hh new file mode 100644 index 00000000000..978c37105c6 --- /dev/null +++ b/src/libutil/processes.hh @@ -0,0 +1,123 @@ +#pragma once +///@file + +#include "types.hh" +#include "error.hh" +#include "logging.hh" +#include "ansicolor.hh" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace nix { + +struct Sink; +struct Source; + +class Pid +{ + pid_t pid = -1; + bool separatePG = false; + int killSignal = SIGKILL; +public: + Pid(); + Pid(pid_t pid); + ~Pid(); + void operator =(pid_t pid); + operator pid_t(); + int kill(); + int wait(); + + void setSeparatePG(bool separatePG); + void setKillSignal(int signal); + pid_t release(); +}; + + +/** + * Kill all processes running under the specified uid by sending them + * a SIGKILL. + */ +void killUser(uid_t uid); + + +/** + * Fork a process that runs the given function, and return the child + * pid to the caller. + */ +struct ProcessOptions +{ + std::string errorPrefix = ""; + bool dieWithParent = true; + bool runExitHandlers = false; + bool allowVfork = false; + /** + * use clone() with the specified flags (Linux only) + */ + int cloneFlags = 0; +}; + +pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); + + +/** + * Run a program and return its stdout in a string (i.e., like the + * shell backtick operator). + */ +std::string runProgram(Path program, bool searchPath = false, + const Strings & args = Strings(), + const std::optional & input = {}, bool isInteractive = false); + +struct RunOptions +{ + Path program; + bool searchPath = true; + Strings args; + std::optional uid; + std::optional gid; + std::optional chdir; + std::optional> environment; + std::optional input; + Source * standardIn = nullptr; + Sink * standardOut = nullptr; + bool mergeStderrToStdout = false; + bool isInteractive = false; +}; + +std::pair runProgram(RunOptions && options); + +void runProgram2(const RunOptions & options); + + +class ExecError : public Error +{ +public: + int status; + + template + ExecError(int status, const Args & ... args) + : Error(args...), status(status) + { } +}; + + +/** + * Convert the exit status of a child as returned by wait() into an + * error string. + */ +std::string statusToString(int status); + +bool statusOk(int status); + +} diff --git a/src/libutil/references.cc b/src/libutil/references.cc index 7f59b4c09d0..9d75606ef69 100644 --- a/src/libutil/references.cc +++ b/src/libutil/references.cc @@ -1,6 +1,5 @@ #include "references.hh" #include "hash.hh" -#include "util.hh" #include "archive.hh" #include diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 3d5121a19fa..f465bd0defd 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -1,5 +1,5 @@ #include "serialise.hh" -#include "util.hh" +#include "signals.hh" #include #include @@ -74,6 +74,10 @@ void Source::operator () (char * data, size_t len) } } +void Source::operator () (std::string_view data) +{ + (*this)((char *)data.data(), data.size()); +} void Source::drainInto(Sink & sink) { @@ -444,7 +448,7 @@ Error readError(Source & source) auto msg = readString(source); ErrorInfo info { .level = level, - .msg = hintformat(fmt("%s", msg)), + .msg = hintfmt(msg), }; auto havePos = readNum(source); assert(havePos == 0); @@ -453,7 +457,7 @@ Error readError(Source & source) havePos = readNum(source); assert(havePos == 0); info.traces.push_back(Trace { - .hint = hintformat(fmt("%s", readString(source))) + .hint = hintfmt(readString(source)) }); } return Error(std::move(info)); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 333c254ea8e..3f57ce88ba7 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -5,6 +5,7 @@ #include "types.hh" #include "util.hh" +#include "file-descriptor.hh" namespace boost::context { struct stack_context; } @@ -72,6 +73,7 @@ struct Source * an error if it is not going to be available. */ void operator () (char * data, size_t len); + void operator () (std::string_view data); /** * Store up to ‘len’ in the buffer pointed to by ‘data’, and diff --git a/src/libutil/signals.cc b/src/libutil/signals.cc new file mode 100644 index 00000000000..4632aa319d8 --- /dev/null +++ b/src/libutil/signals.cc @@ -0,0 +1,188 @@ +#include "signals.hh" +#include "util.hh" +#include "error.hh" +#include "sync.hh" +#include "terminal.hh" + +#include + +namespace nix { + +std::atomic _isInterrupted = false; + +static thread_local bool interruptThrown = false; +thread_local std::function interruptCheck; + +void setInterruptThrown() +{ + interruptThrown = true; +} + +void _interrupted() +{ + /* Block user interrupts while an exception is being handled. + Throwing an exception while another exception is being handled + kills the program! */ + if (!interruptThrown && !std::uncaught_exceptions()) { + interruptThrown = true; + throw Interrupted("interrupted by the user"); + } +} + + +////////////////////////////////////////////////////////////////////// + + +/* We keep track of interrupt callbacks using integer tokens, so we can iterate + safely without having to lock the data structure while executing arbitrary + functions. + */ +struct InterruptCallbacks { + typedef int64_t Token; + + /* We use unique tokens so that we can't accidentally delete the wrong + handler because of an erroneous double delete. */ + Token nextToken = 0; + + /* Used as a list, see InterruptCallbacks comment. */ + std::map> callbacks; +}; + +static Sync _interruptCallbacks; + +static void signalHandlerThread(sigset_t set) +{ + while (true) { + int signal = 0; + sigwait(&set, &signal); + + if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP) + triggerInterrupt(); + + else if (signal == SIGWINCH) { + updateWindowSize(); + } + } +} + +void triggerInterrupt() +{ + _isInterrupted = true; + + { + InterruptCallbacks::Token i = 0; + while (true) { + std::function callback; + { + auto interruptCallbacks(_interruptCallbacks.lock()); + auto lb = interruptCallbacks->callbacks.lower_bound(i); + if (lb == interruptCallbacks->callbacks.end()) + break; + + callback = lb->second; + i = lb->first + 1; + } + + try { + callback(); + } catch (...) { + ignoreException(); + } + } + } +} + + +static sigset_t savedSignalMask; +static bool savedSignalMaskIsSet = false; + +void setChildSignalMask(sigset_t * sigs) +{ + assert(sigs); // C style function, but think of sigs as a reference + +#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE + sigemptyset(&savedSignalMask); + // There's no "assign" or "copy" function, so we rely on (math) idempotence + // of the or operator: a or a = a. + sigorset(&savedSignalMask, sigs, sigs); +#else + // Without sigorset, our best bet is to assume that sigset_t is a type that + // can be assigned directly, such as is the case for a sigset_t defined as + // an integer type. + savedSignalMask = *sigs; +#endif + + savedSignalMaskIsSet = true; +} + +void saveSignalMask() { + if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) + throw SysError("querying signal mask"); + + savedSignalMaskIsSet = true; +} + +void startSignalHandlerThread() +{ + updateWindowSize(); + + saveSignalMask(); + + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGINT); + sigaddset(&set, SIGTERM); + sigaddset(&set, SIGHUP); + sigaddset(&set, SIGPIPE); + sigaddset(&set, SIGWINCH); + if (pthread_sigmask(SIG_BLOCK, &set, nullptr)) + throw SysError("blocking signals"); + + std::thread(signalHandlerThread, set).detach(); +} + +void restoreSignals() +{ + // If startSignalHandlerThread wasn't called, that means we're not running + // in a proper libmain process, but a process that presumably manages its + // own signal handlers. Such a process should call either + // - initNix(), to be a proper libmain process + // - startSignalHandlerThread(), to resemble libmain regarding signal + // handling only + // - saveSignalMask(), for processes that define their own signal handling + // thread + // TODO: Warn about this? Have a default signal mask? The latter depends on + // whether we should generally inherit signal masks from the caller. + // I don't know what the larger unix ecosystem expects from us here. + if (!savedSignalMaskIsSet) + return; + + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); +} + + +/* RAII helper to automatically deregister a callback. */ +struct InterruptCallbackImpl : InterruptCallback +{ + InterruptCallbacks::Token token; + ~InterruptCallbackImpl() override + { + auto interruptCallbacks(_interruptCallbacks.lock()); + interruptCallbacks->callbacks.erase(token); + } +}; + +std::unique_ptr createInterruptCallback(std::function callback) +{ + auto interruptCallbacks(_interruptCallbacks.lock()); + auto token = interruptCallbacks->nextToken++; + interruptCallbacks->callbacks.emplace(token, callback); + + auto res = std::make_unique(); + res->token = token; + + return std::unique_ptr(res.release()); +} + +} diff --git a/src/libutil/signals.hh b/src/libutil/signals.hh new file mode 100644 index 00000000000..7e8beff3326 --- /dev/null +++ b/src/libutil/signals.hh @@ -0,0 +1,104 @@ +#pragma once +///@file + +#include "types.hh" +#include "error.hh" +#include "logging.hh" +#include "ansicolor.hh" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace nix { + +/* User interruption. */ + +extern std::atomic _isInterrupted; + +extern thread_local std::function interruptCheck; + +void setInterruptThrown(); + +void _interrupted(); + +void inline checkInterrupt() +{ + if (_isInterrupted || (interruptCheck && interruptCheck())) + _interrupted(); +} + +MakeError(Interrupted, BaseError); + + +/** + * Start a thread that handles various signals. Also block those signals + * on the current thread (and thus any threads created by it). + * Saves the signal mask before changing the mask to block those signals. + * See saveSignalMask(). + */ +void startSignalHandlerThread(); + +/** + * Saves the signal mask, which is the signal mask that nix will restore + * before creating child processes. + * See setChildSignalMask() to set an arbitrary signal mask instead of the + * current mask. + */ +void saveSignalMask(); + +/** + * To use in a process that already called `startSignalHandlerThread()` + * or `saveSignalMask()` first. + */ +void restoreSignals(); + +/** + * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't + * necessarily match the current thread's mask. + * See saveSignalMask() to set the saved mask to the current mask. + */ +void setChildSignalMask(sigset_t *sigs); + +struct InterruptCallback +{ + virtual ~InterruptCallback() { }; +}; + +/** + * Register a function that gets called on SIGINT (in a non-signal + * context). + */ +std::unique_ptr createInterruptCallback( + std::function callback); + +void triggerInterrupt(); + +/** + * A RAII class that causes the current thread to receive SIGUSR1 when + * the signal handler thread receives SIGINT. That is, this allows + * SIGINT to be multiplexed to multiple threads. + */ +struct ReceiveInterrupts +{ + pthread_t target; + std::unique_ptr callback; + + ReceiveInterrupts() + : target(pthread_self()) + , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); })) + { } +}; + + +} diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc new file mode 100644 index 00000000000..e2114e18f66 --- /dev/null +++ b/src/libutil/source-accessor.cc @@ -0,0 +1,63 @@ +#include "source-accessor.hh" +#include "archive.hh" + +namespace nix { + +static std::atomic nextNumber{0}; + +SourceAccessor::SourceAccessor() + : number(++nextNumber) +{ +} + +bool SourceAccessor::pathExists(const CanonPath & path) +{ + return maybeLstat(path).has_value(); +} + +std::string SourceAccessor::readFile(const CanonPath & path) +{ + StringSink sink; + std::optional size; + readFile(path, sink, [&](uint64_t _size) + { + size = _size; + }); + assert(size && *size == sink.s.size()); + return std::move(sink.s); +} + +void SourceAccessor::readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) +{ + auto s = readFile(path); + sizeCallback(s.size()); + sink(s); +} + +Hash SourceAccessor::hashPath( + const CanonPath & path, + PathFilter & filter, + HashType ht) +{ + HashSink sink(ht); + dumpPath(path, sink, filter); + return sink.finish().first; +} + +SourceAccessor::Stat SourceAccessor::lstat(const CanonPath & path) +{ + if (auto st = maybeLstat(path)) + return *st; + else + throw Error("path '%s' does not exist", showPath(path)); +} + +std::string SourceAccessor::showPath(const CanonPath & path) +{ + return path.abs(); +} + +} diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh new file mode 100644 index 00000000000..1a4e803614f --- /dev/null +++ b/src/libutil/source-accessor.hh @@ -0,0 +1,123 @@ +#pragma once + +#include "canon-path.hh" +#include "hash.hh" + +namespace nix { + +struct Sink; + +/** + * A read-only filesystem abstraction. This is used by the Nix + * evaluator and elsewhere for accessing sources in various + * filesystem-like entities (such as the real filesystem, tarballs or + * Git repositories). + */ +struct SourceAccessor +{ + const size_t number; + + SourceAccessor(); + + virtual ~SourceAccessor() + { } + + /** + * Return the contents of a file as a string. + */ + virtual std::string readFile(const CanonPath & path); + + /** + * Write the contents of a file as a sink. `sizeCallback` must be + * called with the size of the file before any data is written to + * the sink. + * + * Note: subclasses of `SourceAccessor` need to implement at least + * one of the `readFile()` variants. + */ + virtual void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback = [](uint64_t size){}); + + virtual bool pathExists(const CanonPath & path); + + enum Type { + tRegular, tSymlink, tDirectory, + /** + Any other node types that may be encountered on the file system, such as device nodes, sockets, named pipe, and possibly even more exotic things. + + Responsible for `"unknown"` from `builtins.readFileType "/dev/null"`. + + Unlike `DT_UNKNOWN`, this must not be used for deferring the lookup of types. + */ + tMisc + }; + + struct Stat + { + Type type = tMisc; + + /** + * For regular files only: the size of the file. Not all + * accessors return this since it may be too expensive to + * compute. + */ + std::optional fileSize; + + /** + * For regular files only: whether this is an executable. + */ + bool isExecutable = false; + + /** + * For regular files only: the position of the contents of this + * file in the NAR. Only returned by NAR accessors. + */ + std::optional narOffset; + }; + + Stat lstat(const CanonPath & path); + + virtual std::optional maybeLstat(const CanonPath & path) = 0; + + typedef std::optional DirEntry; + + typedef std::map DirEntries; + + virtual DirEntries readDirectory(const CanonPath & path) = 0; + + virtual std::string readLink(const CanonPath & path) = 0; + + virtual void dumpPath( + const CanonPath & path, + Sink & sink, + PathFilter & filter = defaultPathFilter); + + Hash hashPath( + const CanonPath & path, + PathFilter & filter = defaultPathFilter, + HashType ht = htSHA256); + + /** + * Return a corresponding path in the root filesystem, if + * possible. This is only possible for filesystems that are + * materialized in the root filesystem. + */ + virtual std::optional getPhysicalPath(const CanonPath & path) + { return std::nullopt; } + + bool operator == (const SourceAccessor & x) const + { + return number == x.number; + } + + bool operator < (const SourceAccessor & x) const + { + return number < x.number; + } + + virtual std::string showPath(const CanonPath & path); +}; + +} diff --git a/src/libutil/suggestions.cc b/src/libutil/suggestions.cc index 9510a5f0c41..e67e986fb59 100644 --- a/src/libutil/suggestions.cc +++ b/src/libutil/suggestions.cc @@ -1,7 +1,9 @@ #include "suggestions.hh" #include "ansicolor.hh" -#include "util.hh" +#include "terminal.hh" + #include +#include namespace nix { diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 5060a8f24a0..1733c791c34 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -3,6 +3,7 @@ #include "serialise.hh" #include "tarfile.hh" +#include "file-system.hh" namespace nix { diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc new file mode 100644 index 00000000000..8febc8771e3 --- /dev/null +++ b/src/libutil/terminal.cc @@ -0,0 +1,108 @@ +#include "terminal.hh" +#include "environment-variables.hh" +#include "sync.hh" + +#include +#include + +namespace nix { + +bool shouldANSI() +{ + return isatty(STDERR_FILENO) + && getEnv("TERM").value_or("dumb") != "dumb" + && !(getEnv("NO_COLOR").has_value() || getEnv("NOCOLOR").has_value()); +} + +std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) +{ + std::string t, e; + size_t w = 0; + auto i = s.begin(); + + while (w < (size_t) width && i != s.end()) { + + if (*i == '\e') { + std::string e; + e += *i++; + char last = 0; + + if (i != s.end() && *i == '[') { + e += *i++; + // eat parameter bytes + while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; + // eat intermediate bytes + while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; + // eat final byte + if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; + } else { + if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; + } + + if (!filterAll && last == 'm') + t += e; + } + + else if (*i == '\t') { + i++; t += ' '; w++; + while (w < (size_t) width && w % 8) { + t += ' '; w++; + } + } + + else if (*i == '\r' || *i == '\a') + // do nothing for now + i++; + + else { + w++; + // Copy one UTF-8 character. + if ((*i & 0xe0) == 0xc0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } else if ((*i & 0xf0) == 0xe0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } + } else if ((*i & 0xf8) == 0xf0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } + } + } else + t += *i++; + } + } + + return t; +} + + +////////////////////////////////////////////////////////////////////// + +static Sync> windowSize{{0, 0}}; + + +void updateWindowSize() +{ + struct winsize ws; + if (ioctl(2, TIOCGWINSZ, &ws) == 0) { + auto windowSize_(windowSize.lock()); + windowSize_->first = ws.ws_row; + windowSize_->second = ws.ws_col; + } +} + + +std::pair getWindowSize() +{ + return *windowSize.lock(); +} + +} diff --git a/src/libutil/terminal.hh b/src/libutil/terminal.hh new file mode 100644 index 00000000000..9cb191308da --- /dev/null +++ b/src/libutil/terminal.hh @@ -0,0 +1,38 @@ +#pragma once +///@file + +#include "types.hh" + +namespace nix { +/** + * Determine whether ANSI escape sequences are appropriate for the + * present output. + */ +bool shouldANSI(); + +/** + * Truncate a string to 'width' printable characters. If 'filterAll' + * is true, all ANSI escape sequences are filtered out. Otherwise, + * some escape sequences (such as colour setting) are copied but not + * included in the character count. Also, tabs are expanded to + * spaces. + */ +std::string filterANSIEscapes(std::string_view s, + bool filterAll = false, + unsigned int width = std::numeric_limits::max()); + +/** + * Recalculate the window size, updating a global variable. Used in the + * `SIGWINCH` signal handler. + */ +void updateWindowSize(); + +/** + * @return the number of rows and columns of the terminal. + * + * The value is cached so this is quick. The cached result is computed + * by `updateWindowSize()`. + */ +std::pair getWindowSize(); + +} diff --git a/src/libutil/tests/git.cc b/src/libutil/tests/git.cc deleted file mode 100644 index 5b5715fc26b..00000000000 --- a/src/libutil/tests/git.cc +++ /dev/null @@ -1,33 +0,0 @@ -#include "git.hh" -#include - -namespace nix { - - TEST(GitLsRemote, parseSymrefLineWithReference) { - auto line = "ref: refs/head/main HEAD"; - auto res = git::parseLsRemoteLine(line); - ASSERT_TRUE(res.has_value()); - ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic); - ASSERT_EQ(res->target, "refs/head/main"); - ASSERT_EQ(res->reference, "HEAD"); - } - - TEST(GitLsRemote, parseSymrefLineWithNoReference) { - auto line = "ref: refs/head/main"; - auto res = git::parseLsRemoteLine(line); - ASSERT_TRUE(res.has_value()); - ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic); - ASSERT_EQ(res->target, "refs/head/main"); - ASSERT_EQ(res->reference, std::nullopt); - } - - TEST(GitLsRemote, parseObjectRefLine) { - auto line = "abc123 refs/head/main"; - auto res = git::parseLsRemoteLine(line); - ASSERT_TRUE(res.has_value()); - ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Object); - ASSERT_EQ(res->target, "abc123"); - ASSERT_EQ(res->reference, "refs/head/main"); - } -} - diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk deleted file mode 100644 index 167915439fd..00000000000 --- a/src/libutil/tests/local.mk +++ /dev/null @@ -1,29 +0,0 @@ -check: libutil-tests_RUN - -programs += libutil-tests - -libutil-tests-exe_NAME = libnixutil-tests - -libutil-tests-exe_DIR := $(d) - -libutil-tests-exe_INSTALL_DIR := - -libutil-tests-exe_LIBS = libutil-tests - -libutil-tests-exe_LDFLAGS := $(GTEST_LIBS) - -libraries += libutil-tests - -libutil-tests_NAME = libnixutil-tests - -libutil-tests_DIR := $(d) - -libutil-tests_INSTALL_DIR := - -libutil-tests_SOURCES := $(wildcard $(d)/*.cc) - -libutil-tests_CXXFLAGS += -I src/libutil - -libutil-tests_LIBS = libutil - -libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index dc4067f1b3a..9a7dfee568f 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -1,4 +1,6 @@ #include "thread-pool.hh" +#include "signals.hh" +#include "util.hh" namespace nix { @@ -77,6 +79,8 @@ void ThreadPool::process() void ThreadPool::doWork(bool mainThread) { + ReceiveInterrupts receiveInterrupts; + if (!mainThread) interruptCheck = [&]() { return (bool) quit; }; diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh index 0e09fae9762..02765badc82 100644 --- a/src/libutil/thread-pool.hh +++ b/src/libutil/thread-pool.hh @@ -1,8 +1,8 @@ #pragma once ///@file +#include "error.hh" #include "sync.hh" -#include "util.hh" #include #include diff --git a/src/libutil/unix-domain-socket.cc b/src/libutil/unix-domain-socket.cc new file mode 100644 index 00000000000..8949461d247 --- /dev/null +++ b/src/libutil/unix-domain-socket.cc @@ -0,0 +1,100 @@ +#include "file-system.hh" +#include "processes.hh" +#include "unix-domain-socket.hh" + +#include +#include +#include + +namespace nix { + +AutoCloseFD createUnixDomainSocket() +{ + AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM + #ifdef SOCK_CLOEXEC + | SOCK_CLOEXEC + #endif + , 0); + if (!fdSocket) + throw SysError("cannot create Unix domain socket"); + closeOnExec(fdSocket.get()); + return fdSocket; +} + + +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) +{ + auto fdSocket = nix::createUnixDomainSocket(); + + bind(fdSocket.get(), path); + + if (chmod(path.c_str(), mode) == -1) + throw SysError("changing permissions on '%1%'", path); + + if (listen(fdSocket.get(), 100) == -1) + throw SysError("cannot listen on socket '%1%'", path); + + return fdSocket; +} + + +void bind(int fd, const std::string & path) +{ + unlink(path.c_str()); + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + + if (path.size() + 1 >= sizeof(addr.sun_path)) { + Pid pid = startProcess([&]() { + Path dir = dirOf(path); + if (chdir(dir.c_str()) == -1) + throw SysError("chdir to '%s' failed", dir); + std::string base(baseNameOf(path)); + if (base.size() + 1 >= sizeof(addr.sun_path)) + throw Error("socket path '%s' is too long", base); + memcpy(addr.sun_path, base.c_str(), base.size() + 1); + if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot bind to socket '%s'", path); + _exit(0); + }); + int status = pid.wait(); + if (status != 0) + throw Error("cannot bind to socket '%s'", path); + } else { + memcpy(addr.sun_path, path.c_str(), path.size() + 1); + if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot bind to socket '%s'", path); + } +} + + +void connect(int fd, const std::string & path) +{ + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + + if (path.size() + 1 >= sizeof(addr.sun_path)) { + Pid pid = startProcess([&]() { + Path dir = dirOf(path); + if (chdir(dir.c_str()) == -1) + throw SysError("chdir to '%s' failed", dir); + std::string base(baseNameOf(path)); + if (base.size() + 1 >= sizeof(addr.sun_path)) + throw Error("socket path '%s' is too long", base); + memcpy(addr.sun_path, base.c_str(), base.size() + 1); + if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot connect to socket at '%s'", path); + _exit(0); + }); + int status = pid.wait(); + if (status != 0) + throw Error("cannot connect to socket at '%s'", path); + } else { + memcpy(addr.sun_path, path.c_str(), path.size() + 1); + if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot connect to socket at '%s'", path); + } +} + +} diff --git a/src/libutil/unix-domain-socket.hh b/src/libutil/unix-domain-socket.hh new file mode 100644 index 00000000000..b78feb454b1 --- /dev/null +++ b/src/libutil/unix-domain-socket.hh @@ -0,0 +1,31 @@ +#pragma once +///@file + +#include "types.hh" +#include "file-descriptor.hh" + +#include + +namespace nix { + +/** + * Create a Unix domain socket. + */ +AutoCloseFD createUnixDomainSocket(); + +/** + * Create a Unix domain socket in listen mode. + */ +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); + +/** + * Bind a Unix domain socket to a path. + */ +void bind(int fd, const std::string & path); + +/** + * Connect to a Unix domain socket. + */ +void connect(int fd, const std::string & path); + +} diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 98162b0f78a..07bc8d0cde9 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -8,7 +8,7 @@ namespace nix { // URI stuff. const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; -const static std::string schemeRegex = "(?:[a-z][a-z0-9+.-]*)"; +const static std::string schemeNameRegex = "(?:[a-z][a-z0-9+.-]*)"; const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+(?:%\\w+)?"; const static std::string ipv6AddressRegex = "(?:\\[" + ipv6AddressSegmentRegex + "\\]|" + ipv6AddressSegmentRegex + ")"; const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; @@ -30,7 +30,7 @@ extern std::regex refRegex; /// Instead of defining what a good Git Ref is, we define what a bad Git Ref is /// This is because of the definition of a ref in refs.c in https://github.com/git/git -/// See tests/fetchGitRefs.sh for the full definition +/// See tests/functional/fetchGitRefs.sh for the full definition const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$"; extern std::regex badGitRefRegex; @@ -41,7 +41,4 @@ extern std::regex revRegex; /// A ref or revision, or a ref followed by a revision. const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; -const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; -extern std::regex flakeIdRegex; - } diff --git a/src/libutil/url.cc b/src/libutil/url.cc index a8f7d39fd0b..fad438ce5be 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -8,12 +8,11 @@ namespace nix { std::regex refRegex(refRegexS, std::regex::ECMAScript); std::regex badGitRefRegex(badGitRefRegexS, std::regex::ECMAScript); std::regex revRegex(revRegexS, std::regex::ECMAScript); -std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript); ParsedURL parseURL(const std::string & url) { static std::regex uriRegex( - "((" + schemeRegex + "):" + "((" + schemeNameRegex + "):" + "(?:(?://(" + authorityRegex + ")(" + absPathRegex + "))|(/?" + pathRegex + ")))" + "(?:\\?(" + queryRegex + "))?" + "(?:#(" + queryRegex + "))?", @@ -103,7 +102,7 @@ std::string percentEncode(std::string_view s, std::string_view keep) || keep.find(c) != std::string::npos) res += c; else - res += fmt("%%%02X", (unsigned int) c); + res += fmt("%%%02X", c & 0xFF); return res; } @@ -159,4 +158,29 @@ ParsedUrlScheme parseUrlScheme(std::string_view scheme) }; } +std::string fixGitURL(const std::string & url) +{ + std::regex scpRegex("([^/]*)@(.*):(.*)"); + if (!hasPrefix(url, "/") && std::regex_match(url, scpRegex)) + return std::regex_replace(url, scpRegex, "ssh://$1@$2/$3"); + else { + if (url.find("://") == std::string::npos) { + return (ParsedURL { + .scheme = "file", + .authority = "", + .path = url + }).to_string(); + } else + return url; + } +} + +// https://www.rfc-editor.org/rfc/rfc3986#section-3.1 +bool isValidSchemeName(std::string_view s) +{ + static std::regex regex(schemeNameRegex, std::regex::ECMAScript); + + return std::regex_match(s.begin(), s.end(), regex, std::regex_constants::match_default); +} + } diff --git a/src/libutil/url.hh b/src/libutil/url.hh index d2413ec0efc..603ab35b622 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -45,4 +45,18 @@ struct ParsedUrlScheme { ParsedUrlScheme parseUrlScheme(std::string_view scheme); +/* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes + them by removing the `:` and assuming a scheme of `ssh://`. Also + changes absolute paths into file:// URLs. */ +std::string fixGitURL(const std::string & url); + +/** + * Whether a string is valid as RFC 3986 scheme name. + * Colon `:` is part of the URI; not the scheme name, and therefore rejected. + * See https://www.rfc-editor.org/rfc/rfc3986#section-3.1 + * + * Does not check whether the scheme is understood, as that's context-dependent. + */ +bool isValidSchemeName(std::string_view scheme); + } diff --git a/src/libutil/users.cc b/src/libutil/users.cc new file mode 100644 index 00000000000..95a641322a4 --- /dev/null +++ b/src/libutil/users.cc @@ -0,0 +1,116 @@ +#include "util.hh" +#include "users.hh" +#include "environment-variables.hh" +#include "file-system.hh" + +#include +#include +#include + +namespace nix { + +std::string getUserName() +{ + auto pw = getpwuid(geteuid()); + std::string name = pw ? pw->pw_name : getEnv("USER").value_or(""); + if (name.empty()) + throw Error("cannot figure out user name"); + return name; +} + +Path getHomeOf(uid_t userId) +{ + std::vector buf(16384); + struct passwd pwbuf; + struct passwd * pw; + if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0 + || !pw || !pw->pw_dir || !pw->pw_dir[0]) + throw Error("cannot determine user's home directory"); + return pw->pw_dir; +} + +Path getHome() +{ + static Path homeDir = []() + { + std::optional unownedUserHomeDir = {}; + auto homeDir = getEnv("HOME"); + if (homeDir) { + // Only use $HOME if doesn't exist or is owned by the current user. + struct stat st; + int result = stat(homeDir->c_str(), &st); + if (result != 0) { + if (errno != ENOENT) { + warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); + homeDir.reset(); + } + } else if (st.st_uid != geteuid()) { + unownedUserHomeDir.swap(homeDir); + } + } + if (!homeDir) { + homeDir = getHomeOf(geteuid()); + if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { + warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); + } + } + return *homeDir; + }(); + return homeDir; +} + + +Path getCacheDir() +{ + auto cacheDir = getEnv("XDG_CACHE_HOME"); + return cacheDir ? *cacheDir : getHome() + "/.cache"; +} + + +Path getConfigDir() +{ + auto configDir = getEnv("XDG_CONFIG_HOME"); + return configDir ? *configDir : getHome() + "/.config"; +} + +std::vector getConfigDirs() +{ + Path configHome = getConfigDir(); + auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); + std::vector result = tokenizeString>(configDirs, ":"); + result.insert(result.begin(), configHome); + return result; +} + + +Path getDataDir() +{ + auto dataDir = getEnv("XDG_DATA_HOME"); + return dataDir ? *dataDir : getHome() + "/.local/share"; +} + +Path getStateDir() +{ + auto stateDir = getEnv("XDG_STATE_HOME"); + return stateDir ? *stateDir : getHome() + "/.local/state"; +} + +Path createNixStateDir() +{ + Path dir = getStateDir() + "/nix"; + createDirs(dir); + return dir; +} + + +std::string expandTilde(std::string_view path) +{ + // TODO: expand ~user ? + auto tilde = path.substr(0, 2); + if (tilde == "~/" || tilde == "~") + return getHome() + std::string(path.substr(1)); + else + return std::string(path); +} + +} diff --git a/src/libutil/users.hh b/src/libutil/users.hh new file mode 100644 index 00000000000..cecbb8bfb9e --- /dev/null +++ b/src/libutil/users.hh @@ -0,0 +1,58 @@ +#pragma once +///@file + +#include "types.hh" + +#include + +namespace nix { + +std::string getUserName(); + +/** + * @return the given user's home directory from /etc/passwd. + */ +Path getHomeOf(uid_t userId); + +/** + * @return $HOME or the user's home directory from /etc/passwd. + */ +Path getHome(); + +/** + * @return $XDG_CACHE_HOME or $HOME/.cache. + */ +Path getCacheDir(); + +/** + * @return $XDG_CONFIG_HOME or $HOME/.config. + */ +Path getConfigDir(); + +/** + * @return the directories to search for user configuration files + */ +std::vector getConfigDirs(); + +/** + * @return $XDG_DATA_HOME or $HOME/.local/share. + */ +Path getDataDir(); + +/** + * @return $XDG_STATE_HOME or $HOME/.local/state. + */ +Path getStateDir(); + +/** + * Create the Nix state directory and return the path to it. + */ +Path createNixStateDir(); + +/** + * Perform tilde expansion on a path, replacing tilde with the user's + * home directory. + */ +std::string expandTilde(std::string_view path); + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 5a10c69e2df..5bb3f374ba7 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,1164 +1,36 @@ #include "util.hh" -#include "sync.hh" -#include "finally.hh" -#include "serialise.hh" -#include "cgroup.hh" +#include "fmt.hh" #include #include -#include -#include -#include -#include -#include -#include #include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -#include -#endif - -#ifdef __linux__ -#include -#include -#include - -#include -#endif - - -extern char * * environ __attribute__((weak)); - - -namespace nix { - -void initLibUtil() { - // Check that exception handling works. Exception handling has been observed - // not to work on darwin when the linker flags aren't quite right. - // In this case we don't want to expose the user to some unrelated uncaught - // exception, but rather tell them exactly that exception handling is - // broken. - // When exception handling fails, the message tends to be printed by the - // C++ runtime, followed by an abort. - // For example on macOS we might see an error such as - // libc++abi: terminating with uncaught exception of type nix::SysError: error: C++ exception handling is broken. This would appear to be a problem with the way Nix was compiled and/or linked and/or loaded. - bool caught = false; - try { - throwExceptionSelfCheck(); - } catch (const nix::Error & _e) { - caught = true; - } - // This is not actually the main point of this check, but let's make sure anyway: - assert(caught); -} - -std::optional getEnv(const std::string & key) -{ - char * value = getenv(key.c_str()); - if (!value) return {}; - return std::string(value); -} - -std::optional getEnvNonEmpty(const std::string & key) { - auto value = getEnv(key); - if (value == "") return {}; - return value; -} - -std::map getEnv() -{ - std::map env; - for (size_t i = 0; environ[i]; ++i) { - auto s = environ[i]; - auto eq = strchr(s, '='); - if (!eq) - // invalid env, just keep going - continue; - env.emplace(std::string(s, eq), std::string(eq + 1)); - } - return env; -} - - -void clearEnv() -{ - for (auto & name : getEnv()) - unsetenv(name.first.c_str()); -} - -void replaceEnv(const std::map & newEnv) -{ - clearEnv(); - for (auto & newEnvVar : newEnv) - setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); -} - - -Path absPath(Path path, std::optional dir, bool resolveSymlinks) -{ - if (path[0] != '/') { - if (!dir) { -#ifdef __GNU__ - /* GNU (aka. GNU/Hurd) doesn't have any limitation on path - lengths and doesn't define `PATH_MAX'. */ - char *buf = getcwd(NULL, 0); - if (buf == NULL) -#else - char buf[PATH_MAX]; - if (!getcwd(buf, sizeof(buf))) -#endif - throw SysError("cannot get cwd"); - path = concatStrings(buf, "/", path); -#ifdef __GNU__ - free(buf); -#endif - } else - path = concatStrings(*dir, "/", path); - } - return canonPath(path, resolveSymlinks); -} - - -Path canonPath(PathView path, bool resolveSymlinks) -{ - assert(path != ""); - - std::string s; - s.reserve(256); - - if (path[0] != '/') - throw Error("not an absolute path: '%1%'", path); - - std::string temp; - - /* Count the number of times we follow a symlink and stop at some - arbitrary (but high) limit to prevent infinite loops. */ - unsigned int followCount = 0, maxFollow = 1024; - - while (1) { - - /* Skip slashes. */ - while (!path.empty() && path[0] == '/') path.remove_prefix(1); - if (path.empty()) break; - - /* Ignore `.'. */ - if (path == "." || path.substr(0, 2) == "./") - path.remove_prefix(1); - - /* If `..', delete the last component. */ - else if (path == ".." || path.substr(0, 3) == "../") - { - if (!s.empty()) s.erase(s.rfind('/')); - path.remove_prefix(2); - } - - /* Normal component; copy it. */ - else { - s += '/'; - if (const auto slash = path.find('/'); slash == std::string::npos) { - s += path; - path = {}; - } else { - s += path.substr(0, slash); - path = path.substr(slash); - } - - /* If s points to a symlink, resolve it and continue from there */ - if (resolveSymlinks && isLink(s)) { - if (++followCount >= maxFollow) - throw Error("infinite symlink recursion in path '%1%'", path); - temp = concatStrings(readLink(s), path); - path = temp; - if (!temp.empty() && temp[0] == '/') { - s.clear(); /* restart for symlinks pointing to absolute path */ - } else { - s = dirOf(s); - if (s == "/") { // we don’t want trailing slashes here, which dirOf only produces if s = / - s.clear(); - } - } - } - } - } - - return s.empty() ? "/" : std::move(s); -} - - -Path dirOf(const PathView path) -{ - Path::size_type pos = path.rfind('/'); - if (pos == std::string::npos) - return "."; - return pos == 0 ? "/" : Path(path, 0, pos); -} - - -std::string_view baseNameOf(std::string_view path) -{ - if (path.empty()) - return ""; - - auto last = path.size() - 1; - if (path[last] == '/' && last > 0) - last -= 1; - - auto pos = path.rfind('/', last); - if (pos == std::string::npos) - pos = 0; - else - pos += 1; - - return path.substr(pos, last - pos + 1); -} - - -std::string expandTilde(std::string_view path) -{ - // TODO: expand ~user ? - auto tilde = path.substr(0, 2); - if (tilde == "~/" || tilde == "~") - return getHome() + std::string(path.substr(1)); - else - return std::string(path); -} - - -bool isInDir(std::string_view path, std::string_view dir) -{ - return path.substr(0, 1) == "/" - && path.substr(0, dir.size()) == dir - && path.size() >= dir.size() + 2 - && path[dir.size()] == '/'; -} - - -bool isDirOrInDir(std::string_view path, std::string_view dir) -{ - return path == dir || isInDir(path, dir); -} - - -struct stat stat(const Path & path) -{ - struct stat st; - if (stat(path.c_str(), &st)) - throw SysError("getting status of '%1%'", path); - return st; -} - - -struct stat lstat(const Path & path) -{ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting status of '%1%'", path); - return st; -} - - -bool pathExists(const Path & path) -{ - int res; - struct stat st; - res = lstat(path.c_str(), &st); - if (!res) return true; - if (errno != ENOENT && errno != ENOTDIR) - throw SysError("getting status of %1%", path); - return false; -} - -bool pathAccessible(const Path & path) -{ - try { - return pathExists(path); - } catch (SysError & e) { - // swallow EPERM - if (e.errNo == EPERM) return false; - throw; - } -} - - -Path readLink(const Path & path) -{ - checkInterrupt(); - std::vector buf; - for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) { - buf.resize(bufSize); - ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize); - if (rlSize == -1) - if (errno == EINVAL) - throw Error("'%1%' is not a symlink", path); - else - throw SysError("reading symbolic link '%1%'", path); - else if (rlSize < bufSize) - return std::string(buf.data(), rlSize); - } -} - - -bool isLink(const Path & path) -{ - struct stat st = lstat(path); - return S_ISLNK(st.st_mode); -} - - -DirEntries readDirectory(DIR *dir, const Path & path) -{ - DirEntries entries; - entries.reserve(64); - - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir)) { /* sic */ - checkInterrupt(); - std::string name = dirent->d_name; - if (name == "." || name == "..") continue; - entries.emplace_back(name, dirent->d_ino, -#ifdef HAVE_STRUCT_DIRENT_D_TYPE - dirent->d_type -#else - DT_UNKNOWN -#endif - ); - } - if (errno) throw SysError("reading directory '%1%'", path); - - return entries; -} - -DirEntries readDirectory(const Path & path) -{ - AutoCloseDir dir(opendir(path.c_str())); - if (!dir) throw SysError("opening directory '%1%'", path); - - return readDirectory(dir.get(), path); -} - - -unsigned char getFileType(const Path & path) -{ - struct stat st = lstat(path); - if (S_ISDIR(st.st_mode)) return DT_DIR; - if (S_ISLNK(st.st_mode)) return DT_LNK; - if (S_ISREG(st.st_mode)) return DT_REG; - return DT_UNKNOWN; -} - - -std::string readFile(int fd) -{ - struct stat st; - if (fstat(fd, &st) == -1) - throw SysError("statting file"); - - return drainFD(fd, true, st.st_size); -} - - -std::string readFile(const Path & path) -{ - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) - throw SysError("opening file '%1%'", path); - return readFile(fd.get()); -} - - -void readFile(const Path & path, Sink & sink) -{ - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) - throw SysError("opening file '%s'", path); - drainFD(fd.get(), sink); -} - - -void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync) -{ - AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); - if (!fd) - throw SysError("opening file '%1%'", path); - try { - writeFull(fd.get(), s); - } catch (Error & e) { - e.addTrace({}, "writing file '%1%'", path); - throw; - } - if (sync) - fd.fsync(); - // Explicitly close to make sure exceptions are propagated. - fd.close(); - if (sync) - syncParent(path); -} - - -void writeFile(const Path & path, Source & source, mode_t mode, bool sync) -{ - AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); - if (!fd) - throw SysError("opening file '%1%'", path); - - std::vector buf(64 * 1024); - - try { - while (true) { - try { - auto n = source.read(buf.data(), buf.size()); - writeFull(fd.get(), {buf.data(), n}); - } catch (EndOfFile &) { break; } - } - } catch (Error & e) { - e.addTrace({}, "writing file '%1%'", path); - throw; - } - if (sync) - fd.fsync(); - // Explicitly close to make sure exceptions are propagated. - fd.close(); - if (sync) - syncParent(path); -} - -void syncParent(const Path & path) -{ - AutoCloseFD fd = open(dirOf(path).c_str(), O_RDONLY, 0); - if (!fd) - throw SysError("opening file '%1%'", path); - fd.fsync(); -} - -std::string readLine(int fd) -{ - std::string s; - while (1) { - checkInterrupt(); - char ch; - // FIXME: inefficient - ssize_t rd = read(fd, &ch, 1); - if (rd == -1) { - if (errno != EINTR) - throw SysError("reading a line"); - } else if (rd == 0) - throw EndOfFile("unexpected EOF reading a line"); - else { - if (ch == '\n') return s; - s += ch; - } - } -} - - -void writeLine(int fd, std::string s) -{ - s += '\n'; - writeFull(fd, s); -} - - -static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) -{ - checkInterrupt(); - - std::string name(baseNameOf(path)); - - struct stat st; - if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { - if (errno == ENOENT) return; - throw SysError("getting status of '%1%'", path); - } - - if (!S_ISDIR(st.st_mode)) { - /* We are about to delete a file. Will it likely free space? */ - - switch (st.st_nlink) { - /* Yes: last link. */ - case 1: - bytesFreed += st.st_size; - break; - /* Maybe: yes, if 'auto-optimise-store' or manual optimisation - was performed. Instead of checking for real let's assume - it's an optimised file and space will be freed. - - In worst case we will double count on freed space for files - with exactly two hardlinks for unoptimised packages. - */ - case 2: - bytesFreed += st.st_size; - break; - /* No: 3+ links. */ - default: - break; - } - } - - if (S_ISDIR(st.st_mode)) { - /* Make the directory accessible. */ - const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; - if ((st.st_mode & PERM_MASK) != PERM_MASK) { - if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) - throw SysError("chmod '%1%'", path); - } - - int fd = openat(parentfd, path.c_str(), O_RDONLY); - if (fd == -1) - throw SysError("opening directory '%1%'", path); - AutoCloseDir dir(fdopendir(fd)); - if (!dir) - throw SysError("opening directory '%1%'", path); - for (auto & i : readDirectory(dir.get(), path)) - _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed); - } - - int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; - if (unlinkat(parentfd, name.c_str(), flags) == -1) { - if (errno == ENOENT) return; - throw SysError("cannot unlink '%1%'", path); - } -} - -static void _deletePath(const Path & path, uint64_t & bytesFreed) -{ - Path dir = dirOf(path); - if (dir == "") - dir = "/"; - - AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)}; - if (!dirfd) { - if (errno == ENOENT) return; - throw SysError("opening directory '%1%'", path); - } - - _deletePath(dirfd.get(), path, bytesFreed); -} - - -void deletePath(const Path & path) -{ - uint64_t dummy; - deletePath(path, dummy); -} - - -void deletePath(const Path & path, uint64_t & bytesFreed) -{ - //Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path); - bytesFreed = 0; - _deletePath(path, bytesFreed); -} - - -std::string getUserName() -{ - auto pw = getpwuid(geteuid()); - std::string name = pw ? pw->pw_name : getEnv("USER").value_or(""); - if (name.empty()) - throw Error("cannot figure out user name"); - return name; -} - -Path getHomeOf(uid_t userId) -{ - std::vector buf(16384); - struct passwd pwbuf; - struct passwd * pw; - if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0 - || !pw || !pw->pw_dir || !pw->pw_dir[0]) - throw Error("cannot determine user's home directory"); - return pw->pw_dir; -} - -Path getHome() -{ - static Path homeDir = []() - { - std::optional unownedUserHomeDir = {}; - auto homeDir = getEnv("HOME"); - if (homeDir) { - // Only use $HOME if doesn't exist or is owned by the current user. - struct stat st; - int result = stat(homeDir->c_str(), &st); - if (result != 0) { - if (errno != ENOENT) { - warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); - homeDir.reset(); - } - } else if (st.st_uid != geteuid()) { - unownedUserHomeDir.swap(homeDir); - } - } - if (!homeDir) { - homeDir = getHomeOf(geteuid()); - if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { - warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); - } - } - return *homeDir; - }(); - return homeDir; -} - - -Path getCacheDir() -{ - auto cacheDir = getEnv("XDG_CACHE_HOME"); - return cacheDir ? *cacheDir : getHome() + "/.cache"; -} - - -Path getConfigDir() -{ - auto configDir = getEnv("XDG_CONFIG_HOME"); - return configDir ? *configDir : getHome() + "/.config"; -} - -std::vector getConfigDirs() -{ - Path configHome = getConfigDir(); - auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); - std::vector result = tokenizeString>(configDirs, ":"); - result.insert(result.begin(), configHome); - return result; -} - - -Path getDataDir() -{ - auto dataDir = getEnv("XDG_DATA_HOME"); - return dataDir ? *dataDir : getHome() + "/.local/share"; -} - -Path getStateDir() -{ - auto stateDir = getEnv("XDG_STATE_HOME"); - return stateDir ? *stateDir : getHome() + "/.local/state"; -} - -Path createNixStateDir() -{ - Path dir = getStateDir() + "/nix"; - createDirs(dir); - return dir; -} - - -std::optional getSelfExe() -{ - static auto cached = []() -> std::optional - { - #if __linux__ - return readLink("/proc/self/exe"); - #elif __APPLE__ - char buf[1024]; - uint32_t size = sizeof(buf); - if (_NSGetExecutablePath(buf, &size) == 0) - return buf; - else - return std::nullopt; - #else - return std::nullopt; - #endif - }(); - return cached; -} - - -Paths createDirs(const Path & path) -{ - Paths created; - if (path == "/") return created; - - struct stat st; - if (lstat(path.c_str(), &st) == -1) { - created = createDirs(dirOf(path)); - if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) - throw SysError("creating directory '%1%'", path); - st = lstat(path); - created.push_back(path); - } - - if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) - throw SysError("statting symlink '%1%'", path); - - if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path); - - return created; -} - - -void readFull(int fd, char * buf, size_t count) -{ - while (count) { - checkInterrupt(); - ssize_t res = read(fd, buf, count); - if (res == -1) { - if (errno == EINTR) continue; - throw SysError("reading from file"); - } - if (res == 0) throw EndOfFile("unexpected end-of-file"); - count -= res; - buf += res; - } -} - - -void writeFull(int fd, std::string_view s, bool allowInterrupts) -{ - while (!s.empty()) { - if (allowInterrupts) checkInterrupt(); - ssize_t res = write(fd, s.data(), s.size()); - if (res == -1 && errno != EINTR) - throw SysError("writing to file"); - if (res > 0) - s.remove_prefix(res); - } -} - - -std::string drainFD(int fd, bool block, const size_t reserveSize) -{ - // the parser needs two extra bytes to append terminating characters, other users will - // not care very much about the extra memory. - StringSink sink(reserveSize + 2); - drainFD(fd, sink, block); - return std::move(sink.s); -} - - -void drainFD(int fd, Sink & sink, bool block) -{ - // silence GCC maybe-uninitialized warning in finally - int saved = 0; - - if (!block) { - saved = fcntl(fd, F_GETFL); - if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) - throw SysError("making file descriptor non-blocking"); - } - - Finally finally([&]() { - if (!block) { - if (fcntl(fd, F_SETFL, saved) == -1) - throw SysError("making file descriptor blocking"); - } - }); - - std::vector buf(64 * 1024); - while (1) { - checkInterrupt(); - ssize_t rd = read(fd, buf.data(), buf.size()); - if (rd == -1) { - if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) - break; - if (errno != EINTR) - throw SysError("reading from file"); - } - else if (rd == 0) break; - else sink({(char *) buf.data(), (size_t) rd}); - } -} - -////////////////////////////////////////////////////////////////////// - -unsigned int getMaxCPU() -{ - #if __linux__ - try { - auto cgroupFS = getCgroupFS(); - if (!cgroupFS) return 0; - - auto cgroups = getCgroups("/proc/self/cgroup"); - auto cgroup = cgroups[""]; - if (cgroup == "") return 0; - - auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max"; - - auto cpuMax = readFile(cpuFile); - auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); - auto quota = cpuMaxParts[0]; - auto period = cpuMaxParts[1]; - if (quota != "max") - return std::ceil(std::stoi(quota) / std::stof(period)); - } catch (Error &) { ignoreException(lvlDebug); } - #endif - - return 0; -} - -////////////////////////////////////////////////////////////////////// - - -AutoDelete::AutoDelete() : del{false} {} - -AutoDelete::AutoDelete(const std::string & p, bool recursive) : path(p) -{ - del = true; - this->recursive = recursive; -} - -AutoDelete::~AutoDelete() -{ - try { - if (del) { - if (recursive) - deletePath(path); - else { - if (remove(path.c_str()) == -1) - throw SysError("cannot unlink '%1%'", path); - } - } - } catch (...) { - ignoreException(); - } -} - -void AutoDelete::cancel() -{ - del = false; -} - -void AutoDelete::reset(const Path & p, bool recursive) { - path = p; - this->recursive = recursive; - del = true; -} - - - -////////////////////////////////////////////////////////////////////// - - -AutoCloseFD::AutoCloseFD() : fd{-1} {} - - -AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} - - -AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd} -{ - that.fd = -1; -} - - -AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that) -{ - close(); - fd = that.fd; - that.fd = -1; - return *this; -} - - -AutoCloseFD::~AutoCloseFD() -{ - try { - close(); - } catch (...) { - ignoreException(); - } -} - - -int AutoCloseFD::get() const -{ - return fd; -} - - -void AutoCloseFD::close() -{ - if (fd != -1) { - if (::close(fd) == -1) - /* This should never happen. */ - throw SysError("closing file descriptor %1%", fd); - fd = -1; - } -} - -void AutoCloseFD::fsync() -{ - if (fd != -1) { - int result; -#if __APPLE__ - result = ::fcntl(fd, F_FULLFSYNC); -#else - result = ::fsync(fd); -#endif - if (result == -1) - throw SysError("fsync file descriptor %1%", fd); - } -} - - -AutoCloseFD::operator bool() const -{ - return fd != -1; -} - - -int AutoCloseFD::release() -{ - int oldFD = fd; - fd = -1; - return oldFD; -} - - -void Pipe::create() -{ - int fds[2]; -#if HAVE_PIPE2 - if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); -#else - if (pipe(fds) != 0) throw SysError("creating pipe"); - closeOnExec(fds[0]); - closeOnExec(fds[1]); -#endif - readSide = fds[0]; - writeSide = fds[1]; -} - - -void Pipe::close() -{ - readSide.close(); - writeSide.close(); -} - - -////////////////////////////////////////////////////////////////////// - - -Pid::Pid() -{ -} - - -Pid::Pid(pid_t pid) - : pid(pid) -{ -} - - -Pid::~Pid() -{ - if (pid != -1) kill(); -} - - -void Pid::operator =(pid_t pid) -{ - if (this->pid != -1 && this->pid != pid) kill(); - this->pid = pid; - killSignal = SIGKILL; // reset signal to default -} - - -Pid::operator pid_t() -{ - return pid; -} - - -int Pid::kill() -{ - assert(pid != -1); - - debug("killing process %1%", pid); - - /* Send the requested signal to the child. If it has its own - process group, send the signal to every process in the child - process group (which hopefully includes *all* its children). */ - if (::kill(separatePG ? -pid : pid, killSignal) != 0) { - /* On BSDs, killing a process group will return EPERM if all - processes in the group are zombies (or something like - that). So try to detect and ignore that situation. */ -#if __FreeBSD__ || __APPLE__ - if (errno != EPERM || ::kill(pid, 0) != 0) -#endif - logError(SysError("killing process %d", pid).info()); - } - - return wait(); -} - - -int Pid::wait() -{ - assert(pid != -1); - while (1) { - int status; - int res = waitpid(pid, &status, 0); - if (res == pid) { - pid = -1; - return status; - } - if (errno != EINTR) - throw SysError("cannot get exit status of PID %d", pid); - checkInterrupt(); - } -} - - -void Pid::setSeparatePG(bool separatePG) -{ - this->separatePG = separatePG; -} - - -void Pid::setKillSignal(int signal) -{ - this->killSignal = signal; -} - - -pid_t Pid::release() -{ - pid_t p = pid; - pid = -1; - return p; -} - - -void killUser(uid_t uid) -{ - debug("killing all processes running under uid '%1%'", uid); - - assert(uid != 0); /* just to be safe... */ - - /* The system call kill(-1, sig) sends the signal `sig' to all - users to which the current process can send signals. So we - fork a process, switch to uid, and send a mass kill. */ - - Pid pid = startProcess([&]() { - - if (setuid(uid) == -1) - throw SysError("setting uid"); - - while (true) { -#ifdef __APPLE__ - /* OSX's kill syscall takes a third parameter that, among - other things, determines if kill(-1, signo) affects the - calling process. In the OSX libc, it's set to true, - which means "follow POSIX", which we don't want here - */ - if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; -#else - if (kill(-1, SIGKILL) == 0) break; -#endif - if (errno == ESRCH || errno == EPERM) break; /* no more processes */ - if (errno != EINTR) - throw SysError("cannot kill processes for uid '%1%'", uid); - } - - _exit(0); - }); - - int status = pid.wait(); - if (status != 0) - throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status)); - - /* !!! We should really do some check to make sure that there are - no processes left running under `uid', but there is no portable - way to do so (I think). The most reliable way may be `ps -eo - uid | grep -q $uid'. */ -} - - -////////////////////////////////////////////////////////////////////// - - -/* Wrapper around vfork to prevent the child process from clobbering - the caller's stack frame in the parent. */ -static pid_t doFork(bool allowVfork, std::function fun) __attribute__((noinline)); -static pid_t doFork(bool allowVfork, std::function fun) -{ -#ifdef __linux__ - pid_t pid = allowVfork ? vfork() : fork(); -#else - pid_t pid = fork(); -#endif - if (pid != 0) return pid; - fun(); - abort(); -} - - -#if __linux__ -static int childEntry(void * arg) -{ - auto main = (std::function *) arg; - (*main)(); - return 1; -} -#endif - - -pid_t startProcess(std::function fun, const ProcessOptions & options) -{ - std::function wrapper = [&]() { - if (!options.allowVfork) - logger = makeSimpleLogger(); - try { -#if __linux__ - if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) - throw SysError("setting death signal"); -#endif - fun(); - } catch (std::exception & e) { - try { - std::cerr << options.errorPrefix << e.what() << "\n"; - } catch (...) { } - } catch (...) { } - if (options.runExitHandlers) - exit(1); - else - _exit(1); - }; - - pid_t pid = -1; - - if (options.cloneFlags) { - #ifdef __linux__ - // Not supported, since then we don't know when to free the stack. - assert(!(options.cloneFlags & CLONE_VM)); - - size_t stackSize = 1 * 1024 * 1024; - auto stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (stack == MAP_FAILED) throw SysError("allocating stack"); - - Finally freeStack([&]() { munmap(stack, stackSize); }); +#include +#include - pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper); - #else - throw Error("clone flags are only supported on Linux"); - #endif - } else - pid = doFork(options.allowVfork, wrapper); - if (pid == -1) throw SysError("unable to fork"); +namespace nix { - return pid; +void initLibUtil() { + // Check that exception handling works. Exception handling has been observed + // not to work on darwin when the linker flags aren't quite right. + // In this case we don't want to expose the user to some unrelated uncaught + // exception, but rather tell them exactly that exception handling is + // broken. + // When exception handling fails, the message tends to be printed by the + // C++ runtime, followed by an abort. + // For example on macOS we might see an error such as + // libc++abi: terminating with uncaught exception of type nix::SysError: error: C++ exception handling is broken. This would appear to be a problem with the way Nix was compiled and/or linked and/or loaded. + bool caught = false; + try { + throwExceptionSelfCheck(); + } catch (const nix::Error & _e) { + caught = true; + } + // This is not actually the main point of this check, but let's make sure anyway: + assert(caught); } +////////////////////////////////////////////////////////////////////// std::vector stringsToCharPtrs(const Strings & ss) { @@ -1168,211 +40,6 @@ std::vector stringsToCharPtrs(const Strings & ss) return res; } -std::string runProgram(Path program, bool searchPath, const Strings & args, - const std::optional & input, bool isInteractive) -{ - auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive}); - - if (!statusOk(res.first)) - throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); - - return res.second; -} - -// Output = error code + "standard out" output stream -std::pair runProgram(RunOptions && options) -{ - StringSink sink; - options.standardOut = &sink; - - int status = 0; - - try { - runProgram2(options); - } catch (ExecError & e) { - status = e.status; - } - - return {status, std::move(sink.s)}; -} - -void runProgram2(const RunOptions & options) -{ - checkInterrupt(); - - assert(!(options.standardIn && options.input)); - - std::unique_ptr source_; - Source * source = options.standardIn; - - if (options.input) { - source_ = std::make_unique(*options.input); - source = source_.get(); - } - - /* Create a pipe. */ - Pipe out, in; - if (options.standardOut) out.create(); - if (source) in.create(); - - ProcessOptions processOptions; - // vfork implies that the environment of the main process and the fork will - // be shared (technically this is undefined, but in practice that's the - // case), so we can't use it if we alter the environment - processOptions.allowVfork = !options.environment; - - std::optional>> resumeLoggerDefer; - if (options.isInteractive) { - logger->pause(); - resumeLoggerDefer.emplace( - []() { - logger->resume(); - } - ); - } - - /* Fork. */ - Pid pid = startProcess([&]() { - if (options.environment) - replaceEnv(*options.environment); - if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - if (options.mergeStderrToStdout) - if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) - throw SysError("cannot dup stdout into stderr"); - if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) - throw SysError("dupping stdin"); - - if (options.chdir && chdir((*options.chdir).c_str()) == -1) - throw SysError("chdir failed"); - if (options.gid && setgid(*options.gid) == -1) - throw SysError("setgid failed"); - /* Drop all other groups if we're setgid. */ - if (options.gid && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); - if (options.uid && setuid(*options.uid) == -1) - throw SysError("setuid failed"); - - Strings args_(options.args); - args_.push_front(options.program); - - restoreProcessContext(); - - if (options.searchPath) - execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); - // This allows you to refer to a program with a pathname relative - // to the PATH variable. - else - execv(options.program.c_str(), stringsToCharPtrs(args_).data()); - - throw SysError("executing '%1%'", options.program); - }, processOptions); - - out.writeSide.close(); - - std::thread writerThread; - - std::promise promise; - - Finally doJoin([&]() { - if (writerThread.joinable()) - writerThread.join(); - }); - - - if (source) { - in.readSide.close(); - writerThread = std::thread([&]() { - try { - std::vector buf(8 * 1024); - while (true) { - size_t n; - try { - n = source->read(buf.data(), buf.size()); - } catch (EndOfFile &) { - break; - } - writeFull(in.writeSide.get(), {buf.data(), n}); - } - promise.set_value(); - } catch (...) { - promise.set_exception(std::current_exception()); - } - in.writeSide.close(); - }); - } - - if (options.standardOut) - drainFD(out.readSide.get(), *options.standardOut); - - /* Wait for the child to finish. */ - int status = pid.wait(); - - /* Wait for the writer thread to finish. */ - if (source) promise.get_future().get(); - - if (status) - throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); -} - - -void closeMostFDs(const std::set & exceptions) -{ -#if __linux__ - try { - for (auto & s : readDirectory("/proc/self/fd")) { - auto fd = std::stoi(s.name); - if (!exceptions.count(fd)) { - debug("closing leaked FD %d", fd); - close(fd); - } - } - return; - } catch (SysError &) { - } -#endif - - int maxFD = 0; - maxFD = sysconf(_SC_OPEN_MAX); - for (int fd = 0; fd < maxFD; ++fd) - if (!exceptions.count(fd)) - close(fd); /* ignore result */ -} - - -void closeOnExec(int fd) -{ - int prev; - if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || - fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) - throw SysError("setting close-on-exec flag"); -} - - -////////////////////////////////////////////////////////////////////// - - -std::atomic _isInterrupted = false; - -static thread_local bool interruptThrown = false; -thread_local std::function interruptCheck; - -void setInterruptThrown() -{ - interruptThrown = true; -} - -void _interrupted() -{ - /* Block user interrupts while an exception is being handled. - Throwing an exception while another exception is being handled - kills the program! */ - if (!interruptThrown && !std::uncaught_exceptions()) { - interruptThrown = true; - throw Interrupted("interrupted by the user"); - } -} - ////////////////////////////////////////////////////////////////////// @@ -1438,32 +105,6 @@ std::string rewriteStrings(std::string s, const StringMap & rewrites) } -std::string statusToString(int status) -{ - if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { - if (WIFEXITED(status)) - return fmt("failed with exit code %1%", WEXITSTATUS(status)); - else if (WIFSIGNALED(status)) { - int sig = WTERMSIG(status); -#if HAVE_STRSIGNAL - const char * description = strsignal(sig); - return fmt("failed due to signal %1% (%2%)", sig, description); -#else - return fmt("failed due to signal %1%", sig); -#endif - } - else - return "died abnormally"; - } else return "succeeded"; -} - - -bool statusOk(int status) -{ - return WIFEXITED(status) && WEXITSTATUS(status) == 0; -} - - bool hasPrefix(std::string_view s, std::string_view prefix) { return s.compare(0, prefix.size(), prefix) == 0; @@ -1511,82 +152,6 @@ void ignoreException(Verbosity lvl) } catch (...) { } } -bool shouldANSI() -{ - return isatty(STDERR_FILENO) - && getEnv("TERM").value_or("dumb") != "dumb" - && !getEnv("NO_COLOR").has_value(); -} - -std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) -{ - std::string t, e; - size_t w = 0; - auto i = s.begin(); - - while (w < (size_t) width && i != s.end()) { - - if (*i == '\e') { - std::string e; - e += *i++; - char last = 0; - - if (i != s.end() && *i == '[') { - e += *i++; - // eat parameter bytes - while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; - // eat intermediate bytes - while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; - // eat final byte - if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; - } else { - if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; - } - - if (!filterAll && last == 'm') - t += e; - } - - else if (*i == '\t') { - i++; t += ' '; w++; - while (w < (size_t) width && w % 8) { - t += ' '; w++; - } - } - - else if (*i == '\r' || *i == '\a') - // do nothing for now - i++; - - else { - w++; - // Copy one UTF-8 character. - if ((*i & 0xe0) == 0xc0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } else if ((*i & 0xf0) == 0xe0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } - } else if ((*i & 0xf8) == 0xf0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } - } - } else - t += *i++; - } - } - - return t; -} - constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; @@ -1703,386 +268,9 @@ std::pair getLine(std::string_view s) } -////////////////////////////////////////////////////////////////////// - -static Sync> windowSize{{0, 0}}; - - -static void updateWindowSize() -{ - struct winsize ws; - if (ioctl(2, TIOCGWINSZ, &ws) == 0) { - auto windowSize_(windowSize.lock()); - windowSize_->first = ws.ws_row; - windowSize_->second = ws.ws_col; - } -} - - -std::pair getWindowSize() -{ - return *windowSize.lock(); -} - - -/* We keep track of interrupt callbacks using integer tokens, so we can iterate - safely without having to lock the data structure while executing arbitrary - functions. - */ -struct InterruptCallbacks { - typedef int64_t Token; - - /* We use unique tokens so that we can't accidentally delete the wrong - handler because of an erroneous double delete. */ - Token nextToken = 0; - - /* Used as a list, see InterruptCallbacks comment. */ - std::map> callbacks; -}; - -static Sync _interruptCallbacks; - -static void signalHandlerThread(sigset_t set) -{ - while (true) { - int signal = 0; - sigwait(&set, &signal); - - if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP) - triggerInterrupt(); - - else if (signal == SIGWINCH) { - updateWindowSize(); - } - } -} - -void triggerInterrupt() -{ - _isInterrupted = true; - - { - InterruptCallbacks::Token i = 0; - while (true) { - std::function callback; - { - auto interruptCallbacks(_interruptCallbacks.lock()); - auto lb = interruptCallbacks->callbacks.lower_bound(i); - if (lb == interruptCallbacks->callbacks.end()) - break; - - callback = lb->second; - i = lb->first + 1; - } - - try { - callback(); - } catch (...) { - ignoreException(); - } - } - } -} - -static sigset_t savedSignalMask; -static bool savedSignalMaskIsSet = false; - -void setChildSignalMask(sigset_t * sigs) -{ - assert(sigs); // C style function, but think of sigs as a reference - -#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE - sigemptyset(&savedSignalMask); - // There's no "assign" or "copy" function, so we rely on (math) idempotence - // of the or operator: a or a = a. - sigorset(&savedSignalMask, sigs, sigs); -#else - // Without sigorset, our best bet is to assume that sigset_t is a type that - // can be assigned directly, such as is the case for a sigset_t defined as - // an integer type. - savedSignalMask = *sigs; -#endif - - savedSignalMaskIsSet = true; -} - -void saveSignalMask() { - if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) - throw SysError("querying signal mask"); - - savedSignalMaskIsSet = true; -} - -void startSignalHandlerThread() -{ - updateWindowSize(); - - saveSignalMask(); - - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGINT); - sigaddset(&set, SIGTERM); - sigaddset(&set, SIGHUP); - sigaddset(&set, SIGPIPE); - sigaddset(&set, SIGWINCH); - if (pthread_sigmask(SIG_BLOCK, &set, nullptr)) - throw SysError("blocking signals"); - - std::thread(signalHandlerThread, set).detach(); -} - -static void restoreSignals() -{ - // If startSignalHandlerThread wasn't called, that means we're not running - // in a proper libmain process, but a process that presumably manages its - // own signal handlers. Such a process should call either - // - initNix(), to be a proper libmain process - // - startSignalHandlerThread(), to resemble libmain regarding signal - // handling only - // - saveSignalMask(), for processes that define their own signal handling - // thread - // TODO: Warn about this? Have a default signal mask? The latter depends on - // whether we should generally inherit signal masks from the caller. - // I don't know what the larger unix ecosystem expects from us here. - if (!savedSignalMaskIsSet) - return; - - if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) - throw SysError("restoring signals"); -} - -#if __linux__ -rlim_t savedStackSize = 0; -#endif - -void setStackSize(size_t stackSize) -{ - #if __linux__ - struct rlimit limit; - if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { - savedStackSize = limit.rlim_cur; - limit.rlim_cur = stackSize; - setrlimit(RLIMIT_STACK, &limit); - } - #endif -} - -#if __linux__ -static AutoCloseFD fdSavedMountNamespace; -static AutoCloseFD fdSavedRoot; -#endif - -void saveMountNamespace() -{ -#if __linux__ - static std::once_flag done; - std::call_once(done, []() { - fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); - if (!fdSavedMountNamespace) - throw SysError("saving parent mount namespace"); - - fdSavedRoot = open("/proc/self/root", O_RDONLY); - }); -#endif -} - -void restoreMountNamespace() -{ -#if __linux__ - try { - auto savedCwd = absPath("."); - - if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) - throw SysError("restoring parent mount namespace"); - - if (fdSavedRoot) { - if (fchdir(fdSavedRoot.get())) - throw SysError("chdir into saved root"); - if (chroot(".")) - throw SysError("chroot into saved root"); - } - - if (chdir(savedCwd.c_str()) == -1) - throw SysError("restoring cwd"); - } catch (Error & e) { - debug(e.msg()); - } -#endif -} - -void unshareFilesystem() -{ -#ifdef __linux__ - if (unshare(CLONE_FS) != 0 && errno != EPERM) - throw SysError("unsharing filesystem state in download thread"); -#endif -} - -void restoreProcessContext(bool restoreMounts) -{ - restoreSignals(); - if (restoreMounts) { - restoreMountNamespace(); - } - - #if __linux__ - if (savedStackSize) { - struct rlimit limit; - if (getrlimit(RLIMIT_STACK, &limit) == 0) { - limit.rlim_cur = savedStackSize; - setrlimit(RLIMIT_STACK, &limit); - } - } - #endif -} - -/* RAII helper to automatically deregister a callback. */ -struct InterruptCallbackImpl : InterruptCallback -{ - InterruptCallbacks::Token token; - ~InterruptCallbackImpl() override - { - auto interruptCallbacks(_interruptCallbacks.lock()); - interruptCallbacks->callbacks.erase(token); - } -}; - -std::unique_ptr createInterruptCallback(std::function callback) -{ - auto interruptCallbacks(_interruptCallbacks.lock()); - auto token = interruptCallbacks->nextToken++; - interruptCallbacks->callbacks.emplace(token, callback); - - auto res = std::make_unique(); - res->token = token; - - return std::unique_ptr(res.release()); -} - - -AutoCloseFD createUnixDomainSocket() -{ - AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM - #ifdef SOCK_CLOEXEC - | SOCK_CLOEXEC - #endif - , 0); - if (!fdSocket) - throw SysError("cannot create Unix domain socket"); - closeOnExec(fdSocket.get()); - return fdSocket; -} - - -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) -{ - auto fdSocket = nix::createUnixDomainSocket(); - - bind(fdSocket.get(), path); - - if (chmod(path.c_str(), mode) == -1) - throw SysError("changing permissions on '%1%'", path); - - if (listen(fdSocket.get(), 100) == -1) - throw SysError("cannot listen on socket '%1%'", path); - - return fdSocket; -} - - -void bind(int fd, const std::string & path) -{ - unlink(path.c_str()); - - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - - if (path.size() + 1 >= sizeof(addr.sun_path)) { - Pid pid = startProcess([&]() { - Path dir = dirOf(path); - if (chdir(dir.c_str()) == -1) - throw SysError("chdir to '%s' failed", dir); - std::string base(baseNameOf(path)); - if (base.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket path '%s' is too long", base); - memcpy(addr.sun_path, base.c_str(), base.size() + 1); - if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot bind to socket '%s'", path); - _exit(0); - }); - int status = pid.wait(); - if (status != 0) - throw Error("cannot bind to socket '%s'", path); - } else { - memcpy(addr.sun_path, path.c_str(), path.size() + 1); - if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot bind to socket '%s'", path); - } -} - - -void connect(int fd, const std::string & path) -{ - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - - if (path.size() + 1 >= sizeof(addr.sun_path)) { - Pid pid = startProcess([&]() { - Path dir = dirOf(path); - if (chdir(dir.c_str()) == -1) - throw SysError("chdir to '%s' failed", dir); - std::string base(baseNameOf(path)); - if (base.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket path '%s' is too long", base); - memcpy(addr.sun_path, base.c_str(), base.size() + 1); - if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot connect to socket at '%s'", path); - _exit(0); - }); - int status = pid.wait(); - if (status != 0) - throw Error("cannot connect to socket at '%s'", path); - } else { - memcpy(addr.sun_path, path.c_str(), path.size() + 1); - if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot connect to socket at '%s'", path); - } -} - - std::string showBytes(uint64_t bytes) { return fmt("%.2f MiB", bytes / (1024.0 * 1024.0)); } - -// FIXME: move to libstore/build -void commonChildInit() -{ - logger = makeSimpleLogger(); - - const static std::string pathNullDevice = "/dev/null"; - restoreProcessContext(false); - - /* Put the child in a separate session (and thus a separate - process group) so that it has no controlling terminal (meaning - that e.g. ssh cannot open /dev/tty) and it doesn't receive - terminal signals. */ - if (setsid() == -1) - throw SysError("creating a new session"); - - /* Dup stderr to stdout. */ - if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) - throw SysError("cannot dup stderr into stdout"); - - /* Reroute stdin to /dev/null. */ - int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); - if (fdDevNull == -1) - throw SysError("cannot open '%1%'", pathNullDevice); - if (dup2(fdDevNull, STDIN_FILENO) == -1) - throw SysError("cannot dup null device into stdin"); - close(fdDevNull); -} - } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index b302d6f4544..27faa4d6d22 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -4,491 +4,18 @@ #include "types.hh" #include "error.hh" #include "logging.hh" -#include "ansicolor.hh" - -#include -#include -#include -#include -#include #include -#include #include #include #include #include -#ifndef HAVE_STRUCT_DIRENT_D_TYPE -#define DT_UNKNOWN 0 -#define DT_REG 1 -#define DT_LNK 2 -#define DT_DIR 3 -#endif - namespace nix { -struct Sink; -struct Source; - void initLibUtil(); -/** - * The system for which Nix is compiled. - */ -extern const std::string nativeSystem; - - -/** - * @return an environment variable. - */ -std::optional getEnv(const std::string & key); - -/** - * @return a non empty environment variable. Returns nullopt if the env - * variable is set to "" - */ -std::optional getEnvNonEmpty(const std::string & key); - -/** - * Get the entire environment. - */ -std::map getEnv(); - -/** - * Clear the environment. - */ -void clearEnv(); - -/** - * @return An absolutized path, resolving paths relative to the - * specified directory, or the current directory otherwise. The path - * is also canonicalised. - */ -Path absPath(Path path, - std::optional dir = {}, - bool resolveSymlinks = false); - -/** - * Canonicalise a path by removing all `.` or `..` components and - * double or trailing slashes. Optionally resolves all symlink - * components such that each component of the resulting path is *not* - * a symbolic link. - */ -Path canonPath(PathView path, bool resolveSymlinks = false); - -/** - * @return The directory part of the given canonical path, i.e., - * everything before the final `/`. If the path is the root or an - * immediate child thereof (e.g., `/foo`), this means `/` - * is returned. - */ -Path dirOf(const PathView path); - -/** - * @return the base name of the given canonical path, i.e., everything - * following the final `/` (trailing slashes are removed). - */ -std::string_view baseNameOf(std::string_view path); - -/** - * Perform tilde expansion on a path. - */ -std::string expandTilde(std::string_view path); - -/** - * Check whether 'path' is a descendant of 'dir'. Both paths must be - * canonicalized. - */ -bool isInDir(std::string_view path, std::string_view dir); - -/** - * Check whether 'path' is equal to 'dir' or a descendant of - * 'dir'. Both paths must be canonicalized. - */ -bool isDirOrInDir(std::string_view path, std::string_view dir); - -/** - * Get status of `path`. - */ -struct stat stat(const Path & path); -struct stat lstat(const Path & path); - -/** - * @return true iff the given path exists. - */ -bool pathExists(const Path & path); - -/** - * A version of pathExists that returns false on a permission error. - * Useful for inferring default paths across directories that might not - * be readable. - * @return true iff the given path can be accessed and exists - */ -bool pathAccessible(const Path & path); - -/** - * Read the contents (target) of a symbolic link. The result is not - * in any way canonicalised. - */ -Path readLink(const Path & path); - -bool isLink(const Path & path); - -/** - * Read the contents of a directory. The entries `.` and `..` are - * removed. - */ -struct DirEntry -{ - std::string name; - ino_t ino; - /** - * one of DT_* - */ - unsigned char type; - DirEntry(std::string name, ino_t ino, unsigned char type) - : name(std::move(name)), ino(ino), type(type) { } -}; - -typedef std::vector DirEntries; - -DirEntries readDirectory(const Path & path); - -unsigned char getFileType(const Path & path); - -/** - * Read the contents of a file into a string. - */ -std::string readFile(int fd); -std::string readFile(const Path & path); -void readFile(const Path & path, Sink & sink); - -/** - * Write a string to a file. - */ -void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); - -void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); - -/** - * Flush a file's parent directory to disk - */ -void syncParent(const Path & path); - -/** - * Read a line from a file descriptor. - */ -std::string readLine(int fd); - -/** - * Write a line to a file descriptor. - */ -void writeLine(int fd, std::string s); - -/** - * Delete a path; i.e., in the case of a directory, it is deleted - * recursively. It's not an error if the path does not exist. The - * second variant returns the number of bytes and blocks freed. - */ -void deletePath(const Path & path); - -void deletePath(const Path & path, uint64_t & bytesFreed); - -std::string getUserName(); - -/** - * @return the given user's home directory from /etc/passwd. - */ -Path getHomeOf(uid_t userId); - -/** - * @return $HOME or the user's home directory from /etc/passwd. - */ -Path getHome(); - -/** - * @return $XDG_CACHE_HOME or $HOME/.cache. - */ -Path getCacheDir(); - -/** - * @return $XDG_CONFIG_HOME or $HOME/.config. - */ -Path getConfigDir(); - -/** - * @return the directories to search for user configuration files - */ -std::vector getConfigDirs(); - -/** - * @return $XDG_DATA_HOME or $HOME/.local/share. - */ -Path getDataDir(); - -/** - * @return the path of the current executable. - */ -std::optional getSelfExe(); - -/** - * @return $XDG_STATE_HOME or $HOME/.local/state. - */ -Path getStateDir(); - -/** - * Create the Nix state directory and return the path to it. - */ -Path createNixStateDir(); - -/** - * Create a directory and all its parents, if necessary. Returns the - * list of created directories, in order of creation. - */ -Paths createDirs(const Path & path); -inline Paths createDirs(PathView path) -{ - return createDirs(Path(path)); -} - -/** - * Create a symlink. - */ -void createSymlink(const Path & target, const Path & link); - -/** - * Atomically create or replace a symlink. - */ -void replaceSymlink(const Path & target, const Path & link); - -void renameFile(const Path & src, const Path & dst); - -/** - * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` - * are on a different filesystem. - * - * Beware that this might not be atomic because of the copy that happens behind - * the scenes - */ -void moveFile(const Path & src, const Path & dst); - - -/** - * Wrappers arount read()/write() that read/write exactly the - * requested number of bytes. - */ -void readFull(int fd, char * buf, size_t count); -void writeFull(int fd, std::string_view s, bool allowInterrupts = true); - -MakeError(EndOfFile, Error); - - -/** - * Read a file descriptor until EOF occurs. - */ -std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); - -void drainFD(int fd, Sink & sink, bool block = true); - -/** - * If cgroups are active, attempt to calculate the number of CPUs available. - * If cgroups are unavailable or if cpu.max is set to "max", return 0. - */ -unsigned int getMaxCPU(); - -/** - * Automatic cleanup of resources. - */ - - -class AutoDelete -{ - Path path; - bool del; - bool recursive; -public: - AutoDelete(); - AutoDelete(const Path & p, bool recursive = true); - ~AutoDelete(); - void cancel(); - void reset(const Path & p, bool recursive = true); - operator Path() const { return path; } - operator PathView() const { return path; } -}; - - -class AutoCloseFD -{ - int fd; -public: - AutoCloseFD(); - AutoCloseFD(int fd); - AutoCloseFD(const AutoCloseFD & fd) = delete; - AutoCloseFD(AutoCloseFD&& fd); - ~AutoCloseFD(); - AutoCloseFD& operator =(const AutoCloseFD & fd) = delete; - AutoCloseFD& operator =(AutoCloseFD&& fd); - int get() const; - explicit operator bool() const; - int release(); - void close(); - void fsync(); -}; - - -/** - * Create a temporary directory. - */ -Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", - bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); - -/** - * Create a temporary file, returning a file handle and its path. - */ -std::pair createTempFile(const Path & prefix = "nix"); - - -class Pipe -{ -public: - AutoCloseFD readSide, writeSide; - void create(); - void close(); -}; - - -struct DIRDeleter -{ - void operator()(DIR * dir) const { - closedir(dir); - } -}; - -typedef std::unique_ptr AutoCloseDir; - - -class Pid -{ - pid_t pid = -1; - bool separatePG = false; - int killSignal = SIGKILL; -public: - Pid(); - Pid(pid_t pid); - ~Pid(); - void operator =(pid_t pid); - operator pid_t(); - int kill(); - int wait(); - - void setSeparatePG(bool separatePG); - void setKillSignal(int signal); - pid_t release(); -}; - - -/** - * Kill all processes running under the specified uid by sending them - * a SIGKILL. - */ -void killUser(uid_t uid); - - -/** - * Fork a process that runs the given function, and return the child - * pid to the caller. - */ -struct ProcessOptions -{ - std::string errorPrefix = ""; - bool dieWithParent = true; - bool runExitHandlers = false; - bool allowVfork = false; - /** - * use clone() with the specified flags (Linux only) - */ - int cloneFlags = 0; -}; - -pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); - - -/** - * Run a program and return its stdout in a string (i.e., like the - * shell backtick operator). - */ -std::string runProgram(Path program, bool searchPath = false, - const Strings & args = Strings(), - const std::optional & input = {}, bool isInteractive = false); - -struct RunOptions -{ - Path program; - bool searchPath = true; - Strings args; - std::optional uid; - std::optional gid; - std::optional chdir; - std::optional> environment; - std::optional input; - Source * standardIn = nullptr; - Sink * standardOut = nullptr; - bool mergeStderrToStdout = false; - bool isInteractive = false; -}; - -std::pair runProgram(RunOptions && options); - -void runProgram2(const RunOptions & options); - - -/** - * Change the stack size. - */ -void setStackSize(size_t stackSize); - - -/** - * Restore the original inherited Unix process context (such as signal - * masks, stack size). - - * See startSignalHandlerThread(), saveSignalMask(). - */ -void restoreProcessContext(bool restoreMounts = true); - -/** - * Save the current mount namespace. Ignored if called more than - * once. - */ -void saveMountNamespace(); - -/** - * Restore the mount namespace saved by saveMountNamespace(). Ignored - * if saveMountNamespace() was never called. - */ -void restoreMountNamespace(); - -/** - * Cause this thread to not share any FS attributes with the main - * thread, because this causes setns() in restoreMountNamespace() to - * fail. - */ -void unshareFilesystem(); - - -class ExecError : public Error -{ -public: - int status; - - template - ExecError(int status, const Args & ... args) - : Error(args...), status(status) - { } -}; - /** * Convert a list of strings to a null-terminated vector of `char * *`s. The result must not be accessed beyond the lifetime of the @@ -496,36 +23,6 @@ public: */ std::vector stringsToCharPtrs(const Strings & ss); -/** - * Close all file descriptors except those listed in the given set. - * Good practice in child processes. - */ -void closeMostFDs(const std::set & exceptions); - -/** - * Set the close-on-exec flag for the given file descriptor. - */ -void closeOnExec(int fd); - - -/* User interruption. */ - -extern std::atomic _isInterrupted; - -extern thread_local std::function interruptCheck; - -void setInterruptThrown(); - -void _interrupted(); - -void inline checkInterrupt() -{ - if (_isInterrupted || (interruptCheck && interruptCheck())) - _interrupted(); -} - -MakeError(Interrupted, BaseError); - MakeError(FormatError, Error); @@ -601,15 +98,6 @@ std::string replaceStrings( std::string rewriteStrings(std::string s, const StringMap & rewrites); -/** - * Convert the exit status of a child as returned by wait() into an - * error string. - */ -std::string statusToString(int status); - -bool statusOk(int status); - - /** * Parse a string into an integer. */ @@ -701,10 +189,8 @@ std::string toLower(const std::string & s); std::string shellEscape(const std::string_view s); -/** - * Exception handling in destructors: print an error message, then - * ignore the exception. - */ +/* Exception handling in destructors: print an error message, then + ignore the exception. */ void ignoreException(Verbosity lvl = lvlError); @@ -717,23 +203,6 @@ constexpr char treeLast[] = "└───"; constexpr char treeLine[] = "│ "; constexpr char treeNull[] = " "; -/** - * Determine whether ANSI escape sequences are appropriate for the - * present output. - */ -bool shouldANSI(); - -/** - * Truncate a string to 'width' printable characters. If 'filterAll' - * is true, all ANSI escape sequences are filtered out. Otherwise, - * some escape sequences (such as colour setting) are copied but not - * included in the character count. Also, tabs are expanded to - * spaces. - */ -std::string filterANSIEscapes(std::string_view s, - bool filterAll = false, - unsigned int width = std::numeric_limits::max()); - /** * Base64 encoding/decoding. @@ -821,61 +290,6 @@ template class Callback; -/** - * Start a thread that handles various signals. Also block those signals - * on the current thread (and thus any threads created by it). - * Saves the signal mask before changing the mask to block those signals. - * See saveSignalMask(). - */ -void startSignalHandlerThread(); - -/** - * Saves the signal mask, which is the signal mask that nix will restore - * before creating child processes. - * See setChildSignalMask() to set an arbitrary signal mask instead of the - * current mask. - */ -void saveSignalMask(); - -/** - * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't - * necessarily match the current thread's mask. - * See saveSignalMask() to set the saved mask to the current mask. - */ -void setChildSignalMask(sigset_t *sigs); - -struct InterruptCallback -{ - virtual ~InterruptCallback() { }; -}; - -/** - * Register a function that gets called on SIGINT (in a non-signal - * context). - */ -std::unique_ptr createInterruptCallback( - std::function callback); - -void triggerInterrupt(); - -/** - * A RAII class that causes the current thread to receive SIGUSR1 when - * the signal handler thread receives SIGINT. That is, this allows - * SIGINT to be multiplexed to multiple threads. - */ -struct ReceiveInterrupts -{ - pthread_t target; - std::unique_ptr callback; - - ReceiveInterrupts() - : target(pthread_self()) - , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); })) - { } -}; - - - /** * A RAII helper that increments a counter on construction and * decrements it on destruction. @@ -890,45 +304,6 @@ struct MaintainCount }; -/** - * @return the number of rows and columns of the terminal. - */ -std::pair getWindowSize(); - - -/** - * Used in various places. - */ -typedef std::function PathFilter; - -extern PathFilter defaultPathFilter; - -/** - * Common initialisation performed in child processes. - */ -void commonChildInit(); - -/** - * Create a Unix domain socket. - */ -AutoCloseFD createUnixDomainSocket(); - -/** - * Create a Unix domain socket in listen mode. - */ -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); - -/** - * Bind a Unix domain socket to a path. - */ -void bind(int fd, const std::string & path); - -/** - * Connect to a Unix domain socket. - */ -void connect(int fd, const std::string & path); - - /** * A Rust/Python-like enumerate() iterator adapter. * diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index e2189fc669c..75ce12a8c3e 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -9,12 +9,12 @@ #include +#include "current-process.hh" #include "parsed-derivations.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" #include "derivations.hh" -#include "util.hh" #include "shared.hh" #include "path-with-outputs.hh" #include "eval.hh" @@ -34,13 +34,14 @@ extern char * * environ __attribute__((weak)); */ static std::vector shellwords(const std::string & s) { - std::regex whitespace("^(\\s+).*"); + std::regex whitespace("^\\s+"); auto begin = s.cbegin(); std::vector res; std::string cur; enum state { sBegin, - sQuote + sSingleQuote, + sDoubleQuote }; state st = sBegin; auto it = begin; @@ -50,26 +51,39 @@ static std::vector shellwords(const std::string & s) if (regex_search(it, s.cend(), match, whitespace)) { cur.append(begin, it); res.push_back(cur); - cur.clear(); - it = match[1].second; + it = match[0].second; + if (it == s.cend()) return res; begin = it; + cur.clear(); } } switch (*it) { + case '\'': + if (st != sDoubleQuote) { + cur.append(begin, it); + begin = it + 1; + st = st == sBegin ? sSingleQuote : sBegin; + } + break; case '"': - cur.append(begin, it); - begin = it + 1; - st = st == sBegin ? sQuote : sBegin; + if (st != sSingleQuote) { + cur.append(begin, it); + begin = it + 1; + st = st == sBegin ? sDoubleQuote : sBegin; + } break; case '\\': - /* perl shellwords mostly just treats the next char as part of the string with no special processing */ - cur.append(begin, it); - begin = ++it; + if (st != sSingleQuote) { + /* perl shellwords mostly just treats the next char as part of the string with no special processing */ + cur.append(begin, it); + begin = ++it; + } break; } } + if (st != sBegin) throw Error("unterminated quote in shebang line"); cur.append(begin, it); - if (!cur.empty()) res.push_back(cur); + res.push_back(cur); return res; } @@ -128,7 +142,7 @@ static void main_nix_build(int argc, char * * argv) for (auto line : lines) { line = chomp(line); std::smatch match; - if (std::regex_match(line, match, std::regex("^#!\\s*nix-shell (.*)$"))) + if (std::regex_match(line, match, std::regex("^#!\\s*nix-shell\\s+(.*)$"))) for (const auto & word : shellwords(match[1].str())) args.push_back(word); } @@ -344,7 +358,7 @@ static void main_nix_build(int argc, char * * argv) } } - state->printStats(); + state->maybePrintStats(); auto buildPaths = [&](const std::vector & paths) { /* Note: we do this even when !printMissing to efficiently @@ -435,7 +449,7 @@ static void main_nix_build(int argc, char * * argv) } } for (const auto & src : drv.inputSrcs) { - pathsToBuild.push_back(DerivedPath::Opaque{src}); + pathsToBuild.emplace_back(DerivedPath::Opaque{src}); pathsToCopy.insert(src); } diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc old mode 100755 new mode 100644 index 95f4014411f..79db7823696 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -4,9 +4,9 @@ #include "filetransfer.hh" #include "store-api.hh" #include "legacy.hh" -#include "fetchers.hh" #include "eval-settings.hh" // for defexpr -#include "util.hh" +#include "users.hh" +#include "tarball.hh" #include #include diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 70af53b286b..bb3f1bc6add 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -1,3 +1,5 @@ +#include "file-system.hh" +#include "signals.hh" #include "store-api.hh" #include "store-cast.hh" #include "gc-store.hh" diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc old mode 100755 new mode 100644 diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index b112e8cb36a..ab1d8f71317 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1,3 +1,4 @@ +#include "users.hh" #include "attr-path.hh" #include "common-eval-args.hh" #include "derivations.hh" @@ -11,7 +12,6 @@ #include "store-api.hh" #include "local-fs-store.hh" #include "user-env.hh" -#include "util.hh" #include "value-to-json.hh" #include "xml-writer.hh" #include "legacy.hh" @@ -172,7 +172,7 @@ static void loadSourceExpr(EvalState & state, const SourcePath & path, Value & v directory). */ else if (st.type == InputAccessor::tDirectory) { auto attrs = state.buildBindings(maxAttrs); - attrs.alloc("_combineChannels").mkList(0); + state.mkList(attrs.alloc("_combineChannels"), 0); StringSet seen; getAllExprs(state, path, seen, attrs); v.mkAttrs(attrs); @@ -481,12 +481,12 @@ static void printMissing(EvalState & state, DrvInfos & elems) std::vector targets; for (auto & i : elems) if (auto drvPath = i.queryDrvPath()) - targets.push_back(DerivedPath::Built{ + targets.emplace_back(DerivedPath::Built{ .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All { }, }); else - targets.push_back(DerivedPath::Opaque{ + targets.emplace_back(DerivedPath::Opaque{ .path = i.queryOutPath(), }); @@ -1228,7 +1228,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) else { if (v->type() == nString) { attrs2["type"] = "string"; - attrs2["value"] = v->string.s; + attrs2["value"] = v->c_str(); xml.writeEmptyElement("meta", attrs2); } else if (v->type() == nInt) { attrs2["type"] = "int"; @@ -1248,7 +1248,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) for (auto elem : v->listItems()) { if (elem->type() != nString) continue; XMLAttrs attrs3; - attrs3["value"] = elem->string.s; + attrs3["value"] = elem->c_str(); xml.writeEmptyElement("string", attrs3); } } else if (v->type() == nAttrs) { @@ -1260,7 +1260,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) if(a.value->type() != nString) continue; XMLAttrs attrs3; attrs3["type"] = globals.state->symbols[i.name]; - attrs3["value"] = a.value->string.s; + attrs3["value"] = a.value->c_str(); xml.writeEmptyElement("string", attrs3); } } @@ -1531,7 +1531,7 @@ static int main_nix_env(int argc, char * * argv) op(globals, std::move(opFlags), std::move(opArgs)); - globals.state->printStats(); + globals.state->maybePrintStats(); return 0; } diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index d12d70f332e..250224e7da0 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -1,5 +1,4 @@ #include "user-env.hh" -#include "util.hh" #include "derivations.hh" #include "store-api.hh" #include "path-with-outputs.hh" diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 446b27e667e..c67409e89e1 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -6,7 +6,6 @@ #include "attr-path.hh" #include "value-to-xml.hh" #include "value-to-json.hh" -#include "util.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "common-eval-args.hh" @@ -189,7 +188,7 @@ static int main_nix_instantiate(int argc, char * * argv) evalOnly, outputKind, xmlOutputSourceLocation, e); } - state->printStats(); + state->maybePrintStats(); return 0; } diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 577cadceb31..2c530999b55 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -1,5 +1,4 @@ #include "dotgraph.hh" -#include "util.hh" #include "store-api.hh" #include diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index 4395576589c..3e789a2d8b3 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -1,5 +1,4 @@ #include "graphml.hh" -#include "util.hh" #include "store-api.hh" #include "derivations.hh" diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 96c3f7d7e03..123283dfeea 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -9,10 +9,8 @@ #include "local-store.hh" #include "monitor-fd.hh" #include "serve-protocol.hh" +#include "serve-protocol-impl.hh" #include "shared.hh" -#include "util.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" #include "graphml.hh" #include "legacy.hh" #include "path-with-outputs.hh" @@ -407,7 +405,7 @@ static void opQuery(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(j); if (query == qHash) { assert(info->narHash.type == htSHA256); - cout << fmt("%s\n", info->narHash.to_string(Base32, true)); + cout << fmt("%s\n", info->narHash.to_string(HashFormat::Base32, true)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); } @@ -770,8 +768,8 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) if (current.first != info->narHash) { printError("path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(path), - info->narHash.to_string(Base32, true), - current.first.to_string(Base32, true)); + info->narHash.to_string(HashFormat::Base32, true), + current.first.to_string(HashFormat::Base32, true)); status = 1; } } @@ -819,10 +817,16 @@ static void opServe(Strings opFlags, Strings opArgs) if (magic != SERVE_MAGIC_1) throw Error("protocol mismatch"); out << SERVE_MAGIC_2 << SERVE_PROTOCOL_VERSION; out.flush(); - unsigned int clientVersion = readInt(in); + ServeProto::Version clientVersion = readInt(in); - WorkerProto::ReadConn rconn { .from = in }; - WorkerProto::WriteConn wconn { .to = out }; + ServeProto::ReadConn rconn { + .from = in, + .version = clientVersion, + }; + ServeProto::WriteConn wconn { + .to = out, + .version = clientVersion, + }; auto getBuildSettings = [&]() { // FIXME: changing options here doesn't work if we're @@ -867,7 +871,7 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::QueryValidPaths: { bool lock = readInt(in); bool substitute = readInt(in); - auto paths = WorkerProto::Serialise::read(*store, rconn); + auto paths = ServeProto::Serialise::read(*store, rconn); if (lock && writeAllowed) for (auto & path : paths) store->addTempRoot(path); @@ -876,24 +880,24 @@ static void opServe(Strings opFlags, Strings opArgs) store->substitutePaths(paths); } - WorkerProto::write(*store, wconn, store->queryValidPaths(paths)); + ServeProto::write(*store, wconn, store->queryValidPaths(paths)); break; } case ServeProto::Command::QueryPathInfos: { - auto paths = WorkerProto::Serialise::read(*store, rconn); + auto paths = ServeProto::Serialise::read(*store, rconn); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { try { auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - WorkerProto::write(*store, wconn, info->references); + ServeProto::write(*store, wconn, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << info->narHash.to_string(Base32, true) + out << info->narHash.to_string(HashFormat::Base32, true) << renderContentAddress(info->ca) << info->sigs; } catch (InvalidPath &) { @@ -916,7 +920,7 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::ExportPaths: { readInt(in); // obsolete - store->exportPaths(WorkerProto::Serialise::read(*store, rconn), out); + store->exportPaths(ServeProto::Serialise::read(*store, rconn), out); break; } @@ -954,26 +958,16 @@ static void opServe(Strings opFlags, Strings opArgs) MonitorFdHup monitor(in.fd); auto status = store->buildDerivation(drvPath, drv); - out << status.status << status.errorMsg; - - if (GET_PROTOCOL_MINOR(clientVersion) >= 3) - out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime; - if (GET_PROTOCOL_MINOR(clientVersion) >= 6) { - DrvOutputs builtOutputs; - for (auto & [output, realisation] : status.builtOutputs) - builtOutputs.insert_or_assign(realisation.id, realisation); - WorkerProto::write(*store, wconn, builtOutputs); - } - + ServeProto::write(*store, wconn, status); break; } case ServeProto::Command::QueryClosure: { bool includeOutputs = readInt(in); StorePathSet closure; - store->computeFSClosure(WorkerProto::Serialise::read(*store, rconn), + store->computeFSClosure(ServeProto::Serialise::read(*store, rconn), closure, false, includeOutputs); - WorkerProto::write(*store, wconn, closure); + ServeProto::write(*store, wconn, closure); break; } @@ -988,7 +982,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(*store, rconn); + info.references = ServeProto::Serialise::read(*store, rconn); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = ContentAddress::parseOpt(readString(in)); diff --git a/src/nix/add-file.md b/src/nix/add-file.md deleted file mode 100644 index ed237a03500..00000000000 --- a/src/nix/add-file.md +++ /dev/null @@ -1,28 +0,0 @@ -R""( - -# Description - -Copy the regular file *path* to the Nix store, and print the resulting -store path on standard output. - -> **Warning** -> -> The resulting store path is not registered as a garbage -> collector root, so it could be deleted before you have a -> chance to register it. - -# Examples - -Add a regular file to the store: - -```console -# echo foo > bar - -# nix store add-file ./bar -/nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar - -# cat /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar -foo -``` - -)"" diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 39e5cc99dd2..f9d487adab7 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -5,11 +5,22 @@ using namespace nix; +static FileIngestionMethod parseIngestionMethod(std::string_view input) +{ + if (input == "flat") { + return FileIngestionMethod::Flat; + } else if (input == "nar") { + return FileIngestionMethod::Recursive; + } else { + throw UsageError("Unknown hash mode '%s', expect `flat` or `nar`"); + } +} + struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional namePart; - FileIngestionMethod ingestionMethod; + FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive; CmdAddToStore() { @@ -23,6 +34,23 @@ struct CmdAddToStore : MixDryRun, StoreCommand .labels = {"name"}, .handler = {&namePart}, }); + + addFlag({ + .longName = "mode", + .shortName = 'n', + .description = R"( + How to compute the hash of the input. + One of: + + - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. + + - `flat`: Assumes that the input is a single file and directly passes it to the hash function; + )", + .labels = {"hash-mode"}, + .handler = {[this](std::string s) { + this->ingestionMethod = parseIngestionMethod(s); + }}, + }); } void run(ref store) override @@ -62,45 +90,43 @@ struct CmdAddToStore : MixDryRun, StoreCommand } }; -struct CmdAddFile : CmdAddToStore +struct CmdAdd : CmdAddToStore { - CmdAddFile() - { - ingestionMethod = FileIngestionMethod::Flat; - } std::string description() override { - return "add a regular file to the Nix store"; + return "Add a file or directory to the Nix store"; } std::string doc() override { return - #include "add-file.md" + #include "add.md" ; } }; -struct CmdAddPath : CmdAddToStore +struct CmdAddFile : CmdAddToStore { - CmdAddPath() + CmdAddFile() { - ingestionMethod = FileIngestionMethod::Recursive; + ingestionMethod = FileIngestionMethod::Flat; } std::string description() override { - return "add a path to the Nix store"; + return "Deprecated. Use [`nix store add --mode flat`](@docroot@/command-ref/new-cli/nix3-store-add.md) instead."; } +}; - std::string doc() override +struct CmdAddPath : CmdAddToStore +{ + std::string description() override { - return - #include "add-path.md" - ; + return "Deprecated alias to [`nix store add`](@docroot@/command-ref/new-cli/nix3-store-add.md)."; } }; static auto rCmdAddFile = registerCommand2({"store", "add-file"}); static auto rCmdAddPath = registerCommand2({"store", "add-path"}); +static auto rCmdAdd = registerCommand2({"store", "add"}); diff --git a/src/nix/add-path.md b/src/nix/add.md similarity index 94% rename from src/nix/add-path.md rename to src/nix/add.md index 87473611df4..d38cd21d87f 100644 --- a/src/nix/add-path.md +++ b/src/nix/add.md @@ -19,7 +19,7 @@ Add a directory to the store: # mkdir dir # echo foo > dir/bar -# nix store add-path ./dir +# nix store add ./dir /nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir # cat /nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir/bar diff --git a/src/nix/app.cc b/src/nix/app.cc index 34fac9935c9..935ed18ecba 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -20,20 +20,24 @@ StringPairs resolveRewrites( const std::vector & dependencies) { StringPairs res; - if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - for (auto & dep : dependencies) { - if (auto drvDep = std::get_if(&dep.path)) { - for (auto & [ outputName, outputPath ] : drvDep->outputs) { - res.emplace( - DownstreamPlaceholder::fromSingleDerivedPathBuilt( - SingleDerivedPath::Built { - .drvPath = make_ref(drvDep->drvPath->discardOutputPath()), - .output = outputName, - }).render(), - store.printStorePath(outputPath) - ); - } - } + if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + return res; + } + for (auto &dep: dependencies) { + auto drvDep = std::get_if(&dep.path); + if (!drvDep) { + continue; + } + + for (const auto & [ outputName, outputPath ] : drvDep->outputs) { + res.emplace( + DownstreamPlaceholder::fromSingleDerivedPathBuilt( + SingleDerivedPath::Built { + .drvPath = make_ref(drvDep->drvPath->discardOutputPath()), + .output = outputName, + }).render(), + store.printStorePath(outputPath) + ); } } return res; @@ -58,11 +62,11 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) auto type = cursor->getAttr("type")->getString(); - std::string expected = !attrPath.empty() && + std::string expectedType = !attrPath.empty() && (state.symbols[attrPath[0]] == "apps" || state.symbols[attrPath[0]] == "defaultApp") ? "app" : "derivation"; - if (type != expected) - throw Error("attribute '%s' should have type '%s'", cursor->getAttrPathStr(), expected); + if (type != expectedType) + throw Error("attribute '%s' should have type '%s'", cursor->getAttrPathStr(), expectedType); if (type == "app") { auto [program, context] = cursor->getAttr("program")->getStringWithContext(); @@ -91,7 +95,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) }, c.raw)); } - return UnresolvedApp{App { + return UnresolvedApp { App { .context = std::move(context2), .program = program, }}; diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index fbc83b08eab..54cc6a17f36 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -4,7 +4,6 @@ #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "fs-accessor.hh" #include "eval-inline.hh" using namespace nix; @@ -21,8 +20,8 @@ struct CmdBundle : InstallableValueCommand .description = fmt("Use a custom bundler instead of the default (`%s`).", bundler), .labels = {"flake-url"}, .handler = {&bundler}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); }} }); diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 60aa66ce0c6..4df086d4fba 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -1,7 +1,7 @@ #include "command.hh" #include "store-api.hh" -#include "fs-accessor.hh" #include "nar-accessor.hh" +#include "progress-bar.hh" using namespace nix; @@ -9,15 +9,13 @@ struct MixCat : virtual Args { std::string path; - void cat(ref accessor) + void cat(ref accessor) { - auto st = accessor->stat(path); - if (st.type == FSAccessor::Type::tMissing) - throw Error("path '%1%' does not exist", path); - if (st.type != FSAccessor::Type::tRegular) + auto st = accessor->lstat(CanonPath(path)); + if (st.type != SourceAccessor::Type::tRegular) throw Error("path '%1%' is not a regular file", path); - - writeFull(STDOUT_FILENO, accessor->readFile(path)); + stopProgressBar(); + writeFull(STDOUT_FILENO, accessor->readFile(CanonPath(path))); } }; diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index af428018afb..373dedf7cbc 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -1,11 +1,12 @@ ///@file +#include "signals.hh" +#include "unix-domain-socket.hh" #include "command.hh" #include "shared.hh" #include "local-store.hh" #include "remote-store.hh" #include "remote-store-connection.hh" -#include "util.hh" #include "serialise.hh" #include "archive.hh" #include "globals.hh" diff --git a/src/nix/develop.cc b/src/nix/develop.cc index c01ef1a2a8b..38482ed42a4 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -9,8 +9,10 @@ #include "progress-bar.hh" #include "run.hh" +#include #include #include +#include using namespace nix; @@ -51,6 +53,7 @@ struct BuildEnvironment std::map vars; std::map bashFunctions; + std::optional> structuredAttrs; static BuildEnvironment fromJSON(std::string_view in) { @@ -74,6 +77,10 @@ struct BuildEnvironment res.bashFunctions.insert({name, def}); } + if (json.contains("structuredAttrs")) { + res.structuredAttrs = {json["structuredAttrs"][".attrs.json"], json["structuredAttrs"][".attrs.sh"]}; + } + return res; } @@ -102,6 +109,13 @@ struct BuildEnvironment res["bashFunctions"] = bashFunctions; + if (providesStructuredAttrs()) { + auto contents = nlohmann::json::object(); + contents[".attrs.sh"] = getAttrsSH(); + contents[".attrs.json"] = getAttrsJSON(); + res["structuredAttrs"] = std::move(contents); + } + auto json = res.dump(); assert(BuildEnvironment::fromJSON(json) == *this); @@ -109,6 +123,23 @@ struct BuildEnvironment return json; } + bool providesStructuredAttrs() const + { + return structuredAttrs.has_value(); + } + + std::string getAttrsJSON() const + { + assert(providesStructuredAttrs()); + return structuredAttrs->first; + } + + std::string getAttrsSH() const + { + assert(providesStructuredAttrs()); + return structuredAttrs->second; + } + void toBash(std::ostream & out, const std::set & ignoreVars) const { for (auto & [name, value] : vars) { @@ -291,6 +322,7 @@ struct Common : InstallableCommand, MixProfile std::string makeRcScript( ref store, const BuildEnvironment & buildEnvironment, + const Path & tmpDir, const Path & outputsDir = absPath(".") + "/outputs") { // A list of colon-separated environment variables that should be @@ -353,9 +385,48 @@ struct Common : InstallableCommand, MixProfile } } + if (buildEnvironment.providesStructuredAttrs()) { + fixupStructuredAttrs( + "sh", + "NIX_ATTRS_SH_FILE", + buildEnvironment.getAttrsSH(), + rewrites, + buildEnvironment, + tmpDir + ); + fixupStructuredAttrs( + "json", + "NIX_ATTRS_JSON_FILE", + buildEnvironment.getAttrsJSON(), + rewrites, + buildEnvironment, + tmpDir + ); + } + return rewriteStrings(script, rewrites); } + /** + * Replace the value of NIX_ATTRS_*_FILE (`/build/.attrs.*`) with a tmp file + * that's accessible from the interactive shell session. + */ + void fixupStructuredAttrs( + const std::string & ext, + const std::string & envVar, + const std::string & content, + StringMap & rewrites, + const BuildEnvironment & buildEnvironment, + const Path & tmpDir) + { + auto targetFilePath = tmpDir + "/.attrs." + ext; + writeFile(targetFilePath, content); + + auto fileInBuilderEnv = buildEnvironment.vars.find(envVar); + assert(fileInBuilderEnv != buildEnvironment.vars.end()); + rewrites.insert({BuildEnvironment::getString(fileInBuilderEnv->second), targetFilePath}); + } + Strings getDefaultFlakeAttrPaths() override { Strings paths{ @@ -487,7 +558,9 @@ struct CmdDevelop : Common, MixEnvironment auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); - auto script = makeRcScript(store, buildEnvironment); + AutoDelete tmpDir(createTempDir("", "nix-develop"), true); + + auto script = makeRcScript(store, buildEnvironment, (Path) tmpDir); if (verbosity >= lvlDebug) script += "set -x\n"; @@ -615,10 +688,12 @@ struct CmdPrintDevEnv : Common, MixJSON stopProgressBar(); - logger->writeToStdout( - json - ? buildEnvironment.toJSON() - : makeRcScript(store, buildEnvironment)); + if (json) { + logger->writeToStdout(buildEnvironment.toJSON()); + } else { + AutoDelete tmpDir(createTempDir("", "nix-dev-env"), true); + logger->writeToStdout(makeRcScript(store, buildEnvironment, tmpDir)); + } } }; diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 1aa6831d30f..59f9e3e5d7a 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -6,7 +6,6 @@ #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "util.hh" #include "worker-protocol.hh" using namespace nix; diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index c4edc894b89..0850d4c1cdb 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -61,4 +61,12 @@ struct CmdDumpPath2 : Command } }; -static auto rDumpPath2 = registerCommand2({"nar", "dump-path"}); +struct CmdNarDumpPath : CmdDumpPath2 { + void run() override { + warn("'nix nar dump-path' is a deprecated alias for 'nix nar pack'"); + CmdDumpPath2::run(); + } +}; + +static auto rCmdNarPack = registerCommand2({"nar", "pack"}); +static auto rCmdNarDumpPath = registerCommand2({"nar", "dump-path"}); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 66629fab03a..9cbab230b0f 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -1,3 +1,4 @@ +#include "current-process.hh" #include "command-installable-value.hh" #include "shared.hh" #include "eval.hh" diff --git a/src/nix/eval.cc b/src/nix/eval.cc index d880bef0a7b..b34af34e0eb 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -85,7 +85,7 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption state->forceValue(v, pos); if (v.type() == nString) // FIXME: disallow strings with contexts? - writeFile(path, v.string.s); + writeFile(path, v.string_view()); else if (v.type() == nAttrs) { if (mkdir(path.c_str(), 0777) == -1) throw SysError("creating directory '%s'", path); diff --git a/src/nix/flake-lock.md b/src/nix/flake-lock.md index 2af0ad81e0e..6d10258e338 100644 --- a/src/nix/flake-lock.md +++ b/src/nix/flake-lock.md @@ -2,37 +2,39 @@ R""( # Examples -* Update the `nixpkgs` and `nix` inputs of the flake in the current - directory: +* Create the lock file for the flake in the current directory: ```console - # nix flake lock --update-input nixpkgs --update-input nix - * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' - * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' + # nix flake lock + warning: creating lock file '/home/myself/repos/testflake/flake.lock': + • Added input 'nix': + 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' (2023-06-28) + • Added input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) ``` -# Description - -This command updates the lock file of a flake (`flake.lock`) so that -it contains a lock for every flake input specified in -`flake.nix`. Existing lock file entries are not updated unless -required by a flag such as `--update-input`. +* Add missing inputs to the lock file for a flake in a different directory: -Note that every command that operates on a flake will also update the -lock file if needed, and supports the same flags. Therefore, + ```console + # nix flake lock ~/repos/another + warning: updating lock file '/home/myself/repos/another/flake.lock': + • Added input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + ``` -```console -# nix flake lock --update-input nixpkgs -# nix build -``` + > **Note** + > + > When trying to refer to a flake in a subdirectory, write `./another` + > instead of `another`. + > Otherwise Nix will try to look up the flake in the registry. -is equivalent to: +# Description -```console -# nix build --update-input nixpkgs -``` +This command adds inputs to the lock file of a flake (`flake.lock`) +so that it contains a lock for every flake input specified in +`flake.nix`. Existing lock file entries are not updated. -Thus, this command is only useful if you want to update the lock file -separately from any other action such as building. +If you want to update existing lock entries, use +[`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) )"" diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md index 8c6042d94a3..63df3b12afe 100644 --- a/src/nix/flake-update.md +++ b/src/nix/flake-update.md @@ -2,33 +2,57 @@ R""( # Examples -* Recreate the lock file (i.e. update all inputs) and commit the new - lock file: +* Update all inputs (i.e. recreate the lock file from scratch): ```console - # nix flake update --commit-lock-file - * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' - * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' - … - warning: committed new revision '158bcbd9d6cc08ab859c0810186c1beebc982aad' + # nix flake update + warning: updating lock file '/home/myself/repos/testflake/flake.lock': + • Updated input 'nix': + 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' (2023-06-28) + → 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' (2023-07-11) + • Updated input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05) ``` -# Description +* Update only a single input: + + ```console + # nix flake update nixpkgs + warning: updating lock file '/home/myself/repos/testflake/flake.lock': + • Updated input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05) + ``` + +* Update only a single input of a flake in a different directory: -This command recreates the lock file of a flake (`flake.lock`), thus -updating the lock for every unlocked input (like `nixpkgs`) to its -current version. This is equivalent to passing `--recreate-lock-file` -to any command that operates on a flake. That is, + ```console + # nix flake update nixpkgs --flake ~/repos/another + warning: updating lock file '/home/myself/repos/another/flake.lock': + • Updated input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05) + ``` + + > **Note** + > + > When trying to refer to a flake in a subdirectory, write `./another` + > instead of `another`. + > Otherwise Nix will try to look up the flake in the registry. + +# Description -```console -# nix flake update -# nix build -``` +This command updates the inputs in a lock file (`flake.lock`). +**By default, all inputs are updated**. If the lock file doesn't exist +yet, it will be created. If inputs are not in the lock file yet, they will be added. -is equivalent to: +Unlike other `nix flake` commands, `nix flake update` takes a list of names of inputs +to update as its positional arguments and operates on the flake in the current directory. +You can pass a different flake-url with `--flake` to override that default. -```console -# nix build --recreate-lock-file -``` +The related command [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) +also creates lock files and adds missing inputs, but is safer as it +will never update inputs already in the lock file. )"" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 87dd4da1bec..e0c67fdfa44 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -15,6 +15,7 @@ #include "registry.hh" #include "eval-cache.hh" #include "markdown.hh" +#include "users.hh" #include #include @@ -24,8 +25,10 @@ using namespace nix; using namespace nix::flake; using json = nlohmann::json; +struct CmdFlakeUpdate; class FlakeCommand : virtual Args, public MixFlakeOptions { +protected: std::string flakeUrl = "."; public: @@ -36,8 +39,8 @@ class FlakeCommand : virtual Args, public MixFlakeOptions .label = "flake-url", .optional = true, .handler = {&flakeUrl}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); }} }); } @@ -52,14 +55,19 @@ class FlakeCommand : virtual Args, public MixFlakeOptions return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags); } - std::vector getFlakesForCompletion() override + std::vector getFlakeRefsForCompletion() override { - return {flakeUrl}; + return { + // Like getFlakeRef but with expandTilde calld first + parseFlakeRef(expandTilde(flakeUrl), absPath(".")) + }; } }; struct CmdFlakeUpdate : FlakeCommand { +public: + std::string description() override { return "update flake lock file"; @@ -67,9 +75,37 @@ struct CmdFlakeUpdate : FlakeCommand CmdFlakeUpdate() { + expectedArgs.clear(); + addFlag({ + .longName="flake", + .description="The flake to operate on. Default is the current directory.", + .labels={"flake-url"}, + .handler={&flakeUrl}, + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); + }} + }); + expectArgs({ + .label="inputs", + .optional=true, + .handler={[&](std::string inputToUpdate){ + InputPath inputPath; + try { + inputPath = flake::parseInputPath(inputToUpdate); + } catch (Error & e) { + warn("Invalid flake input '%s'. To update a specific flake, use 'nix flake update --flake %s' instead.", inputToUpdate, inputToUpdate); + throw e; + } + if (lockFlags.inputUpdates.contains(inputPath)) + warn("Input '%s' was specified multiple times. You may have done this by accident."); + lockFlags.inputUpdates.insert(inputPath); + }}, + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); + }} + }); + /* Remove flags that don't make sense. */ - removeFlag("recreate-lock-file"); - removeFlag("update-input"); removeFlag("no-update-lock-file"); removeFlag("no-write-lock-file"); } @@ -84,8 +120,9 @@ struct CmdFlakeUpdate : FlakeCommand void run(nix::ref store) override { settings.tarballTtl = 0; + auto updateAll = lockFlags.inputUpdates.empty(); - lockFlags.recreateLockFile = true; + lockFlags.recreateLockFile = updateAll; lockFlags.writeLockFile = true; lockFlags.applyNixConfig = true; @@ -179,14 +216,14 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs()); if (auto rev = flake.lockedRef.input.getRev()) - j["revision"] = rev->to_string(Base16, false); + j["revision"] = rev->to_string(HashFormat::Base16, false); if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) j["dirtyRevision"] = *dirtyRev; if (auto revCount = flake.lockedRef.input.getRevCount()) j["revCount"] = *revCount; if (auto lastModified = flake.lockedRef.input.getLastModified()) j["lastModified"] = *lastModified; - j["path"] = store->printStorePath(flake.sourceInfo->storePath); + j["path"] = store->printStorePath(flake.storePath); j["locks"] = lockedFlake.lockFile.toJSON(); logger->cout("%s", j.dump()); } else { @@ -202,11 +239,11 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON *flake.description); logger->cout( ANSI_BOLD "Path:" ANSI_NORMAL " %s", - store->printStorePath(flake.sourceInfo->storePath)); + store->printStorePath(flake.storePath)); if (auto rev = flake.lockedRef.input.getRev()) logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", - rev->to_string(Base16, false)); + rev->to_string(HashFormat::Base16, false)); if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", @@ -233,9 +270,13 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON bool last = i + 1 == node.inputs.size(); if (auto lockedNode = std::get_if<0>(&input.second)) { - logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s", + std::string lastModifiedStr = ""; + if (auto lastModified = (*lockedNode)->lockedRef.input.getLastModified()) + lastModifiedStr = fmt(" (%s)", std::put_time(std::gmtime(&*lastModified), "%F %T")); + logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s%s", prefix + (last ? treeLast : treeConn), input.first, - (*lockedNode)->lockedRef); + (*lockedNode)->lockedRef, + lastModifiedStr); bool firstVisit = visited.insert(*lockedNode).second; @@ -758,8 +799,9 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand .description = "The template to use.", .labels = {"template"}, .handler = {&templateUrl}, - .completer = {[&](size_t, std::string_view prefix) { + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { completeFlakeRefWithFragment( + completions, getEvalState(), lockFlags, defaultTemplateAttrPathsPrefixes, @@ -972,7 +1014,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun StorePathSet sources; - sources.insert(flake.flake.sourceInfo->storePath); + sources.insert(flake.flake.storePath); // FIXME: use graph output, handle cycles. std::function traverse; @@ -984,7 +1026,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun auto storePath = dryRun ? (*inputNode)->lockedRef.input.computeStorePath(*store) - : (*inputNode)->lockedRef.input.fetch(store).first.storePath; + : (*inputNode)->lockedRef.input.fetch(store).first; if (json) { auto& jsonObj3 = jsonObj2[inputName]; jsonObj3["path"] = store->printStorePath(storePath); @@ -1001,7 +1043,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun if (json) { nlohmann::json jsonRoot = { - {"path", store->printStorePath(flake.flake.sourceInfo->storePath)}, + {"path", store->printStorePath(flake.flake.storePath)}, {"inputs", traverse(*flake.lockFile.root)}, }; logger->cout("%s", jsonRoot); @@ -1335,19 +1377,21 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON { auto originalRef = getFlakeRef(); auto resolvedRef = originalRef.resolve(store); - auto [tree, lockedRef] = resolvedRef.fetchTree(store); - auto hash = store->queryPathInfo(tree.storePath)->narHash; + auto [storePath, lockedRef] = resolvedRef.fetchTree(store); + auto hash = store->queryPathInfo(storePath)->narHash; if (json) { auto res = nlohmann::json::object(); - res["storePath"] = store->printStorePath(tree.storePath); - res["hash"] = hash.to_string(SRI, true); + res["storePath"] = store->printStorePath(storePath); + res["hash"] = hash.to_string(HashFormat::SRI, true); + res["original"] = fetchers::attrsToJSON(resolvedRef.toAttrs()); + res["locked"] = fetchers::attrsToJSON(lockedRef.toAttrs()); logger->cout(res.dump()); } else { notice("Downloaded '%s' to '%s' (hash '%s').", lockedRef.to_string(), - store->printStorePath(tree.storePath), - hash.to_string(SRI, true)); + store->printStorePath(storePath), + hash.to_string(HashFormat::SRI, true)); } } }; diff --git a/src/nix/flake.md b/src/nix/flake.md index 92f477917fd..d8b5bf435f1 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -67,6 +67,11 @@ inputs.nixpkgs = { }; ``` +Following [RFC 3986](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1), +characters outside of the allowed range (i.e. neither [reserved characters](https://datatracker.ietf.org/doc/html/rfc3986#section-2.2) +nor [unreserved characters](https://datatracker.ietf.org/doc/html/rfc3986#section-2.3)) +must be percent-encoded. + ### Examples Here are some examples of flake references in their URL-like representation: @@ -93,7 +98,15 @@ Here are some examples of flake references in their URL-like representation: ## Path-like syntax -Flakes corresponding to a local path can also be referred to by a direct path reference, either `/absolute/path/to/the/flake` or `./relative/path/to/the/flake` (note that the leading `./` is mandatory for relative paths to avoid any ambiguity). +Flakes corresponding to a local path can also be referred to by a direct +path reference, either `/absolute/path/to/the/flake` or`./relative/path/to/the/flake`. +Note that the leading `./` is mandatory for relative paths. If it is +omitted, the path will be interpreted as [URL-like syntax](#url-like-syntax), +which will cause error messages like this: + +```console +error: cannot find flake 'flake:relative/path/to/the/flake' in the flake registries +``` The semantic of such a path is as follows: @@ -103,10 +116,14 @@ The semantic of such a path is as follows: 2. The filesystem root (/), or 3. A folder on a different mount point. +Contrary to URL-like references, path-like flake references can contain arbitrary unicode characters (except `#` and `?`). + ### Examples * `.`: The flake to which the current directory belongs to. * `/home/alice/src/patchelf`: A flake in some other directory. +* `./../sub directory/with Ûñî©ôδ€`: A flake in another relative directory that + has Unicode characters in its name. ## Flake reference attributes @@ -144,18 +161,39 @@ can occur in *locked* flake references and are available to Nix code: Currently the `type` attribute can be one of the following: -* `path`: arbitrary local directories, or local Git trees. The - required attribute `path` specifies the path of the flake. The URL - form is +* `indirect`: *The default*. Indirection through the flake registry. + These have the form + + ``` + [flake:](/(/rev)?)? + ``` + + These perform a lookup of `` in the flake registry. For + example, `nixpkgs` and `nixpkgs/release-20.09` are indirect flake + references. The specified `rev` and/or `ref` are merged with the + entry in the registry; see [nix registry](./nix3-registry.md) for + details. + + For example, these are valid indirect flake references: + + * `nixpkgs` + * `nixpkgs/nixos-unstable` + * `nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293` + * `nixpkgs/nixos-unstable/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293` + * `sub/dir` (if a flake named `sub` is in the registry) + +* `path`: arbitrary local directories. The required attribute `path` + specifies the path of the flake. The URL form is ``` - [path:](\?(\?)? ``` - where *path* is an absolute path. + where *path* is an absolute path to a directory in the file system + containing a file named `flake.nix`. - *path* must be a directory in the file system containing a file - named `flake.nix`. + If the flake at *path* is not inside a git repository, the `path:` + prefix is implied and can be omitted. *path* generally must be an absolute path. However, on the command line, it can be a relative path (e.g. `.` or `./foo`) which is @@ -164,15 +202,25 @@ Currently the `type` attribute can be one of the following: (e.g. `nixpkgs` is a registry lookup; `./nixpkgs` is a relative path). + For example, these are valid path flake references: + + * `path:/home/user/sub/dir` + * `/home/user/sub/dir` (if `dir/flake.nix` is *not* in a git repository) + * `./sub/dir` (when used on the command line and `dir/flake.nix` is *not* in a git repository) + * `git`: Git repositories. The location of the repository is specified by the attribute `url`. They have the URL form ``` - git(+http|+https|+ssh|+git|+file|):(//)?(\?)? + git(+http|+https|+ssh|+git|+file):(//)?(\?)? ``` + If *path* starts with `/` (or `./` when used as an argument on the + command line) and is a local path to a git repository, the leading + `git:` or `+file` prefixes are implied and can be omitted. + The `ref` attribute defaults to resolving the `HEAD` reference. The `rev` attribute must denote a commit that exists in the branch @@ -188,6 +236,9 @@ Currently the `type` attribute can be one of the following: For example, the following are valid Git flake references: + * `git:/home/user/sub/dir` + * `/home/user/sub/dir` (if `dir/flake.nix` is in a git repository) + * `./sub/dir` (when used on the command line and `dir/flake.nix` is in a git repository) * `git+https://example.org/my/repo` * `git+https://example.org/my/repo?dir=flake1` * `git+ssh://git@github.com/NixOS/nix?ref=v1.2.3` @@ -309,19 +360,6 @@ Currently the `type` attribute can be one of the following: * `sourcehut:~misterio/nix-colors/182b4b8709b8ffe4e9774a4c5d6877bf6bb9a21c` * `sourcehut:~misterio/nix-colors/21c1a380a6915d890d408e9f22203436a35bb2de?host=hg.sr.ht` -* `indirect`: Indirections through the flake registry. These have the - form - - ``` - [flake:](/(/rev)?)? - ``` - - These perform a lookup of `` in the flake registry. For - example, `nixpkgs` and `nixpkgs/release-20.09` are indirect flake - references. The specified `rev` and/or `ref` are merged with the - entry in the registry; see [nix registry](./nix3-registry.md) for - details. - # Flake format As an example, here is a simple `flake.nix` that depends on the diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index a7a8a01b994..832cc2f1134 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -1,5 +1,5 @@ set -e -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then source "$NIX_ATTRS_SH_FILE"; fi export IN_NIX_SHELL=impure export dontAddDisableDepTrack=1 @@ -101,7 +101,21 @@ __dumpEnv() { printf "}" done < <(printf "%s\n" "$__vars") - printf '\n }\n}' + printf '\n }' + + if [ -e "$NIX_ATTRS_SH_FILE" ]; then + printf ',\n "structuredAttrs": {\n ' + __escapeString ".attrs.sh" + printf ': ' + __escapeString "$(<"$NIX_ATTRS_SH_FILE")" + printf ',\n ' + __escapeString ".attrs.json" + printf ': ' + __escapeString "$(<"$NIX_ATTRS_JSON_FILE")" + printf '\n }' + fi + + printf '\n}' } __escapeString() { @@ -117,7 +131,7 @@ __escapeString() { # In case of `__structuredAttrs = true;` the list of outputs is an associative # array with a format like `outname => /nix/store/hash-drvname-outname`, so `__olist` # must contain the array's keys (hence `${!...[@]}`) in this case. -if [ -e .attrs.sh ]; then +if [ -e "$NIX_ATTRS_SH_FILE" ]; then __olist="${!outputs[@]}" else __olist=$outputs diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 9feca934557..d6595dcca05 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -11,7 +11,7 @@ using namespace nix; struct CmdHashBase : Command { FileIngestionMethod mode; - Base base = SRI; + HashFormat hashFormat = HashFormat::SRI; bool truncate = false; HashType ht = htSHA256; std::vector paths; @@ -22,25 +22,25 @@ struct CmdHashBase : Command addFlag({ .longName = "sri", .description = "Print the hash in SRI format.", - .handler = {&base, SRI}, + .handler = {&hashFormat, HashFormat::SRI}, }); addFlag({ .longName = "base64", .description = "Print the hash in base-64 format.", - .handler = {&base, Base64}, + .handler = {&hashFormat, HashFormat::Base64}, }); addFlag({ .longName = "base32", .description = "Print the hash in base-32 (Nix-specific) format.", - .handler = {&base, Base32}, + .handler = {&hashFormat, HashFormat::Base32}, }); addFlag({ .longName = "base16", .description = "Print the hash in base-16 format.", - .handler = {&base, Base16}, + .handler = {&hashFormat, HashFormat::Base16}, }); addFlag(Flag::mkHashTypeFlag("type", &ht)); @@ -94,18 +94,18 @@ struct CmdHashBase : Command Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); - logger->cout(h.to_string(base, base == SRI)); + logger->cout(h.to_string(hashFormat, hashFormat == HashFormat::SRI)); } } }; struct CmdToBase : Command { - Base base; + HashFormat hashFormat; std::optional ht; std::vector args; - CmdToBase(Base base) : base(base) + CmdToBase(HashFormat hashFormat) : hashFormat(hashFormat) { addFlag(Flag::mkHashTypeOptFlag("type", &ht)); expectArgs("strings", &args); @@ -114,16 +114,16 @@ struct CmdToBase : Command std::string description() override { return fmt("convert a hash to %s representation", - base == Base16 ? "base-16" : - base == Base32 ? "base-32" : - base == Base64 ? "base-64" : + hashFormat == HashFormat::Base16 ? "base-16" : + hashFormat == HashFormat::Base32 ? "base-32" : + hashFormat == HashFormat::Base64 ? "base-64" : "SRI"); } void run() override { for (auto s : args) - logger->cout(Hash::parseAny(s, ht).to_string(base, base == SRI)); + logger->cout(Hash::parseAny(s, ht).to_string(hashFormat, hashFormat == HashFormat::SRI)); } }; @@ -133,10 +133,10 @@ struct CmdHash : NixMultiCommand : MultiCommand({ {"file", []() { return make_ref(FileIngestionMethod::Flat);; }}, {"path", []() { return make_ref(FileIngestionMethod::Recursive); }}, - {"to-base16", []() { return make_ref(Base16); }}, - {"to-base32", []() { return make_ref(Base32); }}, - {"to-base64", []() { return make_ref(Base64); }}, - {"to-sri", []() { return make_ref(SRI); }}, + {"to-base16", []() { return make_ref(HashFormat::Base16); }}, + {"to-base32", []() { return make_ref(HashFormat::Base32); }}, + {"to-base64", []() { return make_ref(HashFormat::Base64); }}, + {"to-sri", []() { return make_ref(HashFormat::SRI); }}, }) { } @@ -162,7 +162,7 @@ static int compatNixHash(int argc, char * * argv) { std::optional ht; bool flat = false; - Base base = Base16; + HashFormat hashFormat = HashFormat::Base16; bool truncate = false; enum { opHash, opTo } op = opHash; std::vector ss; @@ -173,10 +173,10 @@ static int compatNixHash(int argc, char * * argv) else if (*arg == "--version") printVersion("nix-hash"); else if (*arg == "--flat") flat = true; - else if (*arg == "--base16") base = Base16; - else if (*arg == "--base32") base = Base32; - else if (*arg == "--base64") base = Base64; - else if (*arg == "--sri") base = SRI; + else if (*arg == "--base16") hashFormat = HashFormat::Base16; + else if (*arg == "--base32") hashFormat = HashFormat::Base32; + else if (*arg == "--base64") hashFormat = HashFormat::Base64; + else if (*arg == "--sri") hashFormat = HashFormat::SRI; else if (*arg == "--truncate") truncate = true; else if (*arg == "--type") { std::string s = getArg(*arg, arg, end); @@ -184,19 +184,19 @@ static int compatNixHash(int argc, char * * argv) } else if (*arg == "--to-base16") { op = opTo; - base = Base16; + hashFormat = HashFormat::Base16; } else if (*arg == "--to-base32") { op = opTo; - base = Base32; + hashFormat = HashFormat::Base32; } else if (*arg == "--to-base64") { op = opTo; - base = Base64; + hashFormat = HashFormat::Base64; } else if (*arg == "--to-sri") { op = opTo; - base = SRI; + hashFormat = HashFormat::SRI; } else if (*arg != "" && arg->at(0) == '-') return false; @@ -209,14 +209,14 @@ static int compatNixHash(int argc, char * * argv) CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); if (!ht.has_value()) ht = htMD5; cmd.ht = ht.value(); - cmd.base = base; + cmd.hashFormat = hashFormat; cmd.truncate = truncate; cmd.paths = ss; cmd.run(); } else { - CmdToBase cmd(base); + CmdToBase cmd(hashFormat); cmd.args = ss; if (ht.has_value()) cmd.ht = ht; cmd.run(); diff --git a/src/nix/local.mk b/src/nix/local.mk index 20ea29d10fa..57f8259c45e 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -31,7 +31,7 @@ src/nix/develop.cc: src/nix/get-env.sh.gen.hh src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh -src/nix/main.cc: doc/manual/generate-manpage.nix.gen.hh doc/manual/utils.nix.gen.hh +src/nix/main.cc: doc/manual/generate-manpage.nix.gen.hh doc/manual/utils.nix.gen.hh doc/manual/generate-settings.nix.gen.hh doc/manual/generate-store-info.nix.gen.hh src/nix/doc/files/%.md: doc/manual/src/command-ref/files/%.md @mkdir -p $$(dirname $@) diff --git a/src/nix/ls.cc b/src/nix/ls.cc index c990a303c12..231456c9c6c 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -1,6 +1,5 @@ #include "command.hh" #include "store-api.hh" -#include "fs-accessor.hh" #include "nar-accessor.hh" #include "common-args.hh" #include @@ -39,61 +38,58 @@ struct MixLs : virtual Args, MixJSON }); } - void listText(ref accessor) + void listText(ref accessor) { - std::function doPath; + std::function doPath; - auto showFile = [&](const Path & curPath, const std::string & relPath) { + auto showFile = [&](const CanonPath & curPath, std::string_view relPath) { if (verbose) { - auto st = accessor->stat(curPath); + auto st = accessor->lstat(curPath); std::string tp = - st.type == FSAccessor::Type::tRegular ? + st.type == SourceAccessor::Type::tRegular ? (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : - st.type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : + st.type == SourceAccessor::Type::tSymlink ? "lrwxrwxrwx" : "dr-xr-xr-x"; - auto line = fmt("%s %20d %s", tp, st.fileSize, relPath); - if (st.type == FSAccessor::Type::tSymlink) + auto line = fmt("%s %20d %s", tp, st.fileSize.value_or(0), relPath); + if (st.type == SourceAccessor::Type::tSymlink) line += " -> " + accessor->readLink(curPath); logger->cout(line); - if (recursive && st.type == FSAccessor::Type::tDirectory) + if (recursive && st.type == SourceAccessor::Type::tDirectory) doPath(st, curPath, relPath, false); } else { logger->cout(relPath); if (recursive) { - auto st = accessor->stat(curPath); - if (st.type == FSAccessor::Type::tDirectory) + auto st = accessor->lstat(curPath); + if (st.type == SourceAccessor::Type::tDirectory) doPath(st, curPath, relPath, false); } } }; - doPath = [&](const FSAccessor::Stat & st, const Path & curPath, - const std::string & relPath, bool showDirectory) + doPath = [&](const SourceAccessor::Stat & st, const CanonPath & curPath, + std::string_view relPath, bool showDirectory) { - if (st.type == FSAccessor::Type::tDirectory && !showDirectory) { + if (st.type == SourceAccessor::Type::tDirectory && !showDirectory) { auto names = accessor->readDirectory(curPath); - for (auto & name : names) - showFile(curPath + "/" + name, relPath + "/" + name); + for (auto & [name, type] : names) + showFile(curPath + name, relPath + "/" + name); } else showFile(curPath, relPath); }; - auto st = accessor->stat(path); - if (st.type == FSAccessor::Type::tMissing) - throw Error("path '%1%' does not exist", path); - doPath(st, path, - st.type == FSAccessor::Type::tDirectory ? "." : std::string(baseNameOf(path)), + auto path2 = CanonPath(path); + auto st = accessor->lstat(path2); + doPath(st, path2, + st.type == SourceAccessor::Type::tDirectory ? "." : path2.baseName().value_or(""), showDirectory); } - void list(ref accessor) + void list(ref accessor) { - if (path == "/") path = ""; - if (json) { if (showDirectory) throw UsageError("'--directory' is useless with '--json'"); - logger->cout("%s", listNar(accessor, path, recursive)); + logger->cout("%s", listNar(accessor, CanonPath(path), recursive)); } else listText(accessor); } diff --git a/src/nix/main.cc b/src/nix/main.cc index d05bac68e70..73641f6d231 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -1,5 +1,8 @@ #include +#include "args/root.hh" +#include "current-process.hh" +#include "namespaces.hh" #include "command.hh" #include "common-args.hh" #include "eval.hh" @@ -12,12 +15,14 @@ #include "finally.hh" #include "loggers.hh" #include "markdown.hh" +#include "memory-input-accessor.hh" #include #include #include #include #include +#include #include @@ -55,7 +60,7 @@ static bool haveInternet() std::string programPath; -struct NixArgs : virtual MultiCommand, virtual MixCommonArgs +struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs { bool useNet = true; bool refresh = false; @@ -186,6 +191,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs j["experimentalFeature"] = storeConfig->experimentalFeature(); } res["stores"] = std::move(stores); + res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo(); return res.dump(); } @@ -204,21 +210,29 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) auto vGenerateManpage = state.allocValue(); state.eval(state.parseExprFromString( #include "generate-manpage.nix.gen.hh" - , CanonPath::root), *vGenerateManpage); + , state.rootPath(CanonPath::root)), *vGenerateManpage); - auto vUtils = state.allocValue(); - state.cacheFile( - CanonPath("/utils.nix"), CanonPath("/utils.nix"), - state.parseExprFromString( - #include "utils.nix.gen.hh" - , CanonPath::root), - *vUtils); + state.corepkgsFS->addFile( + CanonPath("utils.nix"), + #include "utils.nix.gen.hh" + ); + + state.corepkgsFS->addFile( + CanonPath("/generate-settings.nix"), + #include "generate-settings.nix.gen.hh" + ); + + state.corepkgsFS->addFile( + CanonPath("/generate-store-info.nix"), + #include "generate-store-info.nix.gen.hh" + ); auto vDump = state.allocValue(); vDump->mkString(toplevel.dumpCli()); auto vRes = state.allocValue(); - state.callFunction(*vGenerateManpage, *vDump, *vRes, noPos); + state.callFunction(*vGenerateManpage, state.getBuiltin("false"), *vRes, noPos); + state.callFunction(*vRes, *vDump, *vRes, noPos); auto attr = vRes->attrs->get(state.symbols.create(mdName + ".md")); if (!attr) @@ -232,10 +246,7 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) static NixArgs & getNixArgs(Command & cmd) { - assert(cmd.parent); - MultiCommand * toplevel = cmd.parent; - while (toplevel->parent) toplevel = toplevel->parent; - return dynamic_cast(*toplevel); + return dynamic_cast(cmd.getRoot()); } struct CmdHelp : Command @@ -403,24 +414,26 @@ void mainWrapped(int argc, char * * argv) Finally printCompletions([&]() { - if (completions) { - switch (completionType) { - case ctNormal: + if (args.completions) { + switch (args.completions->type) { + case Completions::Type::Normal: logger->cout("normal"); break; - case ctFilenames: + case Completions::Type::Filenames: logger->cout("filenames"); break; - case ctAttrs: + case Completions::Type::Attrs: logger->cout("attrs"); break; } - for (auto & s : *completions) + for (auto & s : args.completions->completions) logger->cout(s.completion + "\t" + trim(s.description)); } }); try { - args.parseCmdline(argvToStrings(argc, argv)); + auto isNixCommand = std::regex_search(programName, std::regex("nix$")); + auto allowShebang = isNixCommand && argc > 1; + args.parseCmdline(argvToStrings(argc, argv),allowShebang); } catch (UsageError &) { - if (!args.helpRequested && !completions) throw; + if (!args.helpRequested && !args.completions) throw; } if (args.helpRequested) { @@ -437,10 +450,7 @@ void mainWrapped(int argc, char * * argv) return; } - if (completions) { - args.completionHook(); - return; - } + if (args.completions) return; if (args.showVersion) { printVersion(programName); diff --git a/src/nix/nar-dump-path.md b/src/nix/nar-dump-path.md index 26191ad250f..de82202decd 100644 --- a/src/nix/nar-dump-path.md +++ b/src/nix/nar-dump-path.md @@ -5,7 +5,7 @@ R""( * To serialise directory `foo` as a NAR: ```console - # nix nar dump-path ./foo > foo.nar + # nix nar pack ./foo > foo.nar ``` # Description diff --git a/src/nix/nix.md b/src/nix/nix.md index e0f459d6bc5..eb150f03b41 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -132,6 +132,8 @@ subcommands, these are `packages.`*system*, attributes `packages.x86_64-linux.hello`, `legacyPackages.x86_64-linux.hello` and `hello`. +If *attrpath* begins with `.` then no prefixes or defaults are attempted. This allows the form *flakeref*[`#.`*attrpath*], such as `github:NixOS/nixpkgs#.lib.fakeSha256` to avoid a search of `packages.*system*.lib.fakeSha256` + ### Store path Example: `/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10` @@ -236,4 +238,67 @@ operate are determined as follows: Most `nix` subcommands operate on a *Nix store*. These are documented in [`nix help-stores`](./nix3-help-stores.md). +# Shebang interpreter + +The `nix` command can be used as a `#!` interpreter. +Arguments to Nix can be passed on subsequent lines in the script. + +Verbatim strings may be passed in double backtick (```` `` ````) quotes. +Sequences of _n_ backticks of 3 or longer are parsed as _n-1_ literal backticks. +A single space before the closing ```` `` ```` is ignored if present. + +`--file` and `--expr` resolve relative paths based on the script location. + +Examples: + +``` +#!/usr/bin/env nix +#! nix shell --file ```` hello cowsay --command bash + +hello | cowsay +``` + +or with **flakes**: + +``` +#!/usr/bin/env nix +#! nix shell nixpkgs#bash nixpkgs#hello nixpkgs#cowsay --command bash + +hello | cowsay +``` + +or with an **expression**: + +```bash +#! /usr/bin/env nix +#! nix shell --impure --expr `` +#! nix with (import (builtins.getFlake "nixpkgs") {}); +#! nix terraform.withPlugins (plugins: [ plugins.openstack ]) +#! nix `` +#! nix --command bash + +terraform "$@" +``` + +or with cascading interpreters. Note that the `#! nix` lines don't need to follow after the first line, to accomodate other interpreters. + +``` +#!/usr/bin/env nix +//! ```cargo +//! [dependencies] +//! time = "0.1.25" +//! ``` +/* +#!nix shell nixpkgs#rustc nixpkgs#rust-script nixpkgs#cargo --command rust-script +*/ +fn main() { + for argument in std::env::args().skip(1) { + println!("{}", argument); + }; + println!("{}", std::env::var("HOME").expect("")); + println!("{}", time::now().rfc822z()); +} +// vim: ft=rust +``` + )"" diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 613c5b1918d..23198a12087 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -9,6 +9,75 @@ #include using namespace nix; +using nlohmann::json; + +/** + * @return the total size of a set of store objects (specified by path), + * that is, the sum of the size of the NAR serialisation of each object + * in the set. + */ +static uint64_t getStoreObjectsTotalSize(Store & store, const StorePathSet & closure) +{ + uint64_t totalNarSize = 0; + for (auto & p : closure) { + totalNarSize += store.queryPathInfo(p)->narSize; + } + return totalNarSize; +} + + +/** + * Write a JSON representation of store object metadata, such as the + * hash and the references. + * + * @param showClosureSize If true, the closure size of each path is + * included. + */ +static json pathInfoToJSON( + Store & store, + const StorePathSet & storePaths, + bool showClosureSize) +{ + json::object_t jsonAllObjects = json::object(); + + for (auto & storePath : storePaths) { + json jsonObject; + + try { + auto info = store.queryPathInfo(storePath); + + jsonObject = info->toJSON(store, true, HashFormat::SRI); + + if (showClosureSize) { + StorePathSet closure; + store.computeFSClosure(storePath, closure, false, false); + + jsonObject["closureSize"] = getStoreObjectsTotalSize(store, closure); + + if (auto * narInfo = dynamic_cast(&*info)) { + uint64_t totalDownloadSize = 0; + for (auto & p : closure) { + auto depInfo = store.queryPathInfo(p); + if (auto * depNarInfo = dynamic_cast(&*depInfo)) + totalDownloadSize += depNarInfo->fileSize; + else + throw Error("Missing .narinfo for dep %s of %s", + store.printStorePath(p), + store.printStorePath(storePath)); + } + jsonObject["closureDownloadSize"] = totalDownloadSize; + } + } + + } catch (InvalidPath &) { + jsonObject = nullptr; + } + + jsonAllObjects[store.printStorePath(storePath)] = std::move(jsonObject); + } + return jsonAllObjects; +} + struct CmdPathInfo : StorePathsCommand, MixJSON { @@ -87,10 +156,11 @@ struct CmdPathInfo : StorePathsCommand, MixJSON pathLen = std::max(pathLen, store->printStorePath(storePath).size()); if (json) { - std::cout << store->pathInfoToJSON( + std::cout << pathInfoToJSON( + *store, // FIXME: preserve order? StorePathSet(storePaths.begin(), storePaths.end()), - true, showClosureSize, SRI, AllowInvalid).dump(); + showClosureSize).dump(); } else { @@ -107,8 +177,11 @@ struct CmdPathInfo : StorePathsCommand, MixJSON if (showSize) printSize(info->narSize); - if (showClosureSize) - printSize(store->getClosureSize(info->path).first); + if (showClosureSize) { + StorePathSet closure; + store->computeFSClosure(storePath, closure, false, false); + printSize(getStoreObjectsTotalSize(*store, closure)); + } if (showSigs) { std::cout << '\t'; diff --git a/src/nix/path-info.md b/src/nix/path-info.md index 2dda866d05d..4594854eba2 100644 --- a/src/nix/path-info.md +++ b/src/nix/path-info.md @@ -43,7 +43,7 @@ R""( command): ```console - # nix path-info --json --all | jq -r 'sort_by(.registrationTime)[-11:-1][].path' + # nix path-info --json --all | jq -r 'to_entries | sort_by(.value.registrationTime) | .[-11:-1][] | .key' ``` * Show the size of the entire Nix store: @@ -58,13 +58,13 @@ R""( ```console # nix path-info --json --all --closure-size \ - | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])' + | jq 'map_values(.closureSize | select(. < 1e9)) | to_entries | sort_by(.value)' [ …, - [ - "/nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65", - 5887562256 - ] + { + .key = "/nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65", + .value = 5887562256, + } ] ``` diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index b67d381cae1..3ed7946a8b8 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -310,13 +310,13 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON if (json) { auto res = nlohmann::json::object(); res["storePath"] = store->printStorePath(storePath); - res["hash"] = hash.to_string(SRI, true); + res["hash"] = hash.to_string(HashFormat::SRI, true); logger->cout(res.dump()); } else { notice("Downloaded '%s' to '%s' (hash '%s').", url, store->printStorePath(storePath), - hash.to_string(SRI, true)); + hash.to_string(HashFormat::SRI, true)); } } }; diff --git a/src/nix/registry.cc b/src/nix/registry.cc index cb94bbd317f..f509ccae840 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -175,8 +175,8 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand .label = "locked", .optional = true, .handler = {&locked}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); }} }); } diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 9677c1b48bc..63fe3044bb2 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -47,7 +47,7 @@ struct CmdRepl : RawInstallablesCommand void applyDefaultInstallables(std::vector & rawInstallables) override { - if (!experimentalFeatureSettings.isEnabled(Xp::ReplFlake) && !(file) && rawInstallables.size() >= 1) { + if (!experimentalFeatureSettings.isEnabled(Xp::Flakes) && !(file) && rawInstallables.size() >= 1) { warn("future versions of Nix will require using `--file` to load a file"); if (rawInstallables.size() > 1) warn("more than one input file is not currently supported"); diff --git a/src/nix/repl.md b/src/nix/repl.md index c5113be61d2..32c08e24b24 100644 --- a/src/nix/repl.md +++ b/src/nix/repl.md @@ -36,16 +36,13 @@ R""( Loading Installable ''... Added 1 variables. - # nix repl --extra-experimental-features 'flakes repl-flake' nixpkgs + # nix repl --extra-experimental-features 'flakes' nixpkgs Loading Installable 'flake:nixpkgs#'... Added 5 variables. nix-repl> legacyPackages.x86_64-linux.emacs.name "emacs-27.1" - nix-repl> legacyPackages.x86_64-linux.emacs.name - "emacs-27.1" - nix-repl> :q # nix repl --expr 'import {}' diff --git a/src/nix/run.cc b/src/nix/run.cc index 1baf299ab9f..ea0a1789711 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -1,3 +1,4 @@ +#include "current-process.hh" #include "run.hh" #include "command-installable-value.hh" #include "common-args.hh" @@ -6,7 +7,7 @@ #include "derivations.hh" #include "local-store.hh" #include "finally.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "progress-bar.hh" #include "eval.hh" #include "build/personality.hh" @@ -119,9 +120,9 @@ struct CmdShell : InstallablesCommand, MixEnvironment if (true) unixPath.push_front(store->printStorePath(path) + "/bin"); - auto propPath = store->printStorePath(path) + "/nix-support/propagated-user-env-packages"; - if (accessor->stat(propPath).type == FSAccessor::tRegular) { - for (auto & p : tokenizeString(readFile(propPath))) + auto propPath = CanonPath(store->printStorePath(path)) + "nix-support" + "propagated-user-env-packages"; + if (auto st = accessor->maybeLstat(propPath); st && st->type == SourceAccessor::tRegular) { + for (auto & p : tokenizeString(accessor->readFile(propPath))) todo.push(store->parseStorePath(p)); } } diff --git a/src/nix/shell.md b/src/nix/shell.md index f3691957575..7c315fb3f02 100644 --- a/src/nix/shell.md +++ b/src/nix/shell.md @@ -51,4 +51,120 @@ R""( provides the specified [*installables*](./nix.md#installable). If no command is specified, it starts the default shell of your user account specified by `$SHELL`. +# Use as a `#!`-interpreter + +You can use `nix` as a script interpreter to allow scripts written +in arbitrary languages to obtain their own dependencies via Nix. This is +done by starting the script with the following lines: + +```bash +#! /usr/bin/env nix +#! nix shell installables --command real-interpreter +``` + +where *real-interpreter* is the “real” script interpreter that will be +invoked by `nix shell` after it has obtained the dependencies and +initialised the environment, and *installables* are the attribute names of +the dependencies in Nixpkgs. + +The lines starting with `#! nix` specify options (see above). Note that you +cannot write `#! /usr/bin/env nix shell -i ...` because many operating systems +only allow one argument in `#!` lines. + +For example, here is a Python script that depends on Python and the +`prettytable` package: + +```python +#! /usr/bin/env nix +#! nix shell github:tomberek/-#python3With.prettytable --command python + +import prettytable + +# Print a simple table. +t = prettytable.PrettyTable(["N", "N^2"]) +for n in range(1, 10): t.add_row([n, n * n]) +print t +``` + +Similarly, the following is a Perl script that specifies that it +requires Perl and the `HTML::TokeParser::Simple` and `LWP` packages: + +```perl +#! /usr/bin/env nix +#! nix shell github:tomberek/-#perlWith.HTMLTokeParserSimple.LWP --command perl -x + +use HTML::TokeParser::Simple; + +# Fetch nixos.org and print all hrefs. +my $p = HTML::TokeParser::Simple->new(url => 'http://nixos.org/'); + +while (my $token = $p->get_tag("a")) { + my $href = $token->get_attr("href"); + print "$href\n" if $href; +} +``` + +Sometimes you need to pass a simple Nix expression to customize a +package like Terraform: + +```bash +#! /usr/bin/env nix +#! nix shell --impure --expr `` +#! nix with (import (builtins.getFlake ''nixpkgs'') {}); +#! nix terraform.withPlugins (plugins: [ plugins.openstack ]) +#! nix `` +#! nix --command bash + +terraform "$@" +``` + +> **Note** +> +> You must use double backticks (```` `` ````) when passing a simple Nix expression +> in a nix shell shebang. + +Finally, using the merging of multiple nix shell shebangs the following +Haskell script uses a specific branch of Nixpkgs/NixOS (the 21.11 stable +branch): + +```haskell +#!/usr/bin/env nix +#!nix shell --override-input nixpkgs github:NixOS/nixpkgs/nixos-21.11 +#!nix github:tomberek/-#haskellWith.download-curl.tagsoup --command runghc + +import Network.Curl.Download +import Text.HTML.TagSoup +import Data.Either +import Data.ByteString.Char8 (unpack) + +-- Fetch nixos.org and print all hrefs. +main = do + resp <- openURI "https://nixos.org/" + let tags = filter (isTagOpenName "a") $ parseTags $ unpack $ fromRight undefined resp + let tags' = map (fromAttrib "href") tags + mapM_ putStrLn $ filter (/= "") tags' +``` + +If you want to be even more precise, you can specify a specific revision +of Nixpkgs: + + #!nix shell --override-input nixpkgs github:NixOS/nixpkgs/eabc38219184cc3e04a974fe31857d8e0eac098d + +You can also use a Nix expression to build your own dependencies. For example, +the Python example could have been written as: + +```python +#! /usr/bin/env nix +#! nix shell --impure --file deps.nix -i python +``` + +where the file `deps.nix` in the same directory as the `#!`-script +contains: + +```nix +with import {}; +python3.withPackages (ps: with ps; [ prettytable ]) +``` + + )"" diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 45cd2e1a6c2..39555c9eae3 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -1,7 +1,9 @@ +#include "signals.hh" #include "command.hh" #include "shared.hh" #include "store-api.hh" #include "thread-pool.hh" +#include "progress-bar.hh" #include @@ -173,6 +175,7 @@ struct CmdKeyGenerateSecret : Command if (!keyName) throw UsageError("required argument '--key-name' is missing"); + stopProgressBar(); writeFull(STDOUT_FILENO, SecretKey::generate(*keyName).to_string()); } }; @@ -194,6 +197,7 @@ struct CmdKeyConvertSecretToPublic : Command void run() override { SecretKey secretKey(drainFD(STDIN_FILENO)); + stopProgressBar(); writeFull(STDOUT_FILENO, secretKey.toPublicKey().to_string()); } }; diff --git a/src/nix/ping-store.cc b/src/nix/store-info.cc similarity index 78% rename from src/nix/ping-store.cc rename to src/nix/store-info.cc index ec450e8e0a9..a7c59576146 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/store-info.cc @@ -17,7 +17,7 @@ struct CmdPingStore : StoreCommand, MixJSON std::string doc() override { return - #include "ping-store.md" + #include "store-info.md" ; } @@ -46,4 +46,15 @@ struct CmdPingStore : StoreCommand, MixJSON } }; -static auto rCmdPingStore = registerCommand2({"store", "ping"}); +struct CmdInfoStore : CmdPingStore +{ + void run(nix::ref store) override + { + warn("'nix store ping' is a deprecated alias for 'nix store info'"); + CmdPingStore::run(store); + } +}; + + +static auto rCmdPingStore = registerCommand2({"store", "info"}); +static auto rCmdInfoStore = registerCommand2({"store", "ping"}); diff --git a/src/nix/ping-store.md b/src/nix/store-info.md similarity index 82% rename from src/nix/ping-store.md rename to src/nix/store-info.md index 8c846791b9f..f86efd72249 100644 --- a/src/nix/ping-store.md +++ b/src/nix/store-info.md @@ -5,19 +5,19 @@ R""( * Test whether connecting to a remote Nix store via SSH works: ```console - # nix store ping --store ssh://mac1 + # nix store info --store ssh://mac1 ``` * Test whether a URL is a valid binary cache: ```console - # nix store ping --store https://cache.nixos.org + # nix store info --store https://cache.nixos.org ``` * Test whether the Nix daemon is up and running: ```console - # nix store ping --store daemon + # nix store info --store daemon ``` # Description diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index d238456db5b..4c7a74e16f1 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -1,3 +1,4 @@ +#include "processes.hh" #include "command.hh" #include "common-args.hh" #include "store-api.hh" @@ -13,7 +14,6 @@ using namespace nix; struct CmdUpgradeNix : MixDryRun, StoreCommand { Path profileDir; - std::string storePathsUrl = "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix"; CmdUpgradeNix() { @@ -29,7 +29,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand .longName = "nix-store-paths-url", .description = "The URL of the file that contains the store paths of the latest Nix release.", .labels = {"url"}, - .handler = {&storePathsUrl} + .handler = {&(std::string&) settings.upgradeNixStorePathUrl} }); } @@ -43,7 +43,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand std::string description() override { - return "upgrade Nix to the stable version declared in Nixpkgs"; + return "upgrade Nix to the latest stable version"; } std::string doc() override @@ -144,7 +144,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version"); // FIXME: use nixos.org? - auto req = FileTransferRequest(storePathsUrl); + auto req = FileTransferRequest((std::string&) settings.upgradeNixStorePathUrl); auto res = getFileTransfer()->download(req); auto state = std::make_unique(SearchPath{}, store); diff --git a/src/nix/upgrade-nix.md b/src/nix/upgrade-nix.md index cce88c3970d..3a3bf61b9b0 100644 --- a/src/nix/upgrade-nix.md +++ b/src/nix/upgrade-nix.md @@ -16,8 +16,10 @@ R""( # Description -This command upgrades Nix to the stable version declared in Nixpkgs. -This stable version is defined in [nix-fallback-paths.nix](https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix) +This command upgrades Nix to the stable version. + +By default, the latest stable version is defined by Nixpkgs, in +[nix-fallback-paths.nix](https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix) and updated manually. It may not always be the latest tagged release. By default, it locates the directory containing the `nix` binary in the `$PATH` diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 0b306cc1159..78cb765ce17 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -4,6 +4,7 @@ #include "sync.hh" #include "thread-pool.hh" #include "references.hh" +#include "signals.hh" #include @@ -108,8 +109,8 @@ struct CmdVerify : StorePathsCommand act2.result(resCorruptedPath, store->printStorePath(info->path)); printError("path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(info->path), - info->narHash.to_string(Base32, true), - hash.first.to_string(Base32, true)); + info->narHash.to_string(HashFormat::Base32, true), + hash.first.to_string(HashFormat::Base32, true)); } } diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 592de773ce9..aecf6592222 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -1,7 +1,7 @@ #include "command.hh" #include "store-api.hh" #include "progress-bar.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "shared.hh" #include @@ -38,17 +38,13 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions expectArgs({ .label = "package", .handler = {&_package}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .completer = getCompleteInstallable(), }); expectArgs({ .label = "dependency", .handler = {&_dependency}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .completer = getCompleteInstallable(), }); addFlag({ @@ -179,7 +175,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions struct BailOut { }; printNode = [&](Node & node, const std::string & firstPad, const std::string & tailPad) { - auto pathS = store->printStorePath(node.path); + CanonPath pathS(store->printStorePath(node.path)); assert(node.dist != inf); if (precise) { @@ -187,7 +183,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions firstPad, node.visited ? "\e[38;5;244m" : "", firstPad != "" ? "→ " : "", - pathS); + pathS.abs()); } if (node.path == dependencyPath && !all @@ -214,24 +210,25 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions contain the reference. */ std::map hits; - std::function visitPath; + std::function visitPath; - visitPath = [&](const Path & p) { - auto st = accessor->stat(p); + visitPath = [&](const CanonPath & p) { + auto st = accessor->maybeLstat(p); + assert(st); - auto p2 = p == pathS ? "/" : std::string(p, pathS.size() + 1); + auto p2 = p == pathS ? "/" : p.abs().substr(pathS.abs().size() + 1); auto getColour = [&](const std::string & hash) { return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE; }; - if (st.type == FSAccessor::Type::tDirectory) { + if (st->type == SourceAccessor::Type::tDirectory) { auto names = accessor->readDirectory(p); - for (auto & name : names) - visitPath(p + "/" + name); + for (auto & [name, type] : names) + visitPath(p + name); } - else if (st.type == FSAccessor::Type::tRegular) { + else if (st->type == SourceAccessor::Type::tRegular) { auto contents = accessor->readFile(p); for (auto & hash : hashes) { @@ -249,7 +246,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions } } - else if (st.type == FSAccessor::Type::tSymlink) { + else if (st->type == SourceAccessor::Type::tSymlink) { auto target = accessor->readLink(p); for (auto & hash : hashes) { diff --git a/tests/dyn-drv/dep-built-drv.sh b/tests/dyn-drv/dep-built-drv.sh deleted file mode 100644 index c3daf2c5917..00000000000 --- a/tests/dyn-drv/dep-built-drv.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -source common.sh - -out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link) - -clearStore - -out2=$(nix-build ./text-hashed-output.nix -A wrapper --no-out-link) - -diff -r $out1 $out2 diff --git a/tests/flakes/flakes.sh b/tests/flakes/flakes.sh deleted file mode 100644 index 128f759ea41..00000000000 --- a/tests/flakes/flakes.sh +++ /dev/null @@ -1,511 +0,0 @@ -source ./common.sh - -requireGit - -clearStore -rm -rf $TEST_HOME/.cache $TEST_HOME/.config - -flake1Dir=$TEST_ROOT/flake1 -flake2Dir=$TEST_ROOT/flake2 -flake3Dir=$TEST_ROOT/flake3 -flake5Dir=$TEST_ROOT/flake5 -flake7Dir=$TEST_ROOT/flake7 -nonFlakeDir=$TEST_ROOT/nonFlake -badFlakeDir=$TEST_ROOT/badFlake -flakeGitBare=$TEST_ROOT/flakeGitBare - -for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $nonFlakeDir; do - # Give one repo a non-main initial branch. - extraArgs= - if [[ $repo == $flake2Dir ]]; then - extraArgs="--initial-branch=main" - fi - - createGitRepo "$repo" "$extraArgs" -done - -createSimpleGitFlake $flake1Dir - -cat > $flake2Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/default.nix < $nonFlakeDir/README.md < $flake1Dir/foo -git -C $flake1Dir add $flake1Dir/foo -[[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "$hash1-dirty" ]] - -echo -n '# foo' >> $flake1Dir/flake.nix -flake1OriginalCommit=$(git -C $flake1Dir rev-parse HEAD) -git -C $flake1Dir commit -a -m 'Foo' -flake1NewCommit=$(git -C $flake1Dir rev-parse HEAD) -hash2=$(nix flake metadata flake1 --json --refresh | jq -r .revision) -[[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "null" ]] -[[ $hash1 != $hash2 ]] - -# Test 'nix build' on a flake. -nix build -o $TEST_ROOT/result flake1#foo -[[ -e $TEST_ROOT/result/hello ]] - -# Test packages.default. -nix build -o $TEST_ROOT/result flake1 -[[ -e $TEST_ROOT/result/hello ]] - -nix build -o $TEST_ROOT/result $flake1Dir -nix build -o $TEST_ROOT/result git+file://$flake1Dir - -# Check that store symlinks inside a flake are not interpreted as flakes. -nix build -o $flake1Dir/result git+file://$flake1Dir -nix path-info $flake1Dir/result - -# 'getFlake' on an unlocked flakeref should fail in pure mode, but -# succeed in impure mode. -(! nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default") -nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default" --impure - -# 'getFlake' on a locked flakeref should succeed even in pure mode. -nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"git+file://$flake1Dir?rev=$hash2\").packages.$system.default" - -# Building a flake with an unlocked dependency should fail in pure mode. -(! nix build -o $TEST_ROOT/result flake2#bar --no-registries) -(! nix build -o $TEST_ROOT/result flake2#bar --no-use-registries) -(! nix eval --expr "builtins.getFlake \"$flake2Dir\"") - -# But should succeed in impure mode. -(! nix build -o $TEST_ROOT/result flake2#bar --impure) -nix build -o $TEST_ROOT/result flake2#bar --impure --no-write-lock-file -nix eval --expr "builtins.getFlake \"$flake2Dir\"" --impure - -# Building a local flake with an unlocked dependency should fail with --no-update-lock-file. -expect 1 nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes' - -# But it should succeed without that flag. -nix build -o $TEST_ROOT/result $flake2Dir#bar --no-write-lock-file -expect 1 nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes' -nix build -o $TEST_ROOT/result $flake2Dir#bar --commit-lock-file -[[ -e $flake2Dir/flake.lock ]] -[[ -z $(git -C $flake2Dir diff main || echo failed) ]] - -# Rerunning the build should not change the lockfile. -nix build -o $TEST_ROOT/result $flake2Dir#bar -[[ -z $(git -C $flake2Dir diff main || echo failed) ]] - -# Building with a lockfile should not require a fetch of the registry. -nix build -o $TEST_ROOT/result --flake-registry file:///no-registry.json $flake2Dir#bar --refresh -nix build -o $TEST_ROOT/result --no-registries $flake2Dir#bar --refresh -nix build -o $TEST_ROOT/result --no-use-registries $flake2Dir#bar --refresh - -# Updating the flake should not change the lockfile. -nix flake lock $flake2Dir -[[ -z $(git -C $flake2Dir diff main || echo failed) ]] - -# Now we should be able to build the flake in pure mode. -nix build -o $TEST_ROOT/result flake2#bar - -# Or without a registry. -nix build -o $TEST_ROOT/result --no-registries git+file://$flake2Dir#bar --refresh -nix build -o $TEST_ROOT/result --no-use-registries git+file://$flake2Dir#bar --refresh - -# Test whether indirect dependencies work. -nix build -o $TEST_ROOT/result $flake3Dir#xyzzy -git -C $flake3Dir add flake.lock - -# Add dependency to flake3. -rm $flake3Dir/flake.nix - -cat > $flake3Dir/flake.nix < $flake3Dir/flake.nix < \$out - [[ \$(cat \${inputs.nonFlake}/README.md) = \$(cat \${inputs.nonFlakeFile}) ]] - [[ \${inputs.nonFlakeFile} = \${inputs.nonFlakeFile2} ]] - ''; - }; - }; -} -EOF - -cp ../config.nix $flake3Dir - -git -C $flake3Dir add flake.nix config.nix -git -C $flake3Dir commit -m 'Add nonFlakeInputs' - -# Check whether `nix build` works with a lockfile which is missing a -# nonFlakeInputs. -nix build -o $TEST_ROOT/result $flake3Dir#sth --commit-lock-file - -nix build -o $TEST_ROOT/result flake3#fnord -[[ $(cat $TEST_ROOT/result) = FNORD ]] - -# Check whether flake input fetching is lazy: flake3#sth does not -# depend on flake2, so this shouldn't fail. -rm -rf $TEST_HOME/.cache -clearStore -mv $flake2Dir $flake2Dir.tmp -mv $nonFlakeDir $nonFlakeDir.tmp -nix build -o $TEST_ROOT/result flake3#sth -(! nix build -o $TEST_ROOT/result flake3#xyzzy) -(! nix build -o $TEST_ROOT/result flake3#fnord) -mv $flake2Dir.tmp $flake2Dir -mv $nonFlakeDir.tmp $nonFlakeDir -nix build -o $TEST_ROOT/result flake3#xyzzy flake3#fnord - -# Test doing multiple `lookupFlake`s -nix build -o $TEST_ROOT/result flake4#xyzzy - -# Test 'nix flake update' and --override-flake. -nix flake lock $flake3Dir -[[ -z $(git -C $flake3Dir diff master || echo failed) ]] - -nix flake update $flake3Dir --override-flake flake2 nixpkgs -[[ ! -z $(git -C $flake3Dir diff master || echo failed) ]] - -# Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore -git -C $flake3Dir checkout -b removeXyzzy -rm $flake3Dir/flake.nix - -cat > $flake3Dir/flake.nix < \$out - ''; - }; - }; -} -EOF -nix flake lock $flake3Dir -git -C $flake3Dir add flake.nix flake.lock -git -C $flake3Dir commit -m 'Remove packages.xyzzy' -git -C $flake3Dir checkout master - -# Test whether fuzzy-matching works for registry entries. -(! nix build -o $TEST_ROOT/result flake4/removeXyzzy#xyzzy) -nix build -o $TEST_ROOT/result flake4/removeXyzzy#sth - -# Testing the nix CLI -nix registry add flake1 flake3 -[[ $(nix registry list | wc -l) == 6 ]] -nix registry pin flake1 -[[ $(nix registry list | wc -l) == 6 ]] -nix registry pin flake1 flake3 -[[ $(nix registry list | wc -l) == 6 ]] -nix registry remove flake1 -[[ $(nix registry list | wc -l) == 5 ]] - -# Test 'nix registry list' with a disabled global registry. -nix registry add user-flake1 git+file://$flake1Dir -nix registry add user-flake2 git+file://$flake2Dir -[[ $(nix --flake-registry "" registry list | wc -l) == 2 ]] -nix --flake-registry "" registry list | grepQuietInverse '^global' # nothing in global registry -nix --flake-registry "" registry list | grepQuiet '^user' -nix registry remove user-flake1 -nix registry remove user-flake2 -[[ $(nix registry list | wc -l) == 5 ]] - -# Test 'nix flake clone'. -rm -rf $TEST_ROOT/flake1-v2 -nix flake clone flake1 --dest $TEST_ROOT/flake1-v2 -[ -e $TEST_ROOT/flake1-v2/flake.nix ] - -# Test 'follows' inputs. -cat > $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $badFlakeDir/flake.nix -nix store delete $(nix store add-path $badFlakeDir) - -[[ $(nix path-info $(nix store add-path $flake1Dir)) =~ flake1 ]] -[[ $(nix path-info path:$(nix store add-path $flake1Dir)) =~ simple ]] - -# Test fetching flakerefs in the legacy CLI. -[[ $(nix-instantiate --eval flake:flake3 -A x) = 123 ]] -[[ $(nix-instantiate --eval flake:git+file://$flake3Dir -A x) = 123 ]] -[[ $(nix-instantiate -I flake3=flake:flake3 --eval '' -A x) = 123 ]] -[[ $(NIX_PATH=flake3=flake:flake3 nix-instantiate --eval '' -A x) = 123 ]] - -# Test alternate lockfile paths. -nix flake lock $flake2Dir --output-lock-file $TEST_ROOT/flake2.lock -cmp $flake2Dir/flake.lock $TEST_ROOT/flake2.lock >/dev/null # lockfiles should be identical, since we're referencing flake2's original one - -nix flake lock $flake2Dir --output-lock-file $TEST_ROOT/flake2-overridden.lock --override-input flake1 git+file://$flake1Dir?rev=$flake1OriginalCommit -expectStderr 1 cmp $flake2Dir/flake.lock $TEST_ROOT/flake2-overridden.lock -nix flake metadata $flake2Dir --reference-lock-file $TEST_ROOT/flake2-overridden.lock | grepQuiet $flake1OriginalCommit - -# reference-lock-file can only be used if allow-dirty is set. -expectStderr 1 nix flake metadata $flake2Dir --no-allow-dirty --reference-lock-file $TEST_ROOT/flake2-overridden.lock diff --git a/tests/add.sh b/tests/functional/add.sh similarity index 62% rename from tests/add.sh rename to tests/functional/add.sh index 5c3eed7931a..d0fedcb251c 100644 --- a/tests/add.sh +++ b/tests/functional/add.sh @@ -26,3 +26,20 @@ hash2=$(nix-hash --type sha256 --base32 ./dummy) echo $hash2 test "$hash1" = "sha256:$hash2" + +#### New style commands + +clearStore + +( + path1=$(nix store add ./dummy) + path2=$(nix store add --mode nar ./dummy) + path3=$(nix store add-path ./dummy) + [[ "$path1" == "$path2" ]] + [[ "$path1" == "$path3" ]] +) +( + path1=$(nix store add --mode flat ./dummy) + path2=$(nix store add-file ./dummy) + [[ "$path1" == "$path2" ]] +) diff --git a/tests/bad.tar.xz b/tests/functional/bad.tar.xz similarity index 100% rename from tests/bad.tar.xz rename to tests/functional/bad.tar.xz diff --git a/tests/bash-profile.sh b/tests/functional/bash-profile.sh similarity index 78% rename from tests/bash-profile.sh rename to tests/functional/bash-profile.sh index e2e0d109080..3faeaaba1dc 100644 --- a/tests/bash-profile.sh +++ b/tests/functional/bash-profile.sh @@ -1,6 +1,6 @@ source common.sh -sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../scripts/nix-profile.sh.in > $TEST_ROOT/nix-profile.sh +sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../../scripts/nix-profile.sh.in > $TEST_ROOT/nix-profile.sh user=$(whoami) rm -rf $TEST_HOME $TEST_ROOT/profile-var diff --git a/tests/big-derivation-attr.nix b/tests/functional/big-derivation-attr.nix similarity index 100% rename from tests/big-derivation-attr.nix rename to tests/functional/big-derivation-attr.nix diff --git a/tests/binary-cache-build-remote.sh b/tests/functional/binary-cache-build-remote.sh similarity index 100% rename from tests/binary-cache-build-remote.sh rename to tests/functional/binary-cache-build-remote.sh diff --git a/tests/binary-cache.sh b/tests/functional/binary-cache.sh similarity index 100% rename from tests/binary-cache.sh rename to tests/functional/binary-cache.sh diff --git a/tests/brotli.sh b/tests/functional/brotli.sh similarity index 100% rename from tests/brotli.sh rename to tests/functional/brotli.sh diff --git a/tests/build-delete.sh b/tests/functional/build-delete.sh similarity index 100% rename from tests/build-delete.sh rename to tests/functional/build-delete.sh diff --git a/tests/build-dry.sh b/tests/functional/build-dry.sh similarity index 100% rename from tests/build-dry.sh rename to tests/functional/build-dry.sh diff --git a/tests/build-hook-ca-fixed.nix b/tests/functional/build-hook-ca-fixed.nix similarity index 91% rename from tests/build-hook-ca-fixed.nix rename to tests/functional/build-hook-ca-fixed.nix index 4cb9e85d129..0ce6d9b128b 100644 --- a/tests/build-hook-ca-fixed.nix +++ b/tests/functional/build-hook-ca-fixed.nix @@ -8,7 +8,10 @@ let derivation ({ inherit system; builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; outputHashMode = "recursive"; outputHashAlgo = "sha256"; } // removeAttrs args ["builder" "meta" "passthru"]) diff --git a/tests/build-hook-ca-floating.nix b/tests/functional/build-hook-ca-floating.nix similarity index 100% rename from tests/build-hook-ca-floating.nix rename to tests/functional/build-hook-ca-floating.nix diff --git a/tests/build-hook.nix b/tests/functional/build-hook.nix similarity index 90% rename from tests/build-hook.nix rename to tests/functional/build-hook.nix index 7effd79037f..99a13aee483 100644 --- a/tests/build-hook.nix +++ b/tests/functional/build-hook.nix @@ -14,7 +14,10 @@ let derivation ({ inherit system; builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; } // removeAttrs args ["builder" "meta" "passthru"] // caArgs) // { meta = args.meta or {}; passthru = args.passthru or {}; }; diff --git a/tests/build-remote-content-addressed-fixed.sh b/tests/functional/build-remote-content-addressed-fixed.sh similarity index 100% rename from tests/build-remote-content-addressed-fixed.sh rename to tests/functional/build-remote-content-addressed-fixed.sh diff --git a/tests/build-remote-content-addressed-floating.sh b/tests/functional/build-remote-content-addressed-floating.sh similarity index 100% rename from tests/build-remote-content-addressed-floating.sh rename to tests/functional/build-remote-content-addressed-floating.sh diff --git a/tests/build-remote-input-addressed.sh b/tests/functional/build-remote-input-addressed.sh similarity index 100% rename from tests/build-remote-input-addressed.sh rename to tests/functional/build-remote-input-addressed.sh diff --git a/tests/build-remote-trustless-after.sh b/tests/functional/build-remote-trustless-after.sh similarity index 100% rename from tests/build-remote-trustless-after.sh rename to tests/functional/build-remote-trustless-after.sh diff --git a/tests/build-remote-trustless-should-fail-0.sh b/tests/functional/build-remote-trustless-should-fail-0.sh similarity index 100% rename from tests/build-remote-trustless-should-fail-0.sh rename to tests/functional/build-remote-trustless-should-fail-0.sh diff --git a/tests/build-remote-trustless-should-pass-0.sh b/tests/functional/build-remote-trustless-should-pass-0.sh similarity index 100% rename from tests/build-remote-trustless-should-pass-0.sh rename to tests/functional/build-remote-trustless-should-pass-0.sh diff --git a/tests/build-remote-trustless-should-pass-1.sh b/tests/functional/build-remote-trustless-should-pass-1.sh similarity index 100% rename from tests/build-remote-trustless-should-pass-1.sh rename to tests/functional/build-remote-trustless-should-pass-1.sh diff --git a/tests/build-remote-trustless-should-pass-2.sh b/tests/functional/build-remote-trustless-should-pass-2.sh similarity index 100% rename from tests/build-remote-trustless-should-pass-2.sh rename to tests/functional/build-remote-trustless-should-pass-2.sh diff --git a/tests/build-remote-trustless-should-pass-3.sh b/tests/functional/build-remote-trustless-should-pass-3.sh similarity index 100% rename from tests/build-remote-trustless-should-pass-3.sh rename to tests/functional/build-remote-trustless-should-pass-3.sh diff --git a/tests/build-remote-trustless.sh b/tests/functional/build-remote-trustless.sh similarity index 82% rename from tests/build-remote-trustless.sh rename to tests/functional/build-remote-trustless.sh index 9df44e0c5ef..81e5253bf29 100644 --- a/tests/build-remote-trustless.sh +++ b/tests/functional/build-remote-trustless.sh @@ -6,7 +6,7 @@ unset NIX_STATE_DIR remoteDir=$TEST_ROOT/remote -# Note: ssh{-ng}://localhost bypasses ssh. See tests/build-remote.sh for +# Note: ssh{-ng}://localhost bypasses ssh. See tests/functional/build-remote.sh for # more details. nix-build $file -o $TEST_ROOT/result --max-jobs 0 \ --arg busybox $busybox \ diff --git a/tests/build-remote.sh b/tests/functional/build-remote.sh similarity index 100% rename from tests/build-remote.sh rename to tests/functional/build-remote.sh diff --git a/tests/build.sh b/tests/functional/build.sh similarity index 100% rename from tests/build.sh rename to tests/functional/build.sh diff --git a/tests/ca-shell.nix b/tests/functional/ca-shell.nix similarity index 100% rename from tests/ca-shell.nix rename to tests/functional/ca-shell.nix diff --git a/tests/ca/build-cache.sh b/tests/functional/ca/build-cache.sh similarity index 100% rename from tests/ca/build-cache.sh rename to tests/functional/ca/build-cache.sh diff --git a/tests/ca/build-dry.sh b/tests/functional/ca/build-dry.sh similarity index 100% rename from tests/ca/build-dry.sh rename to tests/functional/ca/build-dry.sh diff --git a/tests/ca/build-with-garbage-path.sh b/tests/functional/ca/build-with-garbage-path.sh similarity index 100% rename from tests/ca/build-with-garbage-path.sh rename to tests/functional/ca/build-with-garbage-path.sh diff --git a/tests/ca/build.sh b/tests/functional/ca/build.sh similarity index 100% rename from tests/ca/build.sh rename to tests/functional/ca/build.sh diff --git a/tests/ca/common.sh b/tests/functional/ca/common.sh similarity index 100% rename from tests/ca/common.sh rename to tests/functional/ca/common.sh diff --git a/tests/ca/concurrent-builds.sh b/tests/functional/ca/concurrent-builds.sh similarity index 100% rename from tests/ca/concurrent-builds.sh rename to tests/functional/ca/concurrent-builds.sh diff --git a/tests/ca/config.nix.in b/tests/functional/ca/config.nix.in similarity index 100% rename from tests/ca/config.nix.in rename to tests/functional/ca/config.nix.in diff --git a/tests/ca/content-addressed.nix b/tests/functional/ca/content-addressed.nix similarity index 100% rename from tests/ca/content-addressed.nix rename to tests/functional/ca/content-addressed.nix diff --git a/tests/ca/derivation-json.sh b/tests/functional/ca/derivation-json.sh similarity index 100% rename from tests/ca/derivation-json.sh rename to tests/functional/ca/derivation-json.sh diff --git a/tests/ca/duplicate-realisation-in-closure.sh b/tests/functional/ca/duplicate-realisation-in-closure.sh similarity index 100% rename from tests/ca/duplicate-realisation-in-closure.sh rename to tests/functional/ca/duplicate-realisation-in-closure.sh diff --git a/tests/ca/flake.nix b/tests/functional/ca/flake.nix similarity index 100% rename from tests/ca/flake.nix rename to tests/functional/ca/flake.nix diff --git a/tests/ca/gc.sh b/tests/functional/ca/gc.sh similarity index 100% rename from tests/ca/gc.sh rename to tests/functional/ca/gc.sh diff --git a/tests/ca/import-derivation.sh b/tests/functional/ca/import-derivation.sh similarity index 100% rename from tests/ca/import-derivation.sh rename to tests/functional/ca/import-derivation.sh diff --git a/tests/ca/local.mk b/tests/functional/ca/local.mk similarity index 94% rename from tests/ca/local.mk rename to tests/functional/ca/local.mk index 0852e592e17..fd87b8d1f8c 100644 --- a/tests/ca/local.mk +++ b/tests/functional/ca/local.mk @@ -25,4 +25,4 @@ clean-files += \ $(d)/config.nix test-deps += \ - tests/ca/config.nix + tests/functional/ca/config.nix diff --git a/tests/ca/new-build-cmd.sh b/tests/functional/ca/new-build-cmd.sh similarity index 100% rename from tests/ca/new-build-cmd.sh rename to tests/functional/ca/new-build-cmd.sh diff --git a/tests/ca/nix-copy.sh b/tests/functional/ca/nix-copy.sh similarity index 100% rename from tests/ca/nix-copy.sh rename to tests/functional/ca/nix-copy.sh diff --git a/tests/ca/nix-run.sh b/tests/functional/ca/nix-run.sh similarity index 100% rename from tests/ca/nix-run.sh rename to tests/functional/ca/nix-run.sh diff --git a/tests/ca/nix-shell.sh b/tests/functional/ca/nix-shell.sh similarity index 100% rename from tests/ca/nix-shell.sh rename to tests/functional/ca/nix-shell.sh diff --git a/tests/ca/nondeterministic.nix b/tests/functional/ca/nondeterministic.nix similarity index 100% rename from tests/ca/nondeterministic.nix rename to tests/functional/ca/nondeterministic.nix diff --git a/tests/ca/post-hook.sh b/tests/functional/ca/post-hook.sh similarity index 100% rename from tests/ca/post-hook.sh rename to tests/functional/ca/post-hook.sh diff --git a/tests/ca/racy.nix b/tests/functional/ca/racy.nix similarity index 100% rename from tests/ca/racy.nix rename to tests/functional/ca/racy.nix diff --git a/tests/ca/recursive.sh b/tests/functional/ca/recursive.sh similarity index 100% rename from tests/ca/recursive.sh rename to tests/functional/ca/recursive.sh diff --git a/tests/ca/repl.sh b/tests/functional/ca/repl.sh similarity index 100% rename from tests/ca/repl.sh rename to tests/functional/ca/repl.sh diff --git a/tests/ca/selfref-gc.sh b/tests/functional/ca/selfref-gc.sh similarity index 100% rename from tests/ca/selfref-gc.sh rename to tests/functional/ca/selfref-gc.sh diff --git a/tests/ca/signatures.sh b/tests/functional/ca/signatures.sh similarity index 100% rename from tests/ca/signatures.sh rename to tests/functional/ca/signatures.sh diff --git a/tests/ca/substitute.sh b/tests/functional/ca/substitute.sh similarity index 100% rename from tests/ca/substitute.sh rename to tests/functional/ca/substitute.sh diff --git a/tests/ca/why-depends.sh b/tests/functional/ca/why-depends.sh similarity index 100% rename from tests/ca/why-depends.sh rename to tests/functional/ca/why-depends.sh diff --git a/tests/case-hack.sh b/tests/functional/case-hack.sh similarity index 100% rename from tests/case-hack.sh rename to tests/functional/case-hack.sh diff --git a/tests/case.nar b/tests/functional/case.nar similarity index 100% rename from tests/case.nar rename to tests/functional/case.nar diff --git a/tests/check-refs.nix b/tests/functional/check-refs.nix similarity index 100% rename from tests/check-refs.nix rename to tests/functional/check-refs.nix diff --git a/tests/check-refs.sh b/tests/functional/check-refs.sh similarity index 100% rename from tests/check-refs.sh rename to tests/functional/check-refs.sh diff --git a/tests/check-reqs.nix b/tests/functional/check-reqs.nix similarity index 100% rename from tests/check-reqs.nix rename to tests/functional/check-reqs.nix diff --git a/tests/check-reqs.sh b/tests/functional/check-reqs.sh similarity index 100% rename from tests/check-reqs.sh rename to tests/functional/check-reqs.sh diff --git a/tests/check.nix b/tests/functional/check.nix similarity index 100% rename from tests/check.nix rename to tests/functional/check.nix diff --git a/tests/check.sh b/tests/functional/check.sh similarity index 100% rename from tests/check.sh rename to tests/functional/check.sh diff --git a/tests/common.sh b/tests/functional/common.sh similarity index 100% rename from tests/common.sh rename to tests/functional/common.sh diff --git a/tests/common/vars-and-functions.sh.in b/tests/functional/common/vars-and-functions.sh.in similarity index 99% rename from tests/common/vars-and-functions.sh.in rename to tests/functional/common/vars-and-functions.sh.in index 8f9ec4b1a38..848988af996 100644 --- a/tests/common/vars-and-functions.sh.in +++ b/tests/functional/common/vars-and-functions.sh.in @@ -4,9 +4,9 @@ if [[ -z "${COMMON_VARS_AND_FUNCTIONS_SH_SOURCED-}" ]]; then COMMON_VARS_AND_FUNCTIONS_SH_SOURCED=1 -export PS4='+(${BASH_SOURCE[0]-$0}:$LINENO) ' +set +x -export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default} +export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default/tests\/functional//} export NIX_STORE_DIR if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then # Maybe the build directory is symlinked. diff --git a/tests/completions.sh b/tests/functional/completions.sh similarity index 81% rename from tests/completions.sh rename to tests/functional/completions.sh index 19dc610989e..d3d5bbd48b1 100644 --- a/tests/completions.sh +++ b/tests/functional/completions.sh @@ -44,13 +44,18 @@ EOF # Input override completion [[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '')" == $'normal\na\t' ]] [[ "$(NIX_GET_COMPLETIONS=5 nix flake show ./foo --override-input '')" == $'normal\na\t' ]] +cd ./foo +[[ "$(NIX_GET_COMPLETIONS=3 nix flake update '')" == $'normal\na\t' ]] +cd .. +[[ "$(NIX_GET_COMPLETIONS=5 nix flake update --flake './foo' '')" == $'normal\na\t' ]] ## With multiple input flakes [[ "$(NIX_GET_COMPLETIONS=5 nix build ./foo ./bar --override-input '')" == $'normal\na\t\nb\t' ]] ## With tilde expansion [[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix build '~/foo' --override-input '')" == $'normal\na\t' ]] +[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=5 nix flake update --flake '~/foo' '')" == $'normal\na\t' ]] ## Out of order -[[ "$(NIX_GET_COMPLETIONS=3 nix build --update-input '' ./foo)" == $'normal\na\t' ]] -[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --update-input '' ./bar)" == $'normal\na\t\nb\t' ]] +[[ "$(NIX_GET_COMPLETIONS=3 nix build --override-input '' '' ./foo)" == $'normal\na\t' ]] +[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '' '' ./bar)" == $'normal\na\t\nb\t' ]] # Cli flag completion NIX_GET_COMPLETIONS=2 nix build --log-form | grep -- "--log-format" diff --git a/tests/compression-levels.sh b/tests/functional/compression-levels.sh similarity index 100% rename from tests/compression-levels.sh rename to tests/functional/compression-levels.sh diff --git a/tests/compute-levels.sh b/tests/functional/compute-levels.sh similarity index 100% rename from tests/compute-levels.sh rename to tests/functional/compute-levels.sh diff --git a/tests/config.nix.in b/tests/functional/config.nix.in similarity index 82% rename from tests/config.nix.in rename to tests/functional/config.nix.in index 7facbdcbc98..00dc007e12f 100644 --- a/tests/config.nix.in +++ b/tests/functional/config.nix.in @@ -20,7 +20,10 @@ rec { derivation ({ inherit system; builder = shell; - args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; PATH = path; } // caArgs // removeAttrs args ["builder" "meta"]) // { meta = args.meta or {}; }; diff --git a/tests/config.sh b/tests/functional/config.sh similarity index 96% rename from tests/config.sh rename to tests/functional/config.sh index 723f575ed1d..0780c55d0bb 100644 --- a/tests/config.sh +++ b/tests/functional/config.sh @@ -50,7 +50,8 @@ exp_cores=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs) exp_features=$(nix show-config | grep '^experimental-features' | cut -d '=' -f 2 | xargs) [[ $prev != $exp_cores ]] [[ $exp_cores == "4242" ]] -[[ $exp_features == "flakes nix-command" ]] +# flakes implies fetch-tree +[[ $exp_features == "fetch-tree flakes nix-command" ]] # Test that it's possible to retrieve a single setting's value val=$(nix show-config | grep '^warn-dirty' | cut -d '=' -f 2 | xargs) diff --git a/tests/config/nix-with-substituters.conf b/tests/functional/config/nix-with-substituters.conf similarity index 100% rename from tests/config/nix-with-substituters.conf rename to tests/functional/config/nix-with-substituters.conf diff --git a/tests/db-migration.sh b/tests/functional/db-migration.sh similarity index 100% rename from tests/db-migration.sh rename to tests/functional/db-migration.sh diff --git a/tests/dependencies.builder0.sh b/tests/functional/dependencies.builder0.sh similarity index 100% rename from tests/dependencies.builder0.sh rename to tests/functional/dependencies.builder0.sh diff --git a/tests/dependencies.nix b/tests/functional/dependencies.nix similarity index 100% rename from tests/dependencies.nix rename to tests/functional/dependencies.nix diff --git a/tests/dependencies.sh b/tests/functional/dependencies.sh similarity index 100% rename from tests/dependencies.sh rename to tests/functional/dependencies.sh diff --git a/tests/derivation-json.sh b/tests/functional/derivation-json.sh similarity index 100% rename from tests/derivation-json.sh rename to tests/functional/derivation-json.sh diff --git a/tests/dummy b/tests/functional/dummy similarity index 100% rename from tests/dummy rename to tests/functional/dummy diff --git a/tests/dump-db.sh b/tests/functional/dump-db.sh similarity index 100% rename from tests/dump-db.sh rename to tests/functional/dump-db.sh diff --git a/tests/dyn-drv/build-built-drv.sh b/tests/functional/dyn-drv/build-built-drv.sh similarity index 83% rename from tests/dyn-drv/build-built-drv.sh rename to tests/functional/dyn-drv/build-built-drv.sh index 94f3550bdc3..647be945716 100644 --- a/tests/dyn-drv/build-built-drv.sh +++ b/tests/functional/dyn-drv/build-built-drv.sh @@ -18,6 +18,4 @@ clearStore drvDep=$(nix-instantiate ./text-hashed-output.nix -A producingDrv) -out2=$(nix build "${drvDep}^out^out" --no-link) - -test $out1 == $out2 +expectStderr 1 nix build "${drvDep}^out^out" --no-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented" diff --git a/tests/dyn-drv/common.sh b/tests/functional/dyn-drv/common.sh similarity index 100% rename from tests/dyn-drv/common.sh rename to tests/functional/dyn-drv/common.sh diff --git a/tests/dyn-drv/config.nix.in b/tests/functional/dyn-drv/config.nix.in similarity index 100% rename from tests/dyn-drv/config.nix.in rename to tests/functional/dyn-drv/config.nix.in diff --git a/tests/functional/dyn-drv/dep-built-drv.sh b/tests/functional/dyn-drv/dep-built-drv.sh new file mode 100644 index 00000000000..4f6e9b080fa --- /dev/null +++ b/tests/functional/dyn-drv/dep-built-drv.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +source common.sh + +out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link) + +clearStore + +expectStderr 1 nix-build ./text-hashed-output.nix -A wrapper --no-out-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented" + +# diff -r $out1 $out2 diff --git a/tests/dyn-drv/eval-outputOf.sh b/tests/functional/dyn-drv/eval-outputOf.sh similarity index 100% rename from tests/dyn-drv/eval-outputOf.sh rename to tests/functional/dyn-drv/eval-outputOf.sh diff --git a/tests/dyn-drv/local.mk b/tests/functional/dyn-drv/local.mk similarity index 87% rename from tests/dyn-drv/local.mk rename to tests/functional/dyn-drv/local.mk index 6b435499b78..c87534944b1 100644 --- a/tests/dyn-drv/local.mk +++ b/tests/functional/dyn-drv/local.mk @@ -12,4 +12,4 @@ clean-files += \ $(d)/config.nix test-deps += \ - tests/dyn-drv/config.nix + tests/functional/dyn-drv/config.nix diff --git a/tests/dyn-drv/old-daemon-error-hack.nix b/tests/functional/dyn-drv/old-daemon-error-hack.nix similarity index 100% rename from tests/dyn-drv/old-daemon-error-hack.nix rename to tests/functional/dyn-drv/old-daemon-error-hack.nix diff --git a/tests/dyn-drv/old-daemon-error-hack.sh b/tests/functional/dyn-drv/old-daemon-error-hack.sh similarity index 100% rename from tests/dyn-drv/old-daemon-error-hack.sh rename to tests/functional/dyn-drv/old-daemon-error-hack.sh diff --git a/tests/dyn-drv/recursive-mod-json.nix b/tests/functional/dyn-drv/recursive-mod-json.nix similarity index 100% rename from tests/dyn-drv/recursive-mod-json.nix rename to tests/functional/dyn-drv/recursive-mod-json.nix diff --git a/tests/dyn-drv/recursive-mod-json.sh b/tests/functional/dyn-drv/recursive-mod-json.sh similarity index 100% rename from tests/dyn-drv/recursive-mod-json.sh rename to tests/functional/dyn-drv/recursive-mod-json.sh diff --git a/tests/dyn-drv/text-hashed-output.nix b/tests/functional/dyn-drv/text-hashed-output.nix similarity index 100% rename from tests/dyn-drv/text-hashed-output.nix rename to tests/functional/dyn-drv/text-hashed-output.nix diff --git a/tests/dyn-drv/text-hashed-output.sh b/tests/functional/dyn-drv/text-hashed-output.sh similarity index 100% rename from tests/dyn-drv/text-hashed-output.sh rename to tests/functional/dyn-drv/text-hashed-output.sh diff --git a/tests/eval-store.sh b/tests/functional/eval-store.sh similarity index 100% rename from tests/eval-store.sh rename to tests/functional/eval-store.sh diff --git a/tests/eval.nix b/tests/functional/eval.nix similarity index 100% rename from tests/eval.nix rename to tests/functional/eval.nix diff --git a/tests/eval.sh b/tests/functional/eval.sh similarity index 100% rename from tests/eval.sh rename to tests/functional/eval.sh diff --git a/tests/experimental-features.sh b/tests/functional/experimental-features.sh similarity index 100% rename from tests/experimental-features.sh rename to tests/functional/experimental-features.sh diff --git a/tests/export-graph.nix b/tests/functional/export-graph.nix similarity index 100% rename from tests/export-graph.nix rename to tests/functional/export-graph.nix diff --git a/tests/export-graph.sh b/tests/functional/export-graph.sh similarity index 100% rename from tests/export-graph.sh rename to tests/functional/export-graph.sh diff --git a/tests/export.sh b/tests/functional/export.sh similarity index 100% rename from tests/export.sh rename to tests/functional/export.sh diff --git a/tests/failing.nix b/tests/functional/failing.nix similarity index 74% rename from tests/failing.nix rename to tests/functional/failing.nix index 2a0350d4d25..d25e2d6b62b 100644 --- a/tests/failing.nix +++ b/tests/functional/failing.nix @@ -6,7 +6,10 @@ let derivation ({ inherit system; builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; } // removeAttrs args ["builder" "meta"]) // { meta = args.meta or {}; }; in diff --git a/tests/fetchClosure.sh b/tests/functional/fetchClosure.sh similarity index 100% rename from tests/fetchClosure.sh rename to tests/functional/fetchClosure.sh diff --git a/tests/fetchGit.sh b/tests/functional/fetchGit.sh similarity index 98% rename from tests/fetchGit.sh rename to tests/functional/fetchGit.sh index 418b4f63fc2..fc89f2040ce 100644 --- a/tests/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -35,6 +35,8 @@ unset _NIX_FORCE_HTTP path0=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath") path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$TEST_ROOT/worktree; }).outPath") [[ $path0 = $path0_ ]] +path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree git+file://$TEST_ROOT/worktree).outPath") +[[ $path0 = $path0_ ]] export _NIX_FORCE_HTTP=1 [[ $(tail -n 1 $path0/hello) = "hello" ]] diff --git a/tests/fetchGitRefs.sh b/tests/functional/fetchGitRefs.sh similarity index 100% rename from tests/fetchGitRefs.sh rename to tests/functional/fetchGitRefs.sh diff --git a/tests/fetchGitSubmodules.sh b/tests/functional/fetchGitSubmodules.sh similarity index 100% rename from tests/fetchGitSubmodules.sh rename to tests/functional/fetchGitSubmodules.sh diff --git a/tests/functional/fetchGitVerification.sh b/tests/functional/fetchGitVerification.sh new file mode 100644 index 00000000000..4d920949868 --- /dev/null +++ b/tests/functional/fetchGitVerification.sh @@ -0,0 +1,76 @@ +source common.sh + +requireGit +[[ $(type -p ssh-keygen) ]] || skipTest "ssh-keygen not installed" # require ssh-keygen + +enableFeatures "verified-fetches" + +clearStore + +repo="$TEST_ROOT/git" + +# generate signing keys +keysDir=$TEST_ROOT/.ssh +mkdir -p "$keysDir" +ssh-keygen -f "$keysDir/testkey1" -t ed25519 -P "" -C "test key 1" +key1File="$keysDir/testkey1.pub" +publicKey1=$(awk '{print $2}' "$key1File") +ssh-keygen -f "$keysDir/testkey2" -t rsa -P "" -C "test key 2" +key2File="$keysDir/testkey2.pub" +publicKey2=$(awk '{print $2}' "$key2File") + +git init $repo +git -C $repo config user.email "foobar@example.com" +git -C $repo config user.name "Foobar" +git -C $repo config gpg.format ssh + +echo 'hello' > $repo/text +git -C $repo add text +git -C $repo -c "user.signingkey=$key1File" commit -S -m 'initial commit' + +out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = \"file://$repo\"; keytype = \"ssh-rsa\"; publicKey = \"$publicKey2\"; }" 2>&1) || status=$? +[[ $status == 1 ]] +[[ $out =~ 'No principal matched.' ]] +[[ $(nix eval --impure --raw --expr "builtins.readFile (builtins.fetchGit { url = \"file://$repo\"; publicKey = \"$publicKey1\"; } + \"/text\")") = 'hello' ]] + +echo 'hello world' > $repo/text +git -C $repo add text +git -C $repo -c "user.signingkey=$key2File" commit -S -m 'second commit' + +[[ $(nix eval --impure --raw --expr "builtins.readFile (builtins.fetchGit { url = \"file://$repo\"; publicKeys = [{key = \"$publicKey1\";} {type = \"ssh-rsa\"; key = \"$publicKey2\";}]; } + \"/text\")") = 'hello world' ]] + +# Flake input test +flakeDir="$TEST_ROOT/flake" +mkdir -p "$flakeDir" +cat > "$flakeDir/flake.nix" < "$flakeDir/flake.nix" <&1) || status=$? +[[ $status == 1 ]] +[[ $out =~ 'No principal matched.' ]] \ No newline at end of file diff --git a/tests/fetchMercurial.sh b/tests/functional/fetchMercurial.sh similarity index 100% rename from tests/fetchMercurial.sh rename to tests/functional/fetchMercurial.sh diff --git a/tests/fetchPath.sh b/tests/functional/fetchPath.sh similarity index 100% rename from tests/fetchPath.sh rename to tests/functional/fetchPath.sh diff --git a/tests/fetchTree-file.sh b/tests/functional/fetchTree-file.sh similarity index 100% rename from tests/fetchTree-file.sh rename to tests/functional/fetchTree-file.sh diff --git a/tests/fetchurl.sh b/tests/functional/fetchurl.sh similarity index 100% rename from tests/fetchurl.sh rename to tests/functional/fetchurl.sh diff --git a/tests/filter-source.nix b/tests/functional/filter-source.nix similarity index 100% rename from tests/filter-source.nix rename to tests/functional/filter-source.nix diff --git a/tests/filter-source.sh b/tests/functional/filter-source.sh similarity index 100% rename from tests/filter-source.sh rename to tests/functional/filter-source.sh diff --git a/tests/fixed.builder1.sh b/tests/functional/fixed.builder1.sh similarity index 100% rename from tests/fixed.builder1.sh rename to tests/functional/fixed.builder1.sh diff --git a/tests/fixed.builder2.sh b/tests/functional/fixed.builder2.sh similarity index 100% rename from tests/fixed.builder2.sh rename to tests/functional/fixed.builder2.sh diff --git a/tests/fixed.nix b/tests/functional/fixed.nix similarity index 100% rename from tests/fixed.nix rename to tests/functional/fixed.nix diff --git a/tests/fixed.sh b/tests/functional/fixed.sh similarity index 100% rename from tests/fixed.sh rename to tests/functional/fixed.sh diff --git a/tests/functional/flakes/absolute-attr-paths.sh b/tests/functional/flakes/absolute-attr-paths.sh new file mode 100644 index 00000000000..491adceb72e --- /dev/null +++ b/tests/functional/flakes/absolute-attr-paths.sh @@ -0,0 +1,17 @@ +source ./common.sh + +flake1Dir=$TEST_ROOT/flake1 + +mkdir -p $flake1Dir +cat > $flake1Dir/flake.nix < "$flake2Dir/flake.nix" < "$flake3Dir/flake.nix" < "$flake3Dir/default.nix" < "$nonFlakeDir/README.md" < "$nonFlakeDir/shebang.sh" < $nonFlakeDir/shebang-comments.sh < $nonFlakeDir/shebang-reject.sh < $nonFlakeDir/shebang-inline-expr.sh <> $nonFlakeDir/shebang-inline-expr.sh <<"EOF" +#! nix --offline shell +#! nix --impure --expr `` +#! nix let flake = (builtins.getFlake (toString ../flake1)).packages; +#! nix fooScript = flake.${builtins.currentSystem}.fooScript; +#! nix /* just a comment !@#$%^&*()__+ # */ +#! nix in fooScript +#! nix `` +#! nix --no-write-lock-file --command bash +set -ex +foo +echo "$@" +EOF +chmod +x $nonFlakeDir/shebang-inline-expr.sh + +cat > $nonFlakeDir/fooScript.nix <<"EOF" +let flake = (builtins.getFlake (toString ../flake1)).packages; + fooScript = flake.${builtins.currentSystem}.fooScript; + in fooScript +EOF + +cat > $nonFlakeDir/shebang-file.sh <> $nonFlakeDir/shebang-file.sh <<"EOF" +#! nix --offline shell +#! nix --impure --file ./fooScript.nix +#! nix --no-write-lock-file --command bash +set -ex +foo +echo "$@" +EOF +chmod +x $nonFlakeDir/shebang-file.sh + +# Construct a custom registry, additionally test the --registry flag +nix registry add --registry "$registry" flake1 "git+file://$flake1Dir" +nix registry add --registry "$registry" flake2 "git+file://$percentEncodedFlake2Dir" +nix registry add --registry "$registry" flake3 "git+file://$percentEncodedFlake3Dir" +nix registry add --registry "$registry" flake4 flake3 +nix registry add --registry "$registry" nixpkgs flake1 + +# Test 'nix registry list'. +[[ $(nix registry list | wc -l) == 5 ]] +nix registry list | grep '^global' +nix registry list | grepInverse '^user' # nothing in user registry + +# Test 'nix flake metadata'. +nix flake metadata flake1 +nix flake metadata flake1 | grepQuiet 'Locked URL:.*flake1.*' + +# Test 'nix flake metadata' on a local flake. +(cd "$flake1Dir" && nix flake metadata) | grepQuiet 'URL:.*flake1.*' +(cd "$flake1Dir" && nix flake metadata .) | grepQuiet 'URL:.*flake1.*' +nix flake metadata "$flake1Dir" | grepQuiet 'URL:.*flake1.*' + +# Test 'nix flake metadata --json'. +json=$(nix flake metadata flake1 --json | jq .) +[[ $(echo "$json" | jq -r .description) = 'Bla bla' ]] +[[ -d $(echo "$json" | jq -r .path) ]] +[[ $(echo "$json" | jq -r .lastModified) = $(git -C "$flake1Dir" log -n1 --format=%ct) ]] +hash1=$(echo "$json" | jq -r .revision) + +echo foo > "$flake1Dir/foo" +git -C "$flake1Dir" add $flake1Dir/foo +[[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "$hash1-dirty" ]] + +echo -n '# foo' >> "$flake1Dir/flake.nix" +flake1OriginalCommit=$(git -C "$flake1Dir" rev-parse HEAD) +git -C "$flake1Dir" commit -a -m 'Foo' +flake1NewCommit=$(git -C "$flake1Dir" rev-parse HEAD) +hash2=$(nix flake metadata flake1 --json --refresh | jq -r .revision) +[[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "null" ]] +[[ $hash1 != $hash2 ]] + +# Test 'nix build' on a flake. +nix build -o "$TEST_ROOT/result" flake1#foo +[[ -e "$TEST_ROOT/result/hello" ]] + +# Test packages.default. +nix build -o "$TEST_ROOT/result" flake1 +[[ -e "$TEST_ROOT/result/hello" ]] + +nix build -o "$TEST_ROOT/result" "$flake1Dir" +nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir" + +# Test explicit packages.default. +nix build -o "$TEST_ROOT/result" "$flake1Dir#default" +nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir#default" + +# Test explicit packages.default with query. +nix build -o "$TEST_ROOT/result" "$flake1Dir?ref=HEAD#default" +nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" + +# Check that store symlinks inside a flake are not interpreted as flakes. +nix build -o "$flake1Dir/result" "git+file://$flake1Dir" +nix path-info "$flake1Dir/result" + +# 'getFlake' on an unlocked flakeref should fail in pure mode, but +# succeed in impure mode. +(! nix build -o "$TEST_ROOT/result" --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default") +nix build -o "$TEST_ROOT/result" --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default" --impure + +# 'getFlake' on a locked flakeref should succeed even in pure mode. +nix build -o "$TEST_ROOT/result" --expr "(builtins.getFlake \"git+file://$flake1Dir?rev=$hash2\").packages.$system.default" + +# Building a flake with an unlocked dependency should fail in pure mode. +(! nix build -o "$TEST_ROOT/result" flake2#bar --no-registries) +(! nix build -o "$TEST_ROOT/result" flake2#bar --no-use-registries) +(! nix eval --expr "builtins.getFlake \"$flake2Dir\"") + +# But should succeed in impure mode. +(! nix build -o "$TEST_ROOT/result" flake2#bar --impure) +nix build -o "$TEST_ROOT/result" flake2#bar --impure --no-write-lock-file +nix eval --expr "builtins.getFlake \"$flake2Dir\"" --impure + +# Building a local flake with an unlocked dependency should fail with --no-update-lock-file. +expect 1 nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-update-lock-file 2>&1 | grep 'requires lock file changes' + +# But it should succeed without that flag. +nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-write-lock-file +expect 1 nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-update-lock-file 2>&1 | grep 'requires lock file changes' +nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --commit-lock-file +[[ -e "$flake2Dir/flake.lock" ]] +[[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] + +# Rerunning the build should not change the lockfile. +nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" +[[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] + +# Building with a lockfile should not require a fetch of the registry. +nix build -o "$TEST_ROOT/result" --flake-registry file:///no-registry.json "$flake2Dir#bar" --refresh +nix build -o "$TEST_ROOT/result" --no-registries "$flake2Dir#bar" --refresh +nix build -o "$TEST_ROOT/result" --no-use-registries "$flake2Dir#bar" --refresh + +# Updating the flake should not change the lockfile. +nix flake lock "$flake2Dir" +[[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] + +# Now we should be able to build the flake in pure mode. +nix build -o "$TEST_ROOT/result" flake2#bar + +# Or without a registry. +nix build -o "$TEST_ROOT/result" --no-registries "git+file://$percentEncodedFlake2Dir#bar" --refresh +nix build -o "$TEST_ROOT/result" --no-use-registries "git+file://$percentEncodedFlake2Dir#bar" --refresh + +# Test whether indirect dependencies work. +nix build -o "$TEST_ROOT/result" "$flake3Dir#xyzzy" +git -C "$flake3Dir" add flake.lock + +# Add dependency to flake3. +rm "$flake3Dir/flake.nix" + +cat > "$flake3Dir/flake.nix" < "$flake3Dir/flake.nix" < \$out + [[ \$(cat \${inputs.nonFlake}/README.md) = \$(cat \${inputs.nonFlakeFile}) ]] + [[ \${inputs.nonFlakeFile} = \${inputs.nonFlakeFile2} ]] + ''; + }; + }; +} +EOF + +cp ../config.nix "$flake3Dir" + +git -C "$flake3Dir" add flake.nix config.nix +git -C "$flake3Dir" commit -m 'Add nonFlakeInputs' + +# Check whether `nix build` works with a lockfile which is missing a +# nonFlakeInputs. +nix build -o "$TEST_ROOT/result" "$flake3Dir#sth" --commit-lock-file + +nix build -o "$TEST_ROOT/result" flake3#fnord +[[ $(cat $TEST_ROOT/result) = FNORD ]] + +# Check whether flake input fetching is lazy: flake3#sth does not +# depend on flake2, so this shouldn't fail. +rm -rf "$TEST_HOME/.cache" +clearStore +mv "$flake2Dir" "$flake2Dir.tmp" +mv "$nonFlakeDir" "$nonFlakeDir.tmp" +nix build -o "$TEST_ROOT/result" flake3#sth +(! nix build -o "$TEST_ROOT/result" flake3#xyzzy) +(! nix build -o "$TEST_ROOT/result" flake3#fnord) +mv "$flake2Dir.tmp" "$flake2Dir" +mv "$nonFlakeDir.tmp" "$nonFlakeDir" +nix build -o "$TEST_ROOT/result" flake3#xyzzy flake3#fnord + +# Test doing multiple `lookupFlake`s +nix build -o "$TEST_ROOT/result" flake4#xyzzy + +# Test 'nix flake update' and --override-flake. +nix flake lock "$flake3Dir" +[[ -z $(git -C "$flake3Dir" diff master || echo failed) ]] + +nix flake update --flake "$flake3Dir" --override-flake flake2 nixpkgs +[[ ! -z $(git -C "$flake3Dir" diff master || echo failed) ]] + +# Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore +git -C "$flake3Dir" checkout -b removeXyzzy +rm "$flake3Dir/flake.nix" + +cat > "$flake3Dir/flake.nix" < \$out + ''; + }; + }; +} +EOF +nix flake lock "$flake3Dir" +git -C "$flake3Dir" add flake.nix flake.lock +git -C "$flake3Dir" commit -m 'Remove packages.xyzzy' +git -C "$flake3Dir" checkout master + +# Test whether fuzzy-matching works for registry entries. +(! nix build -o "$TEST_ROOT/result" flake4/removeXyzzy#xyzzy) +nix build -o "$TEST_ROOT/result" flake4/removeXyzzy#sth + +# Testing the nix CLI +nix registry add flake1 flake3 +[[ $(nix registry list | wc -l) == 6 ]] +nix registry pin flake1 +[[ $(nix registry list | wc -l) == 6 ]] +nix registry pin flake1 flake3 +[[ $(nix registry list | wc -l) == 6 ]] +nix registry remove flake1 +[[ $(nix registry list | wc -l) == 5 ]] + +# Test 'nix registry list' with a disabled global registry. +nix registry add user-flake1 git+file://$flake1Dir +nix registry add user-flake2 "git+file://$percentEncodedFlake2Dir" +[[ $(nix --flake-registry "" registry list | wc -l) == 2 ]] +nix --flake-registry "" registry list | grepQuietInverse '^global' # nothing in global registry +nix --flake-registry "" registry list | grepQuiet '^user' +nix registry remove user-flake1 +nix registry remove user-flake2 +[[ $(nix registry list | wc -l) == 5 ]] + +# Test 'nix flake clone'. +rm -rf $TEST_ROOT/flake1-v2 +nix flake clone flake1 --dest $TEST_ROOT/flake1-v2 +[ -e $TEST_ROOT/flake1-v2/flake.nix ] + +# Test 'follows' inputs. +cat > "$flake3Dir/flake.nix" < "$flake3Dir/flake.nix" < "$flake3Dir/flake.nix" < "$flake3Dir/flake.nix" < "$flake3Dir/flake.nix" < $badFlakeDir/flake.nix +nix store delete $(nix store add-path $badFlakeDir) + +[[ $(nix path-info $(nix store add-path $flake1Dir)) =~ flake1 ]] +[[ $(nix path-info path:$(nix store add-path $flake1Dir)) =~ simple ]] + +# Test fetching flakerefs in the legacy CLI. +[[ $(nix-instantiate --eval flake:flake3 -A x) = 123 ]] +[[ $(nix-instantiate --eval "flake:git+file://$percentEncodedFlake3Dir" -A x) = 123 ]] +[[ $(nix-instantiate -I flake3=flake:flake3 --eval '' -A x) = 123 ]] +[[ $(NIX_PATH=flake3=flake:flake3 nix-instantiate --eval '' -A x) = 123 ]] + +# Test alternate lockfile paths. +nix flake lock "$flake2Dir" --output-lock-file $TEST_ROOT/flake2.lock +cmp "$flake2Dir/flake.lock" $TEST_ROOT/flake2.lock >/dev/null # lockfiles should be identical, since we're referencing flake2's original one + +nix flake lock "$flake2Dir" --output-lock-file $TEST_ROOT/flake2-overridden.lock --override-input flake1 git+file://$flake1Dir?rev=$flake1OriginalCommit +expectStderr 1 cmp "$flake2Dir/flake.lock" $TEST_ROOT/flake2-overridden.lock +nix flake metadata "$flake2Dir" --reference-lock-file $TEST_ROOT/flake2-overridden.lock | grepQuiet $flake1OriginalCommit + +# reference-lock-file can only be used if allow-dirty is set. +expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock-file $TEST_ROOT/flake2-overridden.lock + +# Test shebang +[[ $($nonFlakeDir/shebang.sh) = "foo" ]] +[[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]] +[[ $($nonFlakeDir/shebang-comments.sh ) = "foo" ]] +[[ $($nonFlakeDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]] +[[ $($nonFlakeDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]] +expect 1 $nonFlakeDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' diff --git a/tests/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh similarity index 87% rename from tests/flakes/follow-paths.sh rename to tests/functional/flakes/follow-paths.sh index dc97027aca3..9f23d738b25 100644 --- a/tests/flakes/follow-paths.sh +++ b/tests/functional/flakes/follow-paths.sh @@ -77,7 +77,7 @@ git -C $flakeFollowsA add flake.nix flakeB/flake.nix \ nix flake metadata $flakeFollowsA -nix flake update $flakeFollowsA +nix flake update --flake $flakeFollowsA nix flake lock $flakeFollowsA @@ -167,7 +167,7 @@ nix flake lock "$flakeFollowsA" 2>&1 | grep "warning: input 'B' has an override # # The message was # error: input 'B/D' follows a non-existent input 'B/C/D' -# +# # Note that for `B` to resolve its follow for `D`, it needs `C/D`, for which it needs to resolve the follow on `C` first. flakeFollowsOverloadA="$TEST_ROOT/follows/overload/flakeA" flakeFollowsOverloadB="$TEST_ROOT/follows/overload/flakeA/flakeB" @@ -228,5 +228,35 @@ git -C "$flakeFollowsOverloadA" add flake.nix flakeB/flake.nix \ flakeB/flakeC/flake.nix flakeB/flakeC/flakeD/flake.nix nix flake metadata "$flakeFollowsOverloadA" -nix flake update "$flakeFollowsOverloadA" +nix flake update --flake "$flakeFollowsOverloadA" nix flake lock "$flakeFollowsOverloadA" + +# Now test follow cycle detection +# We construct the following follows graph: +# +# foo +# / ^ +# / \ +# v \ +# bar -> baz +# The message was +# error: follow cycle detected: [baz -> foo -> bar -> baz] +flakeFollowCycle="$TEST_ROOT/follows/followCycle" + +# Test following path flakerefs. +mkdir -p "$flakeFollowCycle" + +cat > $flakeFollowCycle/flake.nix <&1 && fail "nix flake lock should have failed." || true) +echo $checkRes | grep -F "error: follow cycle detected: [baz -> foo -> bar -> baz]" diff --git a/tests/flakes/init.sh b/tests/functional/flakes/init.sh similarity index 100% rename from tests/flakes/init.sh rename to tests/functional/flakes/init.sh diff --git a/tests/flakes/inputs.sh b/tests/functional/flakes/inputs.sh similarity index 100% rename from tests/flakes/inputs.sh rename to tests/functional/flakes/inputs.sh diff --git a/tests/flakes/mercurial.sh b/tests/functional/flakes/mercurial.sh similarity index 100% rename from tests/flakes/mercurial.sh rename to tests/functional/flakes/mercurial.sh diff --git a/tests/flakes/run.sh b/tests/functional/flakes/run.sh similarity index 100% rename from tests/flakes/run.sh rename to tests/functional/flakes/run.sh diff --git a/tests/flakes/search-root.sh b/tests/functional/flakes/search-root.sh similarity index 100% rename from tests/flakes/search-root.sh rename to tests/functional/flakes/search-root.sh diff --git a/tests/flakes/show.sh b/tests/functional/flakes/show.sh similarity index 100% rename from tests/flakes/show.sh rename to tests/functional/flakes/show.sh diff --git a/tests/flakes/unlocked-override.sh b/tests/functional/flakes/unlocked-override.sh similarity index 100% rename from tests/flakes/unlocked-override.sh rename to tests/functional/flakes/unlocked-override.sh diff --git a/tests/fmt.sh b/tests/functional/fmt.sh similarity index 100% rename from tests/fmt.sh rename to tests/functional/fmt.sh diff --git a/tests/fmt.simple.sh b/tests/functional/fmt.simple.sh similarity index 100% rename from tests/fmt.simple.sh rename to tests/functional/fmt.simple.sh diff --git a/tests/function-trace.sh b/tests/functional/function-trace.sh similarity index 100% rename from tests/function-trace.sh rename to tests/functional/function-trace.sh diff --git a/tests/gc-auto.sh b/tests/functional/gc-auto.sh similarity index 100% rename from tests/gc-auto.sh rename to tests/functional/gc-auto.sh diff --git a/tests/gc-concurrent.builder.sh b/tests/functional/gc-concurrent.builder.sh similarity index 100% rename from tests/gc-concurrent.builder.sh rename to tests/functional/gc-concurrent.builder.sh diff --git a/tests/gc-concurrent.nix b/tests/functional/gc-concurrent.nix similarity index 100% rename from tests/gc-concurrent.nix rename to tests/functional/gc-concurrent.nix diff --git a/tests/gc-concurrent.sh b/tests/functional/gc-concurrent.sh similarity index 100% rename from tests/gc-concurrent.sh rename to tests/functional/gc-concurrent.sh diff --git a/tests/gc-concurrent2.builder.sh b/tests/functional/gc-concurrent2.builder.sh similarity index 100% rename from tests/gc-concurrent2.builder.sh rename to tests/functional/gc-concurrent2.builder.sh diff --git a/tests/gc-non-blocking.sh b/tests/functional/gc-non-blocking.sh similarity index 100% rename from tests/gc-non-blocking.sh rename to tests/functional/gc-non-blocking.sh diff --git a/tests/gc-runtime.nix b/tests/functional/gc-runtime.nix similarity index 100% rename from tests/gc-runtime.nix rename to tests/functional/gc-runtime.nix diff --git a/tests/gc-runtime.sh b/tests/functional/gc-runtime.sh similarity index 100% rename from tests/gc-runtime.sh rename to tests/functional/gc-runtime.sh diff --git a/tests/gc.sh b/tests/functional/gc.sh similarity index 100% rename from tests/gc.sh rename to tests/functional/gc.sh diff --git a/tests/hash-check.nix b/tests/functional/hash-check.nix similarity index 100% rename from tests/hash-check.nix rename to tests/functional/hash-check.nix diff --git a/tests/hash.sh b/tests/functional/hash.sh similarity index 100% rename from tests/hash.sh rename to tests/functional/hash.sh diff --git a/tests/hermetic.nix b/tests/functional/hermetic.nix similarity index 90% rename from tests/hermetic.nix rename to tests/functional/hermetic.nix index 4c9d7a51f54..0513540f01f 100644 --- a/tests/hermetic.nix +++ b/tests/functional/hermetic.nix @@ -14,7 +14,10 @@ let derivation ({ inherit system; builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; } // removeAttrs args ["builder" "meta" "passthru"] // caArgs) // { meta = args.meta or {}; passthru = args.passthru or {}; }; diff --git a/tests/ifd.nix b/tests/functional/ifd.nix similarity index 100% rename from tests/ifd.nix rename to tests/functional/ifd.nix diff --git a/tests/import-derivation.nix b/tests/functional/import-derivation.nix similarity index 100% rename from tests/import-derivation.nix rename to tests/functional/import-derivation.nix diff --git a/tests/import-derivation.sh b/tests/functional/import-derivation.sh similarity index 100% rename from tests/import-derivation.sh rename to tests/functional/import-derivation.sh diff --git a/tests/impure-derivations.nix b/tests/functional/impure-derivations.nix similarity index 100% rename from tests/impure-derivations.nix rename to tests/functional/impure-derivations.nix diff --git a/tests/impure-derivations.sh b/tests/functional/impure-derivations.sh similarity index 100% rename from tests/impure-derivations.sh rename to tests/functional/impure-derivations.sh diff --git a/tests/functional/impure-env.nix b/tests/functional/impure-env.nix new file mode 100644 index 00000000000..2b0380ed729 --- /dev/null +++ b/tests/functional/impure-env.nix @@ -0,0 +1,16 @@ +{ var, value }: + +with import ./config.nix; + +mkDerivation { + name = "test"; + buildCommand = '' + echo ${var} = "''$${var}" + echo -n "''$${var}" > "$out" + ''; + + impureEnvVars = [ var ]; + + outputHashAlgo = "sha256"; + outputHash = builtins.hashString "sha256" value; +} diff --git a/tests/functional/impure-env.sh b/tests/functional/impure-env.sh new file mode 100644 index 00000000000..d9e4a34a229 --- /dev/null +++ b/tests/functional/impure-env.sh @@ -0,0 +1,33 @@ +source common.sh + +# Needs the config option 'impure-env' to work +requireDaemonNewerThan "2.18.0pre20230816" + +enableFeatures "configurable-impure-env" +restartDaemon + +varTest() { + local var="$1"; shift + local value="$1"; shift + nix build --no-link -vL --argstr var "$var" --argstr value "$value" --impure "$@" --file impure-env.nix + clearStore +} + +clearStore +startDaemon + +varTest env_name value --impure-env env_name=value + +echo 'impure-env = set_in_config=config_value' >> "$NIX_CONF_DIR/nix.conf" +set_in_config=daemon_value restartDaemon + +varTest set_in_config config_value +varTest set_in_config client_value --impure-env set_in_config=client_value + +sed -i -e '/^trusted-users =/d' "$NIX_CONF_DIR/nix.conf" + +env_name=daemon_value restartDaemon + +varTest env_name daemon_value --impure-env env_name=client_value + +killDaemon diff --git a/tests/init.sh b/tests/functional/init.sh similarity index 100% rename from tests/init.sh rename to tests/functional/init.sh diff --git a/tests/install-darwin.sh b/tests/functional/install-darwin.sh similarity index 100% rename from tests/install-darwin.sh rename to tests/functional/install-darwin.sh diff --git a/tests/lang-test-infra.sh b/tests/functional/lang-test-infra.sh similarity index 100% rename from tests/lang-test-infra.sh rename to tests/functional/lang-test-infra.sh diff --git a/tests/lang.sh b/tests/functional/lang.sh similarity index 94% rename from tests/lang.sh rename to tests/functional/lang.sh index 75dbbc38e73..12df32c8770 100755 --- a/tests/lang.sh +++ b/tests/functional/lang.sh @@ -23,6 +23,7 @@ nix-instantiate --trace-verbose --eval -E 'builtins.traceVerbose "Hello" 123' 2> nix-instantiate --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grepQuietInverse Hello nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grepQuietInverse Hello expectStderr 1 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' | grepQuiet Hello +expectStderr 1 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello %" (throw "Foo")' | grepQuiet 'Hello %' nix-instantiate --eval -E 'let x = builtins.trace { x = x; } true; in x' \ 2>&1 | grepQuiet -E 'trace: { x = «potential infinite recursion»; }' @@ -68,7 +69,7 @@ for i in lang/eval-fail-*.nix; do echo "evaluating $i (should fail)"; i=$(basename "$i" .nix) if - expectStderr 1 nix-instantiate --show-trace "lang/$i.nix" \ + expectStderr 1 nix-instantiate --eval --strict --show-trace "lang/$i.nix" \ | sed "s!$(pwd)!/pwd!g" > "lang/$i.err" then diffAndAccept "$i" err err.exp @@ -134,7 +135,7 @@ else echo '' echo 'You can rerun this test with:' echo '' - echo ' _NIX_TEST_ACCEPT=1 make tests/lang.sh.test' + echo ' _NIX_TEST_ACCEPT=1 make tests/functional/lang.sh.test' echo '' echo 'to regenerate the files containing the expected output,' echo 'and then view the git diff to decide whether a change is' diff --git a/tests/lang/binary-data b/tests/functional/lang/binary-data similarity index 100% rename from tests/lang/binary-data rename to tests/functional/lang/binary-data diff --git a/tests/lang/data b/tests/functional/lang/data similarity index 100% rename from tests/lang/data rename to tests/functional/lang/data diff --git a/tests/lang/dir1/a.nix b/tests/functional/lang/dir1/a.nix similarity index 100% rename from tests/lang/dir1/a.nix rename to tests/functional/lang/dir1/a.nix diff --git a/tests/lang/dir2/a.nix b/tests/functional/lang/dir2/a.nix similarity index 100% rename from tests/lang/dir2/a.nix rename to tests/functional/lang/dir2/a.nix diff --git a/tests/lang/dir2/b.nix b/tests/functional/lang/dir2/b.nix similarity index 100% rename from tests/lang/dir2/b.nix rename to tests/functional/lang/dir2/b.nix diff --git a/tests/lang/dir3/a.nix b/tests/functional/lang/dir3/a.nix similarity index 100% rename from tests/lang/dir3/a.nix rename to tests/functional/lang/dir3/a.nix diff --git a/tests/lang/dir3/b.nix b/tests/functional/lang/dir3/b.nix similarity index 100% rename from tests/lang/dir3/b.nix rename to tests/functional/lang/dir3/b.nix diff --git a/tests/lang/dir3/c.nix b/tests/functional/lang/dir3/c.nix similarity index 100% rename from tests/lang/dir3/c.nix rename to tests/functional/lang/dir3/c.nix diff --git a/tests/lang/dir4/a.nix b/tests/functional/lang/dir4/a.nix similarity index 100% rename from tests/lang/dir4/a.nix rename to tests/functional/lang/dir4/a.nix diff --git a/tests/lang/dir4/c.nix b/tests/functional/lang/dir4/c.nix similarity index 100% rename from tests/lang/dir4/c.nix rename to tests/functional/lang/dir4/c.nix diff --git a/tests/lang/empty.exp b/tests/functional/lang/empty.exp similarity index 100% rename from tests/lang/empty.exp rename to tests/functional/lang/empty.exp diff --git a/tests/lang/eval-fail-abort.err.exp b/tests/functional/lang/eval-fail-abort.err.exp similarity index 100% rename from tests/lang/eval-fail-abort.err.exp rename to tests/functional/lang/eval-fail-abort.err.exp diff --git a/tests/lang/eval-fail-abort.nix b/tests/functional/lang/eval-fail-abort.nix similarity index 100% rename from tests/lang/eval-fail-abort.nix rename to tests/functional/lang/eval-fail-abort.nix diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp new file mode 100644 index 00000000000..ad91a22aa5b --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'addDrvOutputDependencies' builtin + + at /pwd/lang/eval-fail-addDrvOutputDependencies-empty-context.nix:1:1: + + 1| builtins.addDrvOutputDependencies "" + | ^ + 2| + + error: context of string '' must have exactly one element, but has 0 diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.nix b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.nix new file mode 100644 index 00000000000..dc9ee3ba2e5 --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.nix @@ -0,0 +1 @@ +builtins.addDrvOutputDependencies "" diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp new file mode 100644 index 00000000000..bb389db4e4c --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp @@ -0,0 +1,11 @@ +error: + … while calling the 'addDrvOutputDependencies' builtin + + at /pwd/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix:18:4: + + 17| + 18| in builtins.addDrvOutputDependencies combo-path + | ^ + 19| + + error: context of string '/nix/store/pg9yqs4yd85yhdm3f4i5dyaqp5jahrsz-fail.drv/nix/store/2dxd5frb715z451vbf7s8birlf3argbk-fail-2.drv' must have exactly one element, but has 2 diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix new file mode 100644 index 00000000000..dbde264dfae --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix @@ -0,0 +1,18 @@ +let + drv0 = derivation { + name = "fail"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + + drv1 = derivation { + name = "fail-2"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + + combo-path = "${drv0.drvPath}${drv1.drvPath}"; + +in builtins.addDrvOutputDependencies combo-path diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp new file mode 100644 index 00000000000..07038111822 --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp @@ -0,0 +1,11 @@ +error: + … while calling the 'addDrvOutputDependencies' builtin + + at /pwd/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix:9:4: + + 8| + 9| in builtins.addDrvOutputDependencies drv.outPath + | ^ + 10| + + error: `addDrvOutputDependencies` can only act on derivations, not on a derivation output such as 'out' diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix new file mode 100644 index 00000000000..e379e1d9598 --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix @@ -0,0 +1,9 @@ +let + drv = derivation { + name = "fail"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + +in builtins.addDrvOutputDependencies drv.outPath diff --git a/tests/lang/eval-fail-assert.err.exp b/tests/functional/lang/eval-fail-assert.err.exp similarity index 100% rename from tests/lang/eval-fail-assert.err.exp rename to tests/functional/lang/eval-fail-assert.err.exp diff --git a/tests/lang/eval-fail-assert.nix b/tests/functional/lang/eval-fail-assert.nix similarity index 100% rename from tests/lang/eval-fail-assert.nix rename to tests/functional/lang/eval-fail-assert.nix diff --git a/tests/lang/eval-fail-bad-string-interpolation-1.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp similarity index 100% rename from tests/lang/eval-fail-bad-string-interpolation-1.err.exp rename to tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp diff --git a/tests/lang/eval-fail-bad-string-interpolation-1.nix b/tests/functional/lang/eval-fail-bad-string-interpolation-1.nix similarity index 100% rename from tests/lang/eval-fail-bad-string-interpolation-1.nix rename to tests/functional/lang/eval-fail-bad-string-interpolation-1.nix diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp new file mode 100644 index 00000000000..a287067cd1c --- /dev/null +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp @@ -0,0 +1 @@ +error: path '/pwd/lang/fnord' does not exist diff --git a/tests/lang/eval-fail-bad-string-interpolation-2.nix b/tests/functional/lang/eval-fail-bad-string-interpolation-2.nix similarity index 100% rename from tests/lang/eval-fail-bad-string-interpolation-2.nix rename to tests/functional/lang/eval-fail-bad-string-interpolation-2.nix diff --git a/tests/lang/eval-fail-bad-string-interpolation-3.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp similarity index 100% rename from tests/lang/eval-fail-bad-string-interpolation-3.err.exp rename to tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp diff --git a/tests/lang/eval-fail-bad-string-interpolation-3.nix b/tests/functional/lang/eval-fail-bad-string-interpolation-3.nix similarity index 100% rename from tests/lang/eval-fail-bad-string-interpolation-3.nix rename to tests/functional/lang/eval-fail-bad-string-interpolation-3.nix diff --git a/tests/lang/eval-fail-blackhole.err.exp b/tests/functional/lang/eval-fail-blackhole.err.exp similarity index 100% rename from tests/lang/eval-fail-blackhole.err.exp rename to tests/functional/lang/eval-fail-blackhole.err.exp diff --git a/tests/lang/eval-fail-blackhole.nix b/tests/functional/lang/eval-fail-blackhole.nix similarity index 100% rename from tests/lang/eval-fail-blackhole.nix rename to tests/functional/lang/eval-fail-blackhole.nix diff --git a/tests/lang/eval-fail-deepseq.err.exp b/tests/functional/lang/eval-fail-deepseq.err.exp similarity index 100% rename from tests/lang/eval-fail-deepseq.err.exp rename to tests/functional/lang/eval-fail-deepseq.err.exp diff --git a/tests/lang/eval-fail-deepseq.nix b/tests/functional/lang/eval-fail-deepseq.nix similarity index 100% rename from tests/lang/eval-fail-deepseq.nix rename to tests/functional/lang/eval-fail-deepseq.nix diff --git a/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp b/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp new file mode 100644 index 00000000000..c5fa67523b9 --- /dev/null +++ b/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp @@ -0,0 +1,18 @@ +error: + … while evaluating the attribute 'set' + + at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:3: + + 1| { + 2| set = { "${"" + "b"}" = 1; }; + | ^ + 3| set = { "${"b" + ""}" = 2; }; + + error: dynamic attribute 'b' already defined at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:11 + + at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:3:11: + + 2| set = { "${"" + "b"}" = 1; }; + 3| set = { "${"b" + ""}" = 2; }; + | ^ + 4| } diff --git a/tests/lang/eval-fail-dup-dynamic-attrs.nix b/tests/functional/lang/eval-fail-dup-dynamic-attrs.nix similarity index 100% rename from tests/lang/eval-fail-dup-dynamic-attrs.nix rename to tests/functional/lang/eval-fail-dup-dynamic-attrs.nix diff --git a/tests/lang/eval-fail-foldlStrict-strict-op-application.err.exp b/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp similarity index 100% rename from tests/lang/eval-fail-foldlStrict-strict-op-application.err.exp rename to tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp diff --git a/tests/lang/eval-fail-foldlStrict-strict-op-application.nix b/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.nix similarity index 100% rename from tests/lang/eval-fail-foldlStrict-strict-op-application.nix rename to tests/functional/lang/eval-fail-foldlStrict-strict-op-application.nix diff --git a/tests/lang/eval-fail-fromTOML-timestamps.err.exp b/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp similarity index 85% rename from tests/lang/eval-fail-fromTOML-timestamps.err.exp rename to tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp index f6bd19f5a26..5b60d253de3 100644 --- a/tests/lang/eval-fail-fromTOML-timestamps.err.exp +++ b/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp @@ -8,5 +8,3 @@ error: 2| key = "value" error: while parsing a TOML string: Dates and times are not supported - - at «none»:0: (source not available) diff --git a/tests/lang/eval-fail-fromTOML-timestamps.nix b/tests/functional/lang/eval-fail-fromTOML-timestamps.nix similarity index 100% rename from tests/lang/eval-fail-fromTOML-timestamps.nix rename to tests/functional/lang/eval-fail-fromTOML-timestamps.nix diff --git a/tests/lang/eval-fail-hashfile-missing.err.exp b/tests/functional/lang/eval-fail-hashfile-missing.err.exp similarity index 84% rename from tests/lang/eval-fail-hashfile-missing.err.exp rename to tests/functional/lang/eval-fail-hashfile-missing.err.exp index 8e77dec1e50..6d38608c0c5 100644 --- a/tests/lang/eval-fail-hashfile-missing.err.exp +++ b/tests/functional/lang/eval-fail-hashfile-missing.err.exp @@ -10,10 +10,6 @@ error: … while evaluating the first argument passed to builtins.toString - at «none»:0: (source not available) - … while calling the 'hashFile' builtin - at «none»:0: (source not available) - error: opening file '/pwd/lang/this-file-is-definitely-not-there-7392097': No such file or directory diff --git a/tests/lang/eval-fail-hashfile-missing.nix b/tests/functional/lang/eval-fail-hashfile-missing.nix similarity index 100% rename from tests/lang/eval-fail-hashfile-missing.nix rename to tests/functional/lang/eval-fail-hashfile-missing.nix diff --git a/tests/lang/eval-fail-list.err.exp b/tests/functional/lang/eval-fail-list.err.exp similarity index 100% rename from tests/lang/eval-fail-list.err.exp rename to tests/functional/lang/eval-fail-list.err.exp diff --git a/tests/lang/eval-fail-list.nix b/tests/functional/lang/eval-fail-list.nix similarity index 100% rename from tests/lang/eval-fail-list.nix rename to tests/functional/lang/eval-fail-list.nix diff --git a/tests/lang/eval-fail-missing-arg.err.exp b/tests/functional/lang/eval-fail-missing-arg.err.exp similarity index 100% rename from tests/lang/eval-fail-missing-arg.err.exp rename to tests/functional/lang/eval-fail-missing-arg.err.exp diff --git a/tests/lang/eval-fail-missing-arg.nix b/tests/functional/lang/eval-fail-missing-arg.nix similarity index 100% rename from tests/lang/eval-fail-missing-arg.nix rename to tests/functional/lang/eval-fail-missing-arg.nix diff --git a/tests/functional/lang/eval-fail-nonexist-path.err.exp b/tests/functional/lang/eval-fail-nonexist-path.err.exp new file mode 100644 index 00000000000..a287067cd1c --- /dev/null +++ b/tests/functional/lang/eval-fail-nonexist-path.err.exp @@ -0,0 +1 @@ +error: path '/pwd/lang/fnord' does not exist diff --git a/tests/lang/eval-fail-nonexist-path.nix b/tests/functional/lang/eval-fail-nonexist-path.nix similarity index 100% rename from tests/lang/eval-fail-nonexist-path.nix rename to tests/functional/lang/eval-fail-nonexist-path.nix diff --git a/tests/lang/eval-fail-path-slash.err.exp b/tests/functional/lang/eval-fail-path-slash.err.exp similarity index 100% rename from tests/lang/eval-fail-path-slash.err.exp rename to tests/functional/lang/eval-fail-path-slash.err.exp diff --git a/tests/lang/eval-fail-path-slash.nix b/tests/functional/lang/eval-fail-path-slash.nix similarity index 100% rename from tests/lang/eval-fail-path-slash.nix rename to tests/functional/lang/eval-fail-path-slash.nix diff --git a/tests/lang/eval-fail-recursion.err.exp b/tests/functional/lang/eval-fail-recursion.err.exp similarity index 100% rename from tests/lang/eval-fail-recursion.err.exp rename to tests/functional/lang/eval-fail-recursion.err.exp diff --git a/tests/lang/eval-fail-recursion.nix b/tests/functional/lang/eval-fail-recursion.nix similarity index 100% rename from tests/lang/eval-fail-recursion.nix rename to tests/functional/lang/eval-fail-recursion.nix diff --git a/tests/lang/eval-fail-remove.err.exp b/tests/functional/lang/eval-fail-remove.err.exp similarity index 100% rename from tests/lang/eval-fail-remove.err.exp rename to tests/functional/lang/eval-fail-remove.err.exp diff --git a/tests/lang/eval-fail-remove.nix b/tests/functional/lang/eval-fail-remove.nix similarity index 100% rename from tests/lang/eval-fail-remove.nix rename to tests/functional/lang/eval-fail-remove.nix diff --git a/tests/lang/eval-fail-scope-5.err.exp b/tests/functional/lang/eval-fail-scope-5.err.exp similarity index 100% rename from tests/lang/eval-fail-scope-5.err.exp rename to tests/functional/lang/eval-fail-scope-5.err.exp diff --git a/tests/lang/eval-fail-scope-5.nix b/tests/functional/lang/eval-fail-scope-5.nix similarity index 100% rename from tests/lang/eval-fail-scope-5.nix rename to tests/functional/lang/eval-fail-scope-5.nix diff --git a/tests/lang/eval-fail-seq.err.exp b/tests/functional/lang/eval-fail-seq.err.exp similarity index 100% rename from tests/lang/eval-fail-seq.err.exp rename to tests/functional/lang/eval-fail-seq.err.exp diff --git a/tests/lang/eval-fail-seq.nix b/tests/functional/lang/eval-fail-seq.nix similarity index 100% rename from tests/lang/eval-fail-seq.nix rename to tests/functional/lang/eval-fail-seq.nix diff --git a/tests/lang/eval-fail-set-override.err.exp b/tests/functional/lang/eval-fail-set-override.err.exp similarity index 71% rename from tests/lang/eval-fail-set-override.err.exp rename to tests/functional/lang/eval-fail-set-override.err.exp index beb29d678c2..71481683db1 100644 --- a/tests/lang/eval-fail-set-override.err.exp +++ b/tests/functional/lang/eval-fail-set-override.err.exp @@ -1,6 +1,4 @@ error: … while evaluating the `__overrides` attribute - at «none»:0: (source not available) - error: value is an integer while a set was expected diff --git a/tests/lang/eval-fail-set-override.nix b/tests/functional/lang/eval-fail-set-override.nix similarity index 100% rename from tests/lang/eval-fail-set-override.nix rename to tests/functional/lang/eval-fail-set-override.nix diff --git a/tests/lang/eval-fail-set.err.exp b/tests/functional/lang/eval-fail-set.err.exp similarity index 100% rename from tests/lang/eval-fail-set.err.exp rename to tests/functional/lang/eval-fail-set.err.exp diff --git a/tests/lang/eval-fail-set.nix b/tests/functional/lang/eval-fail-set.nix similarity index 100% rename from tests/lang/eval-fail-set.nix rename to tests/functional/lang/eval-fail-set.nix diff --git a/tests/lang/eval-fail-substring.err.exp b/tests/functional/lang/eval-fail-substring.err.exp similarity index 84% rename from tests/lang/eval-fail-substring.err.exp rename to tests/functional/lang/eval-fail-substring.err.exp index dc26a00bdcc..5c58be29a66 100644 --- a/tests/lang/eval-fail-substring.err.exp +++ b/tests/functional/lang/eval-fail-substring.err.exp @@ -8,5 +8,3 @@ error: 2| error: negative start position in 'substring' - - at «none»:0: (source not available) diff --git a/tests/lang/eval-fail-substring.nix b/tests/functional/lang/eval-fail-substring.nix similarity index 100% rename from tests/lang/eval-fail-substring.nix rename to tests/functional/lang/eval-fail-substring.nix diff --git a/tests/lang/eval-fail-to-path.err.exp b/tests/functional/lang/eval-fail-to-path.err.exp similarity index 86% rename from tests/lang/eval-fail-to-path.err.exp rename to tests/functional/lang/eval-fail-to-path.err.exp index 43ed2bdfc4b..4ffa2cf6dbc 100644 --- a/tests/lang/eval-fail-to-path.err.exp +++ b/tests/functional/lang/eval-fail-to-path.err.exp @@ -9,6 +9,4 @@ error: … while evaluating the first argument passed to builtins.toPath - at «none»:0: (source not available) - error: string 'foo/bar' doesn't represent an absolute path diff --git a/tests/lang/eval-fail-to-path.nix b/tests/functional/lang/eval-fail-to-path.nix similarity index 100% rename from tests/lang/eval-fail-to-path.nix rename to tests/functional/lang/eval-fail-to-path.nix diff --git a/tests/lang/eval-fail-toJSON.err.exp b/tests/functional/lang/eval-fail-toJSON.err.exp similarity index 100% rename from tests/lang/eval-fail-toJSON.err.exp rename to tests/functional/lang/eval-fail-toJSON.err.exp diff --git a/tests/lang/eval-fail-toJSON.nix b/tests/functional/lang/eval-fail-toJSON.nix similarity index 100% rename from tests/lang/eval-fail-toJSON.nix rename to tests/functional/lang/eval-fail-toJSON.nix diff --git a/tests/lang/eval-fail-undeclared-arg.err.exp b/tests/functional/lang/eval-fail-undeclared-arg.err.exp similarity index 100% rename from tests/lang/eval-fail-undeclared-arg.err.exp rename to tests/functional/lang/eval-fail-undeclared-arg.err.exp diff --git a/tests/lang/eval-fail-undeclared-arg.nix b/tests/functional/lang/eval-fail-undeclared-arg.nix similarity index 100% rename from tests/lang/eval-fail-undeclared-arg.nix rename to tests/functional/lang/eval-fail-undeclared-arg.nix diff --git a/tests/lang/eval-okay-any-all.exp b/tests/functional/lang/eval-okay-any-all.exp similarity index 100% rename from tests/lang/eval-okay-any-all.exp rename to tests/functional/lang/eval-okay-any-all.exp diff --git a/tests/lang/eval-okay-any-all.nix b/tests/functional/lang/eval-okay-any-all.nix similarity index 100% rename from tests/lang/eval-okay-any-all.nix rename to tests/functional/lang/eval-okay-any-all.nix diff --git a/tests/lang/eval-okay-arithmetic.exp b/tests/functional/lang/eval-okay-arithmetic.exp similarity index 100% rename from tests/lang/eval-okay-arithmetic.exp rename to tests/functional/lang/eval-okay-arithmetic.exp diff --git a/tests/lang/eval-okay-arithmetic.nix b/tests/functional/lang/eval-okay-arithmetic.nix similarity index 100% rename from tests/lang/eval-okay-arithmetic.nix rename to tests/functional/lang/eval-okay-arithmetic.nix diff --git a/tests/lang/eval-okay-attrnames.exp b/tests/functional/lang/eval-okay-attrnames.exp similarity index 100% rename from tests/lang/eval-okay-attrnames.exp rename to tests/functional/lang/eval-okay-attrnames.exp diff --git a/tests/lang/eval-okay-attrnames.nix b/tests/functional/lang/eval-okay-attrnames.nix similarity index 100% rename from tests/lang/eval-okay-attrnames.nix rename to tests/functional/lang/eval-okay-attrnames.nix diff --git a/tests/lang/eval-okay-attrs.exp b/tests/functional/lang/eval-okay-attrs.exp similarity index 100% rename from tests/lang/eval-okay-attrs.exp rename to tests/functional/lang/eval-okay-attrs.exp diff --git a/tests/lang/eval-okay-attrs.nix b/tests/functional/lang/eval-okay-attrs.nix similarity index 100% rename from tests/lang/eval-okay-attrs.nix rename to tests/functional/lang/eval-okay-attrs.nix diff --git a/tests/lang/eval-okay-attrs2.exp b/tests/functional/lang/eval-okay-attrs2.exp similarity index 100% rename from tests/lang/eval-okay-attrs2.exp rename to tests/functional/lang/eval-okay-attrs2.exp diff --git a/tests/lang/eval-okay-attrs2.nix b/tests/functional/lang/eval-okay-attrs2.nix similarity index 100% rename from tests/lang/eval-okay-attrs2.nix rename to tests/functional/lang/eval-okay-attrs2.nix diff --git a/tests/lang/eval-okay-attrs3.exp b/tests/functional/lang/eval-okay-attrs3.exp similarity index 100% rename from tests/lang/eval-okay-attrs3.exp rename to tests/functional/lang/eval-okay-attrs3.exp diff --git a/tests/lang/eval-okay-attrs3.nix b/tests/functional/lang/eval-okay-attrs3.nix similarity index 100% rename from tests/lang/eval-okay-attrs3.nix rename to tests/functional/lang/eval-okay-attrs3.nix diff --git a/tests/lang/eval-okay-attrs4.exp b/tests/functional/lang/eval-okay-attrs4.exp similarity index 100% rename from tests/lang/eval-okay-attrs4.exp rename to tests/functional/lang/eval-okay-attrs4.exp diff --git a/tests/lang/eval-okay-attrs4.nix b/tests/functional/lang/eval-okay-attrs4.nix similarity index 100% rename from tests/lang/eval-okay-attrs4.nix rename to tests/functional/lang/eval-okay-attrs4.nix diff --git a/tests/lang/eval-okay-attrs5.exp b/tests/functional/lang/eval-okay-attrs5.exp similarity index 100% rename from tests/lang/eval-okay-attrs5.exp rename to tests/functional/lang/eval-okay-attrs5.exp diff --git a/tests/lang/eval-okay-attrs5.nix b/tests/functional/lang/eval-okay-attrs5.nix similarity index 100% rename from tests/lang/eval-okay-attrs5.nix rename to tests/functional/lang/eval-okay-attrs5.nix diff --git a/tests/lang/eval-okay-attrs6.exp b/tests/functional/lang/eval-okay-attrs6.exp similarity index 100% rename from tests/lang/eval-okay-attrs6.exp rename to tests/functional/lang/eval-okay-attrs6.exp diff --git a/tests/lang/eval-okay-attrs6.nix b/tests/functional/lang/eval-okay-attrs6.nix similarity index 100% rename from tests/lang/eval-okay-attrs6.nix rename to tests/functional/lang/eval-okay-attrs6.nix diff --git a/tests/lang/eval-okay-autoargs.exp b/tests/functional/lang/eval-okay-autoargs.exp similarity index 100% rename from tests/lang/eval-okay-autoargs.exp rename to tests/functional/lang/eval-okay-autoargs.exp diff --git a/tests/lang/eval-okay-autoargs.flags b/tests/functional/lang/eval-okay-autoargs.flags similarity index 100% rename from tests/lang/eval-okay-autoargs.flags rename to tests/functional/lang/eval-okay-autoargs.flags diff --git a/tests/lang/eval-okay-autoargs.nix b/tests/functional/lang/eval-okay-autoargs.nix similarity index 100% rename from tests/lang/eval-okay-autoargs.nix rename to tests/functional/lang/eval-okay-autoargs.nix diff --git a/tests/lang/eval-okay-backslash-newline-1.exp b/tests/functional/lang/eval-okay-backslash-newline-1.exp similarity index 100% rename from tests/lang/eval-okay-backslash-newline-1.exp rename to tests/functional/lang/eval-okay-backslash-newline-1.exp diff --git a/tests/lang/eval-okay-backslash-newline-1.nix b/tests/functional/lang/eval-okay-backslash-newline-1.nix similarity index 100% rename from tests/lang/eval-okay-backslash-newline-1.nix rename to tests/functional/lang/eval-okay-backslash-newline-1.nix diff --git a/tests/lang/eval-okay-backslash-newline-2.exp b/tests/functional/lang/eval-okay-backslash-newline-2.exp similarity index 100% rename from tests/lang/eval-okay-backslash-newline-2.exp rename to tests/functional/lang/eval-okay-backslash-newline-2.exp diff --git a/tests/lang/eval-okay-backslash-newline-2.nix b/tests/functional/lang/eval-okay-backslash-newline-2.nix similarity index 100% rename from tests/lang/eval-okay-backslash-newline-2.nix rename to tests/functional/lang/eval-okay-backslash-newline-2.nix diff --git a/tests/lang/eval-okay-builtins-add.exp b/tests/functional/lang/eval-okay-builtins-add.exp similarity index 100% rename from tests/lang/eval-okay-builtins-add.exp rename to tests/functional/lang/eval-okay-builtins-add.exp diff --git a/tests/lang/eval-okay-builtins-add.nix b/tests/functional/lang/eval-okay-builtins-add.nix similarity index 100% rename from tests/lang/eval-okay-builtins-add.nix rename to tests/functional/lang/eval-okay-builtins-add.nix diff --git a/tests/lang/eval-okay-builtins.exp b/tests/functional/lang/eval-okay-builtins.exp similarity index 100% rename from tests/lang/eval-okay-builtins.exp rename to tests/functional/lang/eval-okay-builtins.exp diff --git a/tests/lang/eval-okay-builtins.nix b/tests/functional/lang/eval-okay-builtins.nix similarity index 100% rename from tests/lang/eval-okay-builtins.nix rename to tests/functional/lang/eval-okay-builtins.nix diff --git a/tests/lang/eval-okay-callable-attrs.exp b/tests/functional/lang/eval-okay-callable-attrs.exp similarity index 100% rename from tests/lang/eval-okay-callable-attrs.exp rename to tests/functional/lang/eval-okay-callable-attrs.exp diff --git a/tests/lang/eval-okay-callable-attrs.nix b/tests/functional/lang/eval-okay-callable-attrs.nix similarity index 100% rename from tests/lang/eval-okay-callable-attrs.nix rename to tests/functional/lang/eval-okay-callable-attrs.nix diff --git a/tests/lang/eval-okay-catattrs.exp b/tests/functional/lang/eval-okay-catattrs.exp similarity index 100% rename from tests/lang/eval-okay-catattrs.exp rename to tests/functional/lang/eval-okay-catattrs.exp diff --git a/tests/lang/eval-okay-catattrs.nix b/tests/functional/lang/eval-okay-catattrs.nix similarity index 100% rename from tests/lang/eval-okay-catattrs.nix rename to tests/functional/lang/eval-okay-catattrs.nix diff --git a/tests/lang/eval-okay-closure.exp b/tests/functional/lang/eval-okay-closure.exp similarity index 100% rename from tests/lang/eval-okay-closure.exp rename to tests/functional/lang/eval-okay-closure.exp diff --git a/tests/lang/eval-okay-closure.exp.xml b/tests/functional/lang/eval-okay-closure.exp.xml similarity index 100% rename from tests/lang/eval-okay-closure.exp.xml rename to tests/functional/lang/eval-okay-closure.exp.xml diff --git a/tests/lang/eval-okay-closure.nix b/tests/functional/lang/eval-okay-closure.nix similarity index 100% rename from tests/lang/eval-okay-closure.nix rename to tests/functional/lang/eval-okay-closure.nix diff --git a/tests/lang/eval-okay-comments.exp b/tests/functional/lang/eval-okay-comments.exp similarity index 100% rename from tests/lang/eval-okay-comments.exp rename to tests/functional/lang/eval-okay-comments.exp diff --git a/tests/lang/eval-okay-comments.nix b/tests/functional/lang/eval-okay-comments.nix similarity index 100% rename from tests/lang/eval-okay-comments.nix rename to tests/functional/lang/eval-okay-comments.nix diff --git a/tests/lang/eval-okay-concat.exp b/tests/functional/lang/eval-okay-concat.exp similarity index 100% rename from tests/lang/eval-okay-concat.exp rename to tests/functional/lang/eval-okay-concat.exp diff --git a/tests/lang/eval-okay-concat.nix b/tests/functional/lang/eval-okay-concat.nix similarity index 100% rename from tests/lang/eval-okay-concat.nix rename to tests/functional/lang/eval-okay-concat.nix diff --git a/tests/lang/eval-okay-concatmap.exp b/tests/functional/lang/eval-okay-concatmap.exp similarity index 100% rename from tests/lang/eval-okay-concatmap.exp rename to tests/functional/lang/eval-okay-concatmap.exp diff --git a/tests/lang/eval-okay-concatmap.nix b/tests/functional/lang/eval-okay-concatmap.nix similarity index 100% rename from tests/lang/eval-okay-concatmap.nix rename to tests/functional/lang/eval-okay-concatmap.nix diff --git a/tests/lang/eval-okay-concatstringssep.exp b/tests/functional/lang/eval-okay-concatstringssep.exp similarity index 100% rename from tests/lang/eval-okay-concatstringssep.exp rename to tests/functional/lang/eval-okay-concatstringssep.exp diff --git a/tests/lang/eval-okay-concatstringssep.nix b/tests/functional/lang/eval-okay-concatstringssep.nix similarity index 100% rename from tests/lang/eval-okay-concatstringssep.nix rename to tests/functional/lang/eval-okay-concatstringssep.nix diff --git a/tests/functional/lang/eval-okay-context-introspection.exp b/tests/functional/lang/eval-okay-context-introspection.exp new file mode 100644 index 00000000000..a136b0035e0 --- /dev/null +++ b/tests/functional/lang/eval-okay-context-introspection.exp @@ -0,0 +1 @@ +[ true true true true true true true true true true true true true ] diff --git a/tests/lang/eval-okay-context-introspection.nix b/tests/functional/lang/eval-okay-context-introspection.nix similarity index 60% rename from tests/lang/eval-okay-context-introspection.nix rename to tests/functional/lang/eval-okay-context-introspection.nix index 50a78d946e7..8886cf32e94 100644 --- a/tests/lang/eval-okay-context-introspection.nix +++ b/tests/functional/lang/eval-okay-context-introspection.nix @@ -31,11 +31,29 @@ let (builtins.unsafeDiscardStringContext str) (builtins.getContext str); + # Only holds true if string context contains both a `DrvDeep` and + # `Opaque` element. + almostEtaRule = str: + str == builtins.addDrvOutputDependencies + (builtins.unsafeDiscardOutputDependency str); + + addDrvOutputDependencies_idempotent = str: + builtins.addDrvOutputDependencies str == + builtins.addDrvOutputDependencies (builtins.addDrvOutputDependencies str); + + rules = str: [ + (etaRule str) + (almostEtaRule str) + (addDrvOutputDependencies_idempotent str) + ]; + in [ (legit-context == desired-context) (reconstructed-path == combo-path) (etaRule "foo") - (etaRule drv.drvPath) (etaRule drv.foo.outPath) - (etaRule (builtins.unsafeDiscardOutputDependency drv.drvPath)) +] ++ builtins.concatMap rules [ + drv.drvPath + (builtins.addDrvOutputDependencies drv.drvPath) + (builtins.unsafeDiscardOutputDependency drv.drvPath) ] diff --git a/tests/lang/eval-okay-context.exp b/tests/functional/lang/eval-okay-context.exp similarity index 100% rename from tests/lang/eval-okay-context.exp rename to tests/functional/lang/eval-okay-context.exp diff --git a/tests/lang/eval-okay-context.nix b/tests/functional/lang/eval-okay-context.nix similarity index 100% rename from tests/lang/eval-okay-context.nix rename to tests/functional/lang/eval-okay-context.nix diff --git a/tests/functional/lang/eval-okay-convertHash.exp b/tests/functional/lang/eval-okay-convertHash.exp new file mode 100644 index 00000000000..60e0a3c494a --- /dev/null +++ b/tests/functional/lang/eval-okay-convertHash.exp @@ -0,0 +1 @@ +{ hashesBase16 = [ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ]; hashesBase32 = [ "3y8bwfr609h3lh9ch0izcqq7fl" "26mrvc0v1nslch8r0w45zywsbc" "1v4gi57l97pmnylq6lmgxkhd5v" "143xibwh31h9bvxzalr0sjvbbvpa6ffs" "i4hj30pkrfdpgc5dbcgcydqviibfhm6d" "fxz2p030yba2bza71qhss79k3l5y24kd" "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73" "0qy6iz9yh6a079757mxdmypx0gcmnzjd3ij5q78bzk00vxll82lh" "0mkygpci4r4yb8zz5rs2kxcgvw0a2yf5zlj6r8qgfll6pnrqf0xd" "0zdl9zrg8r3i9c1g90lgg9ip5ijzv3yhz91i0zzn3r8ap9ws784gkp9dk9j3aglhgf1amqb0pj21mh7h1nxcl18akqvvf7ggqsy30yg" "19ncrpp37dx0nzzjw4k6zaqkb9mzaq2myhgpzh5aff7qqcj5wwdxslg6ixwncm7gyq8l761gwf87fgsh2bwfyr52s53k2dkqvw8c24x" "2kz74snvckxldmmbisz9ikmy031d28cs6xfdbl6rhxx42glpyz4vww4lajrc5akklxwixl0js4g84233pxvmbykiic5m7i5m9r4nr11" ]; hashesBase64 = [ "1B2M2Y8AsgTpgAmY7PhCfg==" "bGnufyEcZAQZ1TZswHauRg==" "uzQ4+6vUYOptvSfRU+IjOw==" "2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "zVToVowbN88eW62wd5vL84IhIYk=" "bRLhCx0zHa0hDkf9JdTyYIArfnc=" "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; hashesSRI = [ "md5-1B2M2Y8AsgTpgAmY7PhCfg==" "md5-bGnufyEcZAQZ1TZswHauRg==" "md5-uzQ4+6vUYOptvSfRU+IjOw==" "sha1-2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "sha1-zVToVowbN88eW62wd5vL84IhIYk=" "sha1-bRLhCx0zHa0hDkf9JdTyYIArfnc=" "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "sha256-kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "sha256-rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "sha512-nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "sha512-IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; } diff --git a/tests/functional/lang/eval-okay-convertHash.nix b/tests/functional/lang/eval-okay-convertHash.nix new file mode 100644 index 00000000000..cf4909aaf01 --- /dev/null +++ b/tests/functional/lang/eval-okay-convertHash.nix @@ -0,0 +1,31 @@ +let + hashAlgos = [ "md5" "md5" "md5" "sha1" "sha1" "sha1" "sha256" "sha256" "sha256" "sha512" "sha512" "sha512" ]; + hashesBase16 = import ./eval-okay-hashstring.exp; + map2 = f: { fsts, snds }: if fsts == [ ] then [ ] else [ (f (builtins.head fsts) (builtins.head snds)) ] ++ map2 f { fsts = builtins.tail fsts; snds = builtins.tail snds; }; + map2' = f: fsts: snds: map2 f { inherit fsts snds; }; + getOutputHashes = hashes: { + hashesBase16 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base16";}) hashAlgos hashes; + hashesBase32 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";}) hashAlgos hashes; + hashesBase64 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base64";}) hashAlgos hashes; + hashesSRI = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "sri" ;}) hashAlgos hashes; + }; + getOutputHashesColon = hashes: { + hashesBase16 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base16";}) hashAlgos hashes; + hashesBase32 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base32";}) hashAlgos hashes; + hashesBase64 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base64";}) hashAlgos hashes; + hashesSRI = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "sri" ;}) hashAlgos hashes; + }; + outputHashes = getOutputHashes hashesBase16; +in +# map2'` +assert map2' (s1: s2: s1 + s2) [ "a" "b" ] [ "c" "d" ] == [ "ac" "bd" ]; +# hashesBase16 +assert outputHashes.hashesBase16 == hashesBase16; +# standard SRI hashes +assert outputHashes.hashesSRI == (map2' (hashAlgo: hashBody: hashAlgo + "-" + hashBody) hashAlgos outputHashes.hashesBase64); +# without prefix +assert builtins.all (x: getOutputHashes x == outputHashes) (builtins.attrValues outputHashes); +# colon-separated. +# Note that colon prefix must not be applied to the standard SRI. e.g. "sha256:sha256-..." is illegal. +assert builtins.all (x: getOutputHashesColon x == outputHashes) (with outputHashes; [ hashesBase16 hashesBase32 hashesBase64 ]); +outputHashes diff --git a/tests/lang/eval-okay-curpos.exp b/tests/functional/lang/eval-okay-curpos.exp similarity index 100% rename from tests/lang/eval-okay-curpos.exp rename to tests/functional/lang/eval-okay-curpos.exp diff --git a/tests/lang/eval-okay-curpos.nix b/tests/functional/lang/eval-okay-curpos.nix similarity index 100% rename from tests/lang/eval-okay-curpos.nix rename to tests/functional/lang/eval-okay-curpos.nix diff --git a/tests/lang/eval-okay-deepseq.exp b/tests/functional/lang/eval-okay-deepseq.exp similarity index 100% rename from tests/lang/eval-okay-deepseq.exp rename to tests/functional/lang/eval-okay-deepseq.exp diff --git a/tests/lang/eval-okay-deepseq.nix b/tests/functional/lang/eval-okay-deepseq.nix similarity index 100% rename from tests/lang/eval-okay-deepseq.nix rename to tests/functional/lang/eval-okay-deepseq.nix diff --git a/tests/lang/eval-okay-delayed-with-inherit.exp b/tests/functional/lang/eval-okay-delayed-with-inherit.exp similarity index 100% rename from tests/lang/eval-okay-delayed-with-inherit.exp rename to tests/functional/lang/eval-okay-delayed-with-inherit.exp diff --git a/tests/lang/eval-okay-delayed-with-inherit.nix b/tests/functional/lang/eval-okay-delayed-with-inherit.nix similarity index 100% rename from tests/lang/eval-okay-delayed-with-inherit.nix rename to tests/functional/lang/eval-okay-delayed-with-inherit.nix diff --git a/tests/lang/eval-okay-delayed-with.exp b/tests/functional/lang/eval-okay-delayed-with.exp similarity index 100% rename from tests/lang/eval-okay-delayed-with.exp rename to tests/functional/lang/eval-okay-delayed-with.exp diff --git a/tests/lang/eval-okay-delayed-with.nix b/tests/functional/lang/eval-okay-delayed-with.nix similarity index 100% rename from tests/lang/eval-okay-delayed-with.nix rename to tests/functional/lang/eval-okay-delayed-with.nix diff --git a/tests/lang/eval-okay-dynamic-attrs-2.exp b/tests/functional/lang/eval-okay-dynamic-attrs-2.exp similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs-2.exp rename to tests/functional/lang/eval-okay-dynamic-attrs-2.exp diff --git a/tests/lang/eval-okay-dynamic-attrs-2.nix b/tests/functional/lang/eval-okay-dynamic-attrs-2.nix similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs-2.nix rename to tests/functional/lang/eval-okay-dynamic-attrs-2.nix diff --git a/tests/lang/eval-okay-dynamic-attrs-bare.exp b/tests/functional/lang/eval-okay-dynamic-attrs-bare.exp similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs-bare.exp rename to tests/functional/lang/eval-okay-dynamic-attrs-bare.exp diff --git a/tests/lang/eval-okay-dynamic-attrs-bare.nix b/tests/functional/lang/eval-okay-dynamic-attrs-bare.nix similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs-bare.nix rename to tests/functional/lang/eval-okay-dynamic-attrs-bare.nix diff --git a/tests/lang/eval-okay-dynamic-attrs.exp b/tests/functional/lang/eval-okay-dynamic-attrs.exp similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs.exp rename to tests/functional/lang/eval-okay-dynamic-attrs.exp diff --git a/tests/lang/eval-okay-dynamic-attrs.nix b/tests/functional/lang/eval-okay-dynamic-attrs.nix similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs.nix rename to tests/functional/lang/eval-okay-dynamic-attrs.nix diff --git a/tests/lang/eval-okay-elem.exp b/tests/functional/lang/eval-okay-elem.exp similarity index 100% rename from tests/lang/eval-okay-elem.exp rename to tests/functional/lang/eval-okay-elem.exp diff --git a/tests/lang/eval-okay-elem.nix b/tests/functional/lang/eval-okay-elem.nix similarity index 100% rename from tests/lang/eval-okay-elem.nix rename to tests/functional/lang/eval-okay-elem.nix diff --git a/tests/lang/eval-okay-empty-args.exp b/tests/functional/lang/eval-okay-empty-args.exp similarity index 100% rename from tests/lang/eval-okay-empty-args.exp rename to tests/functional/lang/eval-okay-empty-args.exp diff --git a/tests/lang/eval-okay-empty-args.nix b/tests/functional/lang/eval-okay-empty-args.nix similarity index 100% rename from tests/lang/eval-okay-empty-args.nix rename to tests/functional/lang/eval-okay-empty-args.nix diff --git a/tests/lang/eval-okay-eq-derivations.exp b/tests/functional/lang/eval-okay-eq-derivations.exp similarity index 100% rename from tests/lang/eval-okay-eq-derivations.exp rename to tests/functional/lang/eval-okay-eq-derivations.exp diff --git a/tests/lang/eval-okay-eq-derivations.nix b/tests/functional/lang/eval-okay-eq-derivations.nix similarity index 100% rename from tests/lang/eval-okay-eq-derivations.nix rename to tests/functional/lang/eval-okay-eq-derivations.nix diff --git a/tests/lang/eval-okay-eq.exp b/tests/functional/lang/eval-okay-eq.exp similarity index 100% rename from tests/lang/eval-okay-eq.exp rename to tests/functional/lang/eval-okay-eq.exp diff --git a/tests/lang/eval-okay-eq.nix b/tests/functional/lang/eval-okay-eq.nix similarity index 100% rename from tests/lang/eval-okay-eq.nix rename to tests/functional/lang/eval-okay-eq.nix diff --git a/tests/lang/eval-okay-filter.exp b/tests/functional/lang/eval-okay-filter.exp similarity index 100% rename from tests/lang/eval-okay-filter.exp rename to tests/functional/lang/eval-okay-filter.exp diff --git a/tests/lang/eval-okay-filter.nix b/tests/functional/lang/eval-okay-filter.nix similarity index 100% rename from tests/lang/eval-okay-filter.nix rename to tests/functional/lang/eval-okay-filter.nix diff --git a/tests/lang/eval-okay-flake-ref-to-string.exp b/tests/functional/lang/eval-okay-flake-ref-to-string.exp similarity index 100% rename from tests/lang/eval-okay-flake-ref-to-string.exp rename to tests/functional/lang/eval-okay-flake-ref-to-string.exp diff --git a/tests/lang/eval-okay-flake-ref-to-string.nix b/tests/functional/lang/eval-okay-flake-ref-to-string.nix similarity index 100% rename from tests/lang/eval-okay-flake-ref-to-string.nix rename to tests/functional/lang/eval-okay-flake-ref-to-string.nix diff --git a/tests/lang/eval-okay-flatten.exp b/tests/functional/lang/eval-okay-flatten.exp similarity index 100% rename from tests/lang/eval-okay-flatten.exp rename to tests/functional/lang/eval-okay-flatten.exp diff --git a/tests/lang/eval-okay-flatten.nix b/tests/functional/lang/eval-okay-flatten.nix similarity index 100% rename from tests/lang/eval-okay-flatten.nix rename to tests/functional/lang/eval-okay-flatten.nix diff --git a/tests/lang/eval-okay-float.exp b/tests/functional/lang/eval-okay-float.exp similarity index 100% rename from tests/lang/eval-okay-float.exp rename to tests/functional/lang/eval-okay-float.exp diff --git a/tests/lang/eval-okay-float.nix b/tests/functional/lang/eval-okay-float.nix similarity index 100% rename from tests/lang/eval-okay-float.nix rename to tests/functional/lang/eval-okay-float.nix diff --git a/tests/lang/eval-okay-floor-ceil.exp b/tests/functional/lang/eval-okay-floor-ceil.exp similarity index 100% rename from tests/lang/eval-okay-floor-ceil.exp rename to tests/functional/lang/eval-okay-floor-ceil.exp diff --git a/tests/lang/eval-okay-floor-ceil.nix b/tests/functional/lang/eval-okay-floor-ceil.nix similarity index 100% rename from tests/lang/eval-okay-floor-ceil.nix rename to tests/functional/lang/eval-okay-floor-ceil.nix diff --git a/tests/lang/eval-okay-foldlStrict-lazy-elements.exp b/tests/functional/lang/eval-okay-foldlStrict-lazy-elements.exp similarity index 100% rename from tests/lang/eval-okay-foldlStrict-lazy-elements.exp rename to tests/functional/lang/eval-okay-foldlStrict-lazy-elements.exp diff --git a/tests/lang/eval-okay-foldlStrict-lazy-elements.nix b/tests/functional/lang/eval-okay-foldlStrict-lazy-elements.nix similarity index 100% rename from tests/lang/eval-okay-foldlStrict-lazy-elements.nix rename to tests/functional/lang/eval-okay-foldlStrict-lazy-elements.nix diff --git a/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp b/tests/functional/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp similarity index 100% rename from tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp rename to tests/functional/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp diff --git a/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix b/tests/functional/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix similarity index 100% rename from tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix rename to tests/functional/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix diff --git a/tests/lang/eval-okay-foldlStrict.exp b/tests/functional/lang/eval-okay-foldlStrict.exp similarity index 100% rename from tests/lang/eval-okay-foldlStrict.exp rename to tests/functional/lang/eval-okay-foldlStrict.exp diff --git a/tests/lang/eval-okay-foldlStrict.nix b/tests/functional/lang/eval-okay-foldlStrict.nix similarity index 100% rename from tests/lang/eval-okay-foldlStrict.nix rename to tests/functional/lang/eval-okay-foldlStrict.nix diff --git a/tests/lang/eval-okay-fromTOML-timestamps.exp b/tests/functional/lang/eval-okay-fromTOML-timestamps.exp similarity index 100% rename from tests/lang/eval-okay-fromTOML-timestamps.exp rename to tests/functional/lang/eval-okay-fromTOML-timestamps.exp diff --git a/tests/lang/eval-okay-fromTOML-timestamps.flags b/tests/functional/lang/eval-okay-fromTOML-timestamps.flags similarity index 100% rename from tests/lang/eval-okay-fromTOML-timestamps.flags rename to tests/functional/lang/eval-okay-fromTOML-timestamps.flags diff --git a/tests/lang/eval-okay-fromTOML-timestamps.nix b/tests/functional/lang/eval-okay-fromTOML-timestamps.nix similarity index 100% rename from tests/lang/eval-okay-fromTOML-timestamps.nix rename to tests/functional/lang/eval-okay-fromTOML-timestamps.nix diff --git a/tests/lang/eval-okay-fromTOML.exp b/tests/functional/lang/eval-okay-fromTOML.exp similarity index 100% rename from tests/lang/eval-okay-fromTOML.exp rename to tests/functional/lang/eval-okay-fromTOML.exp diff --git a/tests/lang/eval-okay-fromTOML.nix b/tests/functional/lang/eval-okay-fromTOML.nix similarity index 100% rename from tests/lang/eval-okay-fromTOML.nix rename to tests/functional/lang/eval-okay-fromTOML.nix diff --git a/tests/lang/eval-okay-fromjson-escapes.exp b/tests/functional/lang/eval-okay-fromjson-escapes.exp similarity index 100% rename from tests/lang/eval-okay-fromjson-escapes.exp rename to tests/functional/lang/eval-okay-fromjson-escapes.exp diff --git a/tests/lang/eval-okay-fromjson-escapes.nix b/tests/functional/lang/eval-okay-fromjson-escapes.nix similarity index 100% rename from tests/lang/eval-okay-fromjson-escapes.nix rename to tests/functional/lang/eval-okay-fromjson-escapes.nix diff --git a/tests/lang/eval-okay-fromjson.exp b/tests/functional/lang/eval-okay-fromjson.exp similarity index 100% rename from tests/lang/eval-okay-fromjson.exp rename to tests/functional/lang/eval-okay-fromjson.exp diff --git a/tests/lang/eval-okay-fromjson.nix b/tests/functional/lang/eval-okay-fromjson.nix similarity index 100% rename from tests/lang/eval-okay-fromjson.nix rename to tests/functional/lang/eval-okay-fromjson.nix diff --git a/tests/lang/eval-okay-functionargs.exp b/tests/functional/lang/eval-okay-functionargs.exp similarity index 100% rename from tests/lang/eval-okay-functionargs.exp rename to tests/functional/lang/eval-okay-functionargs.exp diff --git a/tests/lang/eval-okay-functionargs.exp.xml b/tests/functional/lang/eval-okay-functionargs.exp.xml similarity index 100% rename from tests/lang/eval-okay-functionargs.exp.xml rename to tests/functional/lang/eval-okay-functionargs.exp.xml diff --git a/tests/lang/eval-okay-functionargs.nix b/tests/functional/lang/eval-okay-functionargs.nix similarity index 100% rename from tests/lang/eval-okay-functionargs.nix rename to tests/functional/lang/eval-okay-functionargs.nix diff --git a/tests/lang/eval-okay-getattrpos-functionargs.exp b/tests/functional/lang/eval-okay-getattrpos-functionargs.exp similarity index 100% rename from tests/lang/eval-okay-getattrpos-functionargs.exp rename to tests/functional/lang/eval-okay-getattrpos-functionargs.exp diff --git a/tests/lang/eval-okay-getattrpos-functionargs.nix b/tests/functional/lang/eval-okay-getattrpos-functionargs.nix similarity index 100% rename from tests/lang/eval-okay-getattrpos-functionargs.nix rename to tests/functional/lang/eval-okay-getattrpos-functionargs.nix diff --git a/tests/lang/eval-okay-getattrpos-undefined.exp b/tests/functional/lang/eval-okay-getattrpos-undefined.exp similarity index 100% rename from tests/lang/eval-okay-getattrpos-undefined.exp rename to tests/functional/lang/eval-okay-getattrpos-undefined.exp diff --git a/tests/lang/eval-okay-getattrpos-undefined.nix b/tests/functional/lang/eval-okay-getattrpos-undefined.nix similarity index 100% rename from tests/lang/eval-okay-getattrpos-undefined.nix rename to tests/functional/lang/eval-okay-getattrpos-undefined.nix diff --git a/tests/lang/eval-okay-getattrpos.exp b/tests/functional/lang/eval-okay-getattrpos.exp similarity index 100% rename from tests/lang/eval-okay-getattrpos.exp rename to tests/functional/lang/eval-okay-getattrpos.exp diff --git a/tests/lang/eval-okay-getattrpos.nix b/tests/functional/lang/eval-okay-getattrpos.nix similarity index 100% rename from tests/lang/eval-okay-getattrpos.nix rename to tests/functional/lang/eval-okay-getattrpos.nix diff --git a/tests/lang/eval-okay-getenv.exp b/tests/functional/lang/eval-okay-getenv.exp similarity index 100% rename from tests/lang/eval-okay-getenv.exp rename to tests/functional/lang/eval-okay-getenv.exp diff --git a/tests/lang/eval-okay-getenv.nix b/tests/functional/lang/eval-okay-getenv.nix similarity index 100% rename from tests/lang/eval-okay-getenv.nix rename to tests/functional/lang/eval-okay-getenv.nix diff --git a/tests/lang/eval-okay-groupBy.exp b/tests/functional/lang/eval-okay-groupBy.exp similarity index 100% rename from tests/lang/eval-okay-groupBy.exp rename to tests/functional/lang/eval-okay-groupBy.exp diff --git a/tests/lang/eval-okay-groupBy.nix b/tests/functional/lang/eval-okay-groupBy.nix similarity index 100% rename from tests/lang/eval-okay-groupBy.nix rename to tests/functional/lang/eval-okay-groupBy.nix diff --git a/tests/lang/eval-okay-hash.exp b/tests/functional/lang/eval-okay-hash.exp similarity index 100% rename from tests/lang/eval-okay-hash.exp rename to tests/functional/lang/eval-okay-hash.exp diff --git a/tests/lang/eval-okay-hashfile.exp b/tests/functional/lang/eval-okay-hashfile.exp similarity index 100% rename from tests/lang/eval-okay-hashfile.exp rename to tests/functional/lang/eval-okay-hashfile.exp diff --git a/tests/lang/eval-okay-hashfile.nix b/tests/functional/lang/eval-okay-hashfile.nix similarity index 100% rename from tests/lang/eval-okay-hashfile.nix rename to tests/functional/lang/eval-okay-hashfile.nix diff --git a/tests/lang/eval-okay-hashstring.exp b/tests/functional/lang/eval-okay-hashstring.exp similarity index 100% rename from tests/lang/eval-okay-hashstring.exp rename to tests/functional/lang/eval-okay-hashstring.exp diff --git a/tests/lang/eval-okay-hashstring.nix b/tests/functional/lang/eval-okay-hashstring.nix similarity index 100% rename from tests/lang/eval-okay-hashstring.nix rename to tests/functional/lang/eval-okay-hashstring.nix diff --git a/tests/lang/eval-okay-if.exp b/tests/functional/lang/eval-okay-if.exp similarity index 100% rename from tests/lang/eval-okay-if.exp rename to tests/functional/lang/eval-okay-if.exp diff --git a/tests/lang/eval-okay-if.nix b/tests/functional/lang/eval-okay-if.nix similarity index 100% rename from tests/lang/eval-okay-if.nix rename to tests/functional/lang/eval-okay-if.nix diff --git a/tests/lang/eval-okay-import.exp b/tests/functional/lang/eval-okay-import.exp similarity index 100% rename from tests/lang/eval-okay-import.exp rename to tests/functional/lang/eval-okay-import.exp diff --git a/tests/lang/eval-okay-import.nix b/tests/functional/lang/eval-okay-import.nix similarity index 100% rename from tests/lang/eval-okay-import.nix rename to tests/functional/lang/eval-okay-import.nix diff --git a/tests/lang/eval-okay-ind-string.exp b/tests/functional/lang/eval-okay-ind-string.exp similarity index 100% rename from tests/lang/eval-okay-ind-string.exp rename to tests/functional/lang/eval-okay-ind-string.exp diff --git a/tests/lang/eval-okay-ind-string.nix b/tests/functional/lang/eval-okay-ind-string.nix similarity index 100% rename from tests/lang/eval-okay-ind-string.nix rename to tests/functional/lang/eval-okay-ind-string.nix diff --git a/tests/lang/eval-okay-intersectAttrs.exp b/tests/functional/lang/eval-okay-intersectAttrs.exp similarity index 100% rename from tests/lang/eval-okay-intersectAttrs.exp rename to tests/functional/lang/eval-okay-intersectAttrs.exp diff --git a/tests/lang/eval-okay-intersectAttrs.nix b/tests/functional/lang/eval-okay-intersectAttrs.nix similarity index 100% rename from tests/lang/eval-okay-intersectAttrs.nix rename to tests/functional/lang/eval-okay-intersectAttrs.nix diff --git a/tests/lang/eval-okay-let.exp b/tests/functional/lang/eval-okay-let.exp similarity index 100% rename from tests/lang/eval-okay-let.exp rename to tests/functional/lang/eval-okay-let.exp diff --git a/tests/lang/eval-okay-let.nix b/tests/functional/lang/eval-okay-let.nix similarity index 100% rename from tests/lang/eval-okay-let.nix rename to tests/functional/lang/eval-okay-let.nix diff --git a/tests/lang/eval-okay-list.exp b/tests/functional/lang/eval-okay-list.exp similarity index 100% rename from tests/lang/eval-okay-list.exp rename to tests/functional/lang/eval-okay-list.exp diff --git a/tests/lang/eval-okay-list.nix b/tests/functional/lang/eval-okay-list.nix similarity index 100% rename from tests/lang/eval-okay-list.nix rename to tests/functional/lang/eval-okay-list.nix diff --git a/tests/lang/eval-okay-listtoattrs.exp b/tests/functional/lang/eval-okay-listtoattrs.exp similarity index 100% rename from tests/lang/eval-okay-listtoattrs.exp rename to tests/functional/lang/eval-okay-listtoattrs.exp diff --git a/tests/lang/eval-okay-listtoattrs.nix b/tests/functional/lang/eval-okay-listtoattrs.nix similarity index 100% rename from tests/lang/eval-okay-listtoattrs.nix rename to tests/functional/lang/eval-okay-listtoattrs.nix diff --git a/tests/lang/eval-okay-logic.exp b/tests/functional/lang/eval-okay-logic.exp similarity index 100% rename from tests/lang/eval-okay-logic.exp rename to tests/functional/lang/eval-okay-logic.exp diff --git a/tests/lang/eval-okay-logic.nix b/tests/functional/lang/eval-okay-logic.nix similarity index 100% rename from tests/lang/eval-okay-logic.nix rename to tests/functional/lang/eval-okay-logic.nix diff --git a/tests/lang/eval-okay-map.exp b/tests/functional/lang/eval-okay-map.exp similarity index 100% rename from tests/lang/eval-okay-map.exp rename to tests/functional/lang/eval-okay-map.exp diff --git a/tests/lang/eval-okay-map.nix b/tests/functional/lang/eval-okay-map.nix similarity index 100% rename from tests/lang/eval-okay-map.nix rename to tests/functional/lang/eval-okay-map.nix diff --git a/tests/lang/eval-okay-mapattrs.exp b/tests/functional/lang/eval-okay-mapattrs.exp similarity index 100% rename from tests/lang/eval-okay-mapattrs.exp rename to tests/functional/lang/eval-okay-mapattrs.exp diff --git a/tests/lang/eval-okay-mapattrs.nix b/tests/functional/lang/eval-okay-mapattrs.nix similarity index 100% rename from tests/lang/eval-okay-mapattrs.nix rename to tests/functional/lang/eval-okay-mapattrs.nix diff --git a/tests/lang/eval-okay-merge-dynamic-attrs.exp b/tests/functional/lang/eval-okay-merge-dynamic-attrs.exp similarity index 100% rename from tests/lang/eval-okay-merge-dynamic-attrs.exp rename to tests/functional/lang/eval-okay-merge-dynamic-attrs.exp diff --git a/tests/lang/eval-okay-merge-dynamic-attrs.nix b/tests/functional/lang/eval-okay-merge-dynamic-attrs.nix similarity index 100% rename from tests/lang/eval-okay-merge-dynamic-attrs.nix rename to tests/functional/lang/eval-okay-merge-dynamic-attrs.nix diff --git a/tests/lang/eval-okay-nested-with.exp b/tests/functional/lang/eval-okay-nested-with.exp similarity index 100% rename from tests/lang/eval-okay-nested-with.exp rename to tests/functional/lang/eval-okay-nested-with.exp diff --git a/tests/lang/eval-okay-nested-with.nix b/tests/functional/lang/eval-okay-nested-with.nix similarity index 100% rename from tests/lang/eval-okay-nested-with.nix rename to tests/functional/lang/eval-okay-nested-with.nix diff --git a/tests/lang/eval-okay-new-let.exp b/tests/functional/lang/eval-okay-new-let.exp similarity index 100% rename from tests/lang/eval-okay-new-let.exp rename to tests/functional/lang/eval-okay-new-let.exp diff --git a/tests/lang/eval-okay-new-let.nix b/tests/functional/lang/eval-okay-new-let.nix similarity index 100% rename from tests/lang/eval-okay-new-let.nix rename to tests/functional/lang/eval-okay-new-let.nix diff --git a/tests/lang/eval-okay-null-dynamic-attrs.exp b/tests/functional/lang/eval-okay-null-dynamic-attrs.exp similarity index 100% rename from tests/lang/eval-okay-null-dynamic-attrs.exp rename to tests/functional/lang/eval-okay-null-dynamic-attrs.exp diff --git a/tests/lang/eval-okay-null-dynamic-attrs.nix b/tests/functional/lang/eval-okay-null-dynamic-attrs.nix similarity index 100% rename from tests/lang/eval-okay-null-dynamic-attrs.nix rename to tests/functional/lang/eval-okay-null-dynamic-attrs.nix diff --git a/tests/lang/eval-okay-overrides.exp b/tests/functional/lang/eval-okay-overrides.exp similarity index 100% rename from tests/lang/eval-okay-overrides.exp rename to tests/functional/lang/eval-okay-overrides.exp diff --git a/tests/lang/eval-okay-overrides.nix b/tests/functional/lang/eval-okay-overrides.nix similarity index 100% rename from tests/lang/eval-okay-overrides.nix rename to tests/functional/lang/eval-okay-overrides.nix diff --git a/tests/lang/eval-okay-parse-flake-ref.exp b/tests/functional/lang/eval-okay-parse-flake-ref.exp similarity index 100% rename from tests/lang/eval-okay-parse-flake-ref.exp rename to tests/functional/lang/eval-okay-parse-flake-ref.exp diff --git a/tests/lang/eval-okay-parse-flake-ref.nix b/tests/functional/lang/eval-okay-parse-flake-ref.nix similarity index 100% rename from tests/lang/eval-okay-parse-flake-ref.nix rename to tests/functional/lang/eval-okay-parse-flake-ref.nix diff --git a/tests/lang/eval-okay-partition.exp b/tests/functional/lang/eval-okay-partition.exp similarity index 100% rename from tests/lang/eval-okay-partition.exp rename to tests/functional/lang/eval-okay-partition.exp diff --git a/tests/lang/eval-okay-partition.nix b/tests/functional/lang/eval-okay-partition.nix similarity index 100% rename from tests/lang/eval-okay-partition.nix rename to tests/functional/lang/eval-okay-partition.nix diff --git a/tests/lang/eval-okay-path-string-interpolation.exp b/tests/functional/lang/eval-okay-path-string-interpolation.exp similarity index 100% rename from tests/lang/eval-okay-path-string-interpolation.exp rename to tests/functional/lang/eval-okay-path-string-interpolation.exp diff --git a/tests/lang/eval-okay-path-string-interpolation.nix b/tests/functional/lang/eval-okay-path-string-interpolation.nix similarity index 100% rename from tests/lang/eval-okay-path-string-interpolation.nix rename to tests/functional/lang/eval-okay-path-string-interpolation.nix diff --git a/tests/functional/lang/eval-okay-path.exp b/tests/functional/lang/eval-okay-path.exp new file mode 100644 index 00000000000..635e2243a2a --- /dev/null +++ b/tests/functional/lang/eval-okay-path.exp @@ -0,0 +1 @@ +[ "/nix/store/ya937r4ydw0l6kayq8jkyqaips9c75jm-output" "/nix/store/m7y372g6jb0g4hh1dzmj847rd356fhnz-output" ] diff --git a/tests/functional/lang/eval-okay-path.nix b/tests/functional/lang/eval-okay-path.nix new file mode 100644 index 00000000000..599b3354147 --- /dev/null +++ b/tests/functional/lang/eval-okay-path.nix @@ -0,0 +1,15 @@ +[ + (builtins.path + { path = ./.; + filter = path: _: baseNameOf path == "data"; + recursive = true; + sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw"; + name = "output"; + }) + (builtins.path + { path = ./data; + recursive = false; + sha256 = "0k4lwj58f2w5yh92ilrwy9917pycipbrdrr13vbb3yd02j09vfxm"; + name = "output"; + }) +] diff --git a/tests/lang/eval-okay-pathexists.exp b/tests/functional/lang/eval-okay-pathexists.exp similarity index 100% rename from tests/lang/eval-okay-pathexists.exp rename to tests/functional/lang/eval-okay-pathexists.exp diff --git a/tests/functional/lang/eval-okay-pathexists.nix b/tests/functional/lang/eval-okay-pathexists.nix new file mode 100644 index 00000000000..31697f66a90 --- /dev/null +++ b/tests/functional/lang/eval-okay-pathexists.nix @@ -0,0 +1,31 @@ +builtins.pathExists (./lib.nix) +&& builtins.pathExists (builtins.toPath ./lib.nix) +&& builtins.pathExists (builtins.toString ./lib.nix) +&& !builtins.pathExists (builtins.toString ./lib.nix + "/") +&& !builtins.pathExists (builtins.toString ./lib.nix + "/.") +# FIXME +# && !builtins.pathExists (builtins.toString ./lib.nix + "/..") +# && !builtins.pathExists (builtins.toString ./lib.nix + "/a/..") +# && !builtins.pathExists (builtins.toString ./lib.nix + "/../lib.nix") +&& !builtins.pathExists (builtins.toString ./lib.nix + "/./") +&& !builtins.pathExists (builtins.toString ./lib.nix + "/./.") +&& builtins.pathExists (builtins.toString ./.. + "/lang/lib.nix") +&& !builtins.pathExists (builtins.toString ./.. + "lang/lib.nix") +&& builtins.pathExists (builtins.toString ./. + "/../lang/lib.nix") +&& builtins.pathExists (builtins.toString ./. + "/../lang/./lib.nix") +&& builtins.pathExists (builtins.toString ./.) +&& builtins.pathExists (builtins.toString ./. + "/") +&& builtins.pathExists (builtins.toString ./. + "/../lang") +&& builtins.pathExists (builtins.toString ./. + "/../lang/") +&& builtins.pathExists (builtins.toString ./. + "/../lang/.") +&& builtins.pathExists (builtins.toString ./. + "/../lang/./") +&& builtins.pathExists (builtins.toString ./. + "/../lang//./") +&& builtins.pathExists (builtins.toString ./. + "/../lang/..") +&& builtins.pathExists (builtins.toString ./. + "/../lang/../") +&& builtins.pathExists (builtins.toString ./. + "/../lang/..//") +&& builtins.pathExists (builtins.toPath (builtins.toString ./lib.nix)) +&& !builtins.pathExists (builtins.toPath (builtins.toString ./bla.nix)) +&& builtins.pathExists (builtins.toPath { __toString = x: builtins.toString ./lib.nix; }) +&& builtins.pathExists (builtins.toPath { outPath = builtins.toString ./lib.nix; }) +&& builtins.pathExists ./lib.nix +&& !builtins.pathExists ./bla.nix diff --git a/tests/lang/eval-okay-patterns.exp b/tests/functional/lang/eval-okay-patterns.exp similarity index 100% rename from tests/lang/eval-okay-patterns.exp rename to tests/functional/lang/eval-okay-patterns.exp diff --git a/tests/lang/eval-okay-patterns.nix b/tests/functional/lang/eval-okay-patterns.nix similarity index 100% rename from tests/lang/eval-okay-patterns.nix rename to tests/functional/lang/eval-okay-patterns.nix diff --git a/tests/lang/eval-okay-print.err.exp b/tests/functional/lang/eval-okay-print.err.exp similarity index 100% rename from tests/lang/eval-okay-print.err.exp rename to tests/functional/lang/eval-okay-print.err.exp diff --git a/tests/lang/eval-okay-print.exp b/tests/functional/lang/eval-okay-print.exp similarity index 100% rename from tests/lang/eval-okay-print.exp rename to tests/functional/lang/eval-okay-print.exp diff --git a/tests/lang/eval-okay-print.nix b/tests/functional/lang/eval-okay-print.nix similarity index 100% rename from tests/lang/eval-okay-print.nix rename to tests/functional/lang/eval-okay-print.nix diff --git a/tests/lang/eval-okay-readDir.exp b/tests/functional/lang/eval-okay-readDir.exp similarity index 100% rename from tests/lang/eval-okay-readDir.exp rename to tests/functional/lang/eval-okay-readDir.exp diff --git a/tests/lang/eval-okay-readDir.nix b/tests/functional/lang/eval-okay-readDir.nix similarity index 100% rename from tests/lang/eval-okay-readDir.nix rename to tests/functional/lang/eval-okay-readDir.nix diff --git a/tests/lang/eval-okay-readFileType.exp b/tests/functional/lang/eval-okay-readFileType.exp similarity index 100% rename from tests/lang/eval-okay-readFileType.exp rename to tests/functional/lang/eval-okay-readFileType.exp diff --git a/tests/lang/eval-okay-readFileType.nix b/tests/functional/lang/eval-okay-readFileType.nix similarity index 100% rename from tests/lang/eval-okay-readFileType.nix rename to tests/functional/lang/eval-okay-readFileType.nix diff --git a/tests/lang/eval-okay-readfile.exp b/tests/functional/lang/eval-okay-readfile.exp similarity index 100% rename from tests/lang/eval-okay-readfile.exp rename to tests/functional/lang/eval-okay-readfile.exp diff --git a/tests/lang/eval-okay-readfile.nix b/tests/functional/lang/eval-okay-readfile.nix similarity index 100% rename from tests/lang/eval-okay-readfile.nix rename to tests/functional/lang/eval-okay-readfile.nix diff --git a/tests/lang/eval-okay-redefine-builtin.exp b/tests/functional/lang/eval-okay-redefine-builtin.exp similarity index 100% rename from tests/lang/eval-okay-redefine-builtin.exp rename to tests/functional/lang/eval-okay-redefine-builtin.exp diff --git a/tests/lang/eval-okay-redefine-builtin.nix b/tests/functional/lang/eval-okay-redefine-builtin.nix similarity index 100% rename from tests/lang/eval-okay-redefine-builtin.nix rename to tests/functional/lang/eval-okay-redefine-builtin.nix diff --git a/tests/lang/eval-okay-regex-match.exp b/tests/functional/lang/eval-okay-regex-match.exp similarity index 100% rename from tests/lang/eval-okay-regex-match.exp rename to tests/functional/lang/eval-okay-regex-match.exp diff --git a/tests/lang/eval-okay-regex-match.nix b/tests/functional/lang/eval-okay-regex-match.nix similarity index 100% rename from tests/lang/eval-okay-regex-match.nix rename to tests/functional/lang/eval-okay-regex-match.nix diff --git a/tests/lang/eval-okay-regex-split.exp b/tests/functional/lang/eval-okay-regex-split.exp similarity index 100% rename from tests/lang/eval-okay-regex-split.exp rename to tests/functional/lang/eval-okay-regex-split.exp diff --git a/tests/lang/eval-okay-regex-split.nix b/tests/functional/lang/eval-okay-regex-split.nix similarity index 100% rename from tests/lang/eval-okay-regex-split.nix rename to tests/functional/lang/eval-okay-regex-split.nix diff --git a/tests/lang/eval-okay-regression-20220122.exp b/tests/functional/lang/eval-okay-regression-20220122.exp similarity index 100% rename from tests/lang/eval-okay-regression-20220122.exp rename to tests/functional/lang/eval-okay-regression-20220122.exp diff --git a/tests/lang/eval-okay-regression-20220122.nix b/tests/functional/lang/eval-okay-regression-20220122.nix similarity index 100% rename from tests/lang/eval-okay-regression-20220122.nix rename to tests/functional/lang/eval-okay-regression-20220122.nix diff --git a/tests/lang/eval-okay-regression-20220125.exp b/tests/functional/lang/eval-okay-regression-20220125.exp similarity index 100% rename from tests/lang/eval-okay-regression-20220125.exp rename to tests/functional/lang/eval-okay-regression-20220125.exp diff --git a/tests/lang/eval-okay-regression-20220125.nix b/tests/functional/lang/eval-okay-regression-20220125.nix similarity index 100% rename from tests/lang/eval-okay-regression-20220125.nix rename to tests/functional/lang/eval-okay-regression-20220125.nix diff --git a/tests/lang/eval-okay-remove.exp b/tests/functional/lang/eval-okay-remove.exp similarity index 100% rename from tests/lang/eval-okay-remove.exp rename to tests/functional/lang/eval-okay-remove.exp diff --git a/tests/lang/eval-okay-remove.nix b/tests/functional/lang/eval-okay-remove.nix similarity index 100% rename from tests/lang/eval-okay-remove.nix rename to tests/functional/lang/eval-okay-remove.nix diff --git a/tests/lang/eval-okay-replacestrings.exp b/tests/functional/lang/eval-okay-replacestrings.exp similarity index 100% rename from tests/lang/eval-okay-replacestrings.exp rename to tests/functional/lang/eval-okay-replacestrings.exp diff --git a/tests/lang/eval-okay-replacestrings.nix b/tests/functional/lang/eval-okay-replacestrings.nix similarity index 100% rename from tests/lang/eval-okay-replacestrings.nix rename to tests/functional/lang/eval-okay-replacestrings.nix diff --git a/tests/lang/eval-okay-scope-1.exp b/tests/functional/lang/eval-okay-scope-1.exp similarity index 100% rename from tests/lang/eval-okay-scope-1.exp rename to tests/functional/lang/eval-okay-scope-1.exp diff --git a/tests/lang/eval-okay-scope-1.nix b/tests/functional/lang/eval-okay-scope-1.nix similarity index 100% rename from tests/lang/eval-okay-scope-1.nix rename to tests/functional/lang/eval-okay-scope-1.nix diff --git a/tests/lang/eval-okay-scope-2.exp b/tests/functional/lang/eval-okay-scope-2.exp similarity index 100% rename from tests/lang/eval-okay-scope-2.exp rename to tests/functional/lang/eval-okay-scope-2.exp diff --git a/tests/lang/eval-okay-scope-2.nix b/tests/functional/lang/eval-okay-scope-2.nix similarity index 100% rename from tests/lang/eval-okay-scope-2.nix rename to tests/functional/lang/eval-okay-scope-2.nix diff --git a/tests/lang/eval-okay-scope-3.exp b/tests/functional/lang/eval-okay-scope-3.exp similarity index 100% rename from tests/lang/eval-okay-scope-3.exp rename to tests/functional/lang/eval-okay-scope-3.exp diff --git a/tests/lang/eval-okay-scope-3.nix b/tests/functional/lang/eval-okay-scope-3.nix similarity index 100% rename from tests/lang/eval-okay-scope-3.nix rename to tests/functional/lang/eval-okay-scope-3.nix diff --git a/tests/lang/eval-okay-scope-4.exp b/tests/functional/lang/eval-okay-scope-4.exp similarity index 100% rename from tests/lang/eval-okay-scope-4.exp rename to tests/functional/lang/eval-okay-scope-4.exp diff --git a/tests/lang/eval-okay-scope-4.nix b/tests/functional/lang/eval-okay-scope-4.nix similarity index 100% rename from tests/lang/eval-okay-scope-4.nix rename to tests/functional/lang/eval-okay-scope-4.nix diff --git a/tests/lang/eval-okay-scope-6.exp b/tests/functional/lang/eval-okay-scope-6.exp similarity index 100% rename from tests/lang/eval-okay-scope-6.exp rename to tests/functional/lang/eval-okay-scope-6.exp diff --git a/tests/lang/eval-okay-scope-6.nix b/tests/functional/lang/eval-okay-scope-6.nix similarity index 100% rename from tests/lang/eval-okay-scope-6.nix rename to tests/functional/lang/eval-okay-scope-6.nix diff --git a/tests/lang/eval-okay-scope-7.exp b/tests/functional/lang/eval-okay-scope-7.exp similarity index 100% rename from tests/lang/eval-okay-scope-7.exp rename to tests/functional/lang/eval-okay-scope-7.exp diff --git a/tests/lang/eval-okay-scope-7.nix b/tests/functional/lang/eval-okay-scope-7.nix similarity index 100% rename from tests/lang/eval-okay-scope-7.nix rename to tests/functional/lang/eval-okay-scope-7.nix diff --git a/tests/lang/eval-okay-search-path.exp b/tests/functional/lang/eval-okay-search-path.exp similarity index 100% rename from tests/lang/eval-okay-search-path.exp rename to tests/functional/lang/eval-okay-search-path.exp diff --git a/tests/lang/eval-okay-search-path.flags b/tests/functional/lang/eval-okay-search-path.flags similarity index 100% rename from tests/lang/eval-okay-search-path.flags rename to tests/functional/lang/eval-okay-search-path.flags diff --git a/tests/lang/eval-okay-search-path.nix b/tests/functional/lang/eval-okay-search-path.nix similarity index 100% rename from tests/lang/eval-okay-search-path.nix rename to tests/functional/lang/eval-okay-search-path.nix diff --git a/tests/lang/eval-okay-seq.exp b/tests/functional/lang/eval-okay-seq.exp similarity index 100% rename from tests/lang/eval-okay-seq.exp rename to tests/functional/lang/eval-okay-seq.exp diff --git a/tests/lang/eval-okay-seq.nix b/tests/functional/lang/eval-okay-seq.nix similarity index 100% rename from tests/lang/eval-okay-seq.nix rename to tests/functional/lang/eval-okay-seq.nix diff --git a/tests/lang/eval-okay-sort.exp b/tests/functional/lang/eval-okay-sort.exp similarity index 100% rename from tests/lang/eval-okay-sort.exp rename to tests/functional/lang/eval-okay-sort.exp diff --git a/tests/lang/eval-okay-sort.nix b/tests/functional/lang/eval-okay-sort.nix similarity index 100% rename from tests/lang/eval-okay-sort.nix rename to tests/functional/lang/eval-okay-sort.nix diff --git a/tests/lang/eval-okay-splitversion.exp b/tests/functional/lang/eval-okay-splitversion.exp similarity index 100% rename from tests/lang/eval-okay-splitversion.exp rename to tests/functional/lang/eval-okay-splitversion.exp diff --git a/tests/lang/eval-okay-splitversion.nix b/tests/functional/lang/eval-okay-splitversion.nix similarity index 100% rename from tests/lang/eval-okay-splitversion.nix rename to tests/functional/lang/eval-okay-splitversion.nix diff --git a/tests/lang/eval-okay-string.exp b/tests/functional/lang/eval-okay-string.exp similarity index 100% rename from tests/lang/eval-okay-string.exp rename to tests/functional/lang/eval-okay-string.exp diff --git a/tests/lang/eval-okay-string.nix b/tests/functional/lang/eval-okay-string.nix similarity index 100% rename from tests/lang/eval-okay-string.nix rename to tests/functional/lang/eval-okay-string.nix diff --git a/tests/lang/eval-okay-strings-as-attrs-names.exp b/tests/functional/lang/eval-okay-strings-as-attrs-names.exp similarity index 100% rename from tests/lang/eval-okay-strings-as-attrs-names.exp rename to tests/functional/lang/eval-okay-strings-as-attrs-names.exp diff --git a/tests/lang/eval-okay-strings-as-attrs-names.nix b/tests/functional/lang/eval-okay-strings-as-attrs-names.nix similarity index 100% rename from tests/lang/eval-okay-strings-as-attrs-names.nix rename to tests/functional/lang/eval-okay-strings-as-attrs-names.nix diff --git a/tests/lang/eval-okay-substring.exp b/tests/functional/lang/eval-okay-substring.exp similarity index 100% rename from tests/lang/eval-okay-substring.exp rename to tests/functional/lang/eval-okay-substring.exp diff --git a/tests/lang/eval-okay-substring.nix b/tests/functional/lang/eval-okay-substring.nix similarity index 100% rename from tests/lang/eval-okay-substring.nix rename to tests/functional/lang/eval-okay-substring.nix diff --git a/tests/functional/lang/eval-okay-symlink-resolution.exp b/tests/functional/lang/eval-okay-symlink-resolution.exp new file mode 100644 index 00000000000..8b8441b91da --- /dev/null +++ b/tests/functional/lang/eval-okay-symlink-resolution.exp @@ -0,0 +1 @@ +"test" diff --git a/tests/functional/lang/eval-okay-symlink-resolution.nix b/tests/functional/lang/eval-okay-symlink-resolution.nix new file mode 100644 index 00000000000..ffb1818bde0 --- /dev/null +++ b/tests/functional/lang/eval-okay-symlink-resolution.nix @@ -0,0 +1 @@ +import symlink-resolution/foo/overlays/overlay.nix diff --git a/tests/lang/eval-okay-tail-call-1.exp-disabled b/tests/functional/lang/eval-okay-tail-call-1.exp-disabled similarity index 100% rename from tests/lang/eval-okay-tail-call-1.exp-disabled rename to tests/functional/lang/eval-okay-tail-call-1.exp-disabled diff --git a/tests/lang/eval-okay-tail-call-1.nix b/tests/functional/lang/eval-okay-tail-call-1.nix similarity index 100% rename from tests/lang/eval-okay-tail-call-1.nix rename to tests/functional/lang/eval-okay-tail-call-1.nix diff --git a/tests/lang/eval-okay-tojson.exp b/tests/functional/lang/eval-okay-tojson.exp similarity index 100% rename from tests/lang/eval-okay-tojson.exp rename to tests/functional/lang/eval-okay-tojson.exp diff --git a/tests/lang/eval-okay-tojson.nix b/tests/functional/lang/eval-okay-tojson.nix similarity index 100% rename from tests/lang/eval-okay-tojson.nix rename to tests/functional/lang/eval-okay-tojson.nix diff --git a/tests/lang/eval-okay-toxml.exp b/tests/functional/lang/eval-okay-toxml.exp similarity index 100% rename from tests/lang/eval-okay-toxml.exp rename to tests/functional/lang/eval-okay-toxml.exp diff --git a/tests/lang/eval-okay-toxml.nix b/tests/functional/lang/eval-okay-toxml.nix similarity index 100% rename from tests/lang/eval-okay-toxml.nix rename to tests/functional/lang/eval-okay-toxml.nix diff --git a/tests/lang/eval-okay-toxml2.exp b/tests/functional/lang/eval-okay-toxml2.exp similarity index 100% rename from tests/lang/eval-okay-toxml2.exp rename to tests/functional/lang/eval-okay-toxml2.exp diff --git a/tests/lang/eval-okay-toxml2.nix b/tests/functional/lang/eval-okay-toxml2.nix similarity index 100% rename from tests/lang/eval-okay-toxml2.nix rename to tests/functional/lang/eval-okay-toxml2.nix diff --git a/tests/lang/eval-okay-tryeval.exp b/tests/functional/lang/eval-okay-tryeval.exp similarity index 100% rename from tests/lang/eval-okay-tryeval.exp rename to tests/functional/lang/eval-okay-tryeval.exp diff --git a/tests/lang/eval-okay-tryeval.nix b/tests/functional/lang/eval-okay-tryeval.nix similarity index 100% rename from tests/lang/eval-okay-tryeval.nix rename to tests/functional/lang/eval-okay-tryeval.nix diff --git a/tests/lang/eval-okay-types.exp b/tests/functional/lang/eval-okay-types.exp similarity index 100% rename from tests/lang/eval-okay-types.exp rename to tests/functional/lang/eval-okay-types.exp diff --git a/tests/lang/eval-okay-types.nix b/tests/functional/lang/eval-okay-types.nix similarity index 100% rename from tests/lang/eval-okay-types.nix rename to tests/functional/lang/eval-okay-types.nix diff --git a/tests/lang/eval-okay-versions.exp b/tests/functional/lang/eval-okay-versions.exp similarity index 100% rename from tests/lang/eval-okay-versions.exp rename to tests/functional/lang/eval-okay-versions.exp diff --git a/tests/lang/eval-okay-versions.nix b/tests/functional/lang/eval-okay-versions.nix similarity index 100% rename from tests/lang/eval-okay-versions.nix rename to tests/functional/lang/eval-okay-versions.nix diff --git a/tests/lang/eval-okay-with.exp b/tests/functional/lang/eval-okay-with.exp similarity index 100% rename from tests/lang/eval-okay-with.exp rename to tests/functional/lang/eval-okay-with.exp diff --git a/tests/lang/eval-okay-with.nix b/tests/functional/lang/eval-okay-with.nix similarity index 100% rename from tests/lang/eval-okay-with.nix rename to tests/functional/lang/eval-okay-with.nix diff --git a/tests/lang/eval-okay-xml.exp.xml b/tests/functional/lang/eval-okay-xml.exp.xml similarity index 100% rename from tests/lang/eval-okay-xml.exp.xml rename to tests/functional/lang/eval-okay-xml.exp.xml diff --git a/tests/lang/eval-okay-xml.nix b/tests/functional/lang/eval-okay-xml.nix similarity index 100% rename from tests/lang/eval-okay-xml.nix rename to tests/functional/lang/eval-okay-xml.nix diff --git a/tests/lang/eval-okay-zipAttrsWith.exp b/tests/functional/lang/eval-okay-zipAttrsWith.exp similarity index 100% rename from tests/lang/eval-okay-zipAttrsWith.exp rename to tests/functional/lang/eval-okay-zipAttrsWith.exp diff --git a/tests/lang/eval-okay-zipAttrsWith.nix b/tests/functional/lang/eval-okay-zipAttrsWith.nix similarity index 100% rename from tests/lang/eval-okay-zipAttrsWith.nix rename to tests/functional/lang/eval-okay-zipAttrsWith.nix diff --git a/tests/lang/framework.sh b/tests/functional/lang/framework.sh similarity index 100% rename from tests/lang/framework.sh rename to tests/functional/lang/framework.sh diff --git a/tests/lang/imported.nix b/tests/functional/lang/imported.nix similarity index 100% rename from tests/lang/imported.nix rename to tests/functional/lang/imported.nix diff --git a/tests/lang/imported2.nix b/tests/functional/lang/imported2.nix similarity index 100% rename from tests/lang/imported2.nix rename to tests/functional/lang/imported2.nix diff --git a/tests/lang/lib.nix b/tests/functional/lang/lib.nix similarity index 100% rename from tests/lang/lib.nix rename to tests/functional/lang/lib.nix diff --git a/tests/lang/parse-fail-dup-attrs-1.err.exp b/tests/functional/lang/parse-fail-dup-attrs-1.err.exp similarity index 100% rename from tests/lang/parse-fail-dup-attrs-1.err.exp rename to tests/functional/lang/parse-fail-dup-attrs-1.err.exp diff --git a/tests/lang/parse-fail-dup-attrs-1.nix b/tests/functional/lang/parse-fail-dup-attrs-1.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-1.nix rename to tests/functional/lang/parse-fail-dup-attrs-1.nix diff --git a/tests/lang/parse-fail-dup-attrs-2.err.exp b/tests/functional/lang/parse-fail-dup-attrs-2.err.exp similarity index 100% rename from tests/lang/parse-fail-dup-attrs-2.err.exp rename to tests/functional/lang/parse-fail-dup-attrs-2.err.exp diff --git a/tests/lang/parse-fail-dup-attrs-2.nix b/tests/functional/lang/parse-fail-dup-attrs-2.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-2.nix rename to tests/functional/lang/parse-fail-dup-attrs-2.nix diff --git a/tests/lang/parse-fail-dup-attrs-3.err.exp b/tests/functional/lang/parse-fail-dup-attrs-3.err.exp similarity index 100% rename from tests/lang/parse-fail-dup-attrs-3.err.exp rename to tests/functional/lang/parse-fail-dup-attrs-3.err.exp diff --git a/tests/lang/parse-fail-dup-attrs-3.nix b/tests/functional/lang/parse-fail-dup-attrs-3.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-3.nix rename to tests/functional/lang/parse-fail-dup-attrs-3.nix diff --git a/tests/lang/parse-fail-dup-attrs-4.err.exp b/tests/functional/lang/parse-fail-dup-attrs-4.err.exp similarity index 100% rename from tests/lang/parse-fail-dup-attrs-4.err.exp rename to tests/functional/lang/parse-fail-dup-attrs-4.err.exp diff --git a/tests/lang/parse-fail-dup-attrs-4.nix b/tests/functional/lang/parse-fail-dup-attrs-4.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-4.nix rename to tests/functional/lang/parse-fail-dup-attrs-4.nix diff --git a/tests/lang/parse-fail-dup-attrs-7.err.exp b/tests/functional/lang/parse-fail-dup-attrs-7.err.exp similarity index 100% rename from tests/lang/parse-fail-dup-attrs-7.err.exp rename to tests/functional/lang/parse-fail-dup-attrs-7.err.exp diff --git a/tests/lang/parse-fail-dup-attrs-7.nix b/tests/functional/lang/parse-fail-dup-attrs-7.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-7.nix rename to tests/functional/lang/parse-fail-dup-attrs-7.nix diff --git a/tests/lang/parse-fail-dup-formals.err.exp b/tests/functional/lang/parse-fail-dup-formals.err.exp similarity index 100% rename from tests/lang/parse-fail-dup-formals.err.exp rename to tests/functional/lang/parse-fail-dup-formals.err.exp diff --git a/tests/lang/parse-fail-dup-formals.nix b/tests/functional/lang/parse-fail-dup-formals.nix similarity index 100% rename from tests/lang/parse-fail-dup-formals.nix rename to tests/functional/lang/parse-fail-dup-formals.nix diff --git a/tests/lang/parse-fail-eof-in-string.err.exp b/tests/functional/lang/parse-fail-eof-in-string.err.exp similarity index 100% rename from tests/lang/parse-fail-eof-in-string.err.exp rename to tests/functional/lang/parse-fail-eof-in-string.err.exp diff --git a/tests/lang/parse-fail-eof-in-string.nix b/tests/functional/lang/parse-fail-eof-in-string.nix similarity index 100% rename from tests/lang/parse-fail-eof-in-string.nix rename to tests/functional/lang/parse-fail-eof-in-string.nix diff --git a/tests/lang/parse-fail-mixed-nested-attrs1.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp similarity index 100% rename from tests/lang/parse-fail-mixed-nested-attrs1.err.exp rename to tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp diff --git a/tests/lang/parse-fail-mixed-nested-attrs1.nix b/tests/functional/lang/parse-fail-mixed-nested-attrs1.nix similarity index 100% rename from tests/lang/parse-fail-mixed-nested-attrs1.nix rename to tests/functional/lang/parse-fail-mixed-nested-attrs1.nix diff --git a/tests/lang/parse-fail-mixed-nested-attrs2.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp similarity index 100% rename from tests/lang/parse-fail-mixed-nested-attrs2.err.exp rename to tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp diff --git a/tests/lang/parse-fail-mixed-nested-attrs2.nix b/tests/functional/lang/parse-fail-mixed-nested-attrs2.nix similarity index 100% rename from tests/lang/parse-fail-mixed-nested-attrs2.nix rename to tests/functional/lang/parse-fail-mixed-nested-attrs2.nix diff --git a/tests/lang/parse-fail-patterns-1.err.exp b/tests/functional/lang/parse-fail-patterns-1.err.exp similarity index 100% rename from tests/lang/parse-fail-patterns-1.err.exp rename to tests/functional/lang/parse-fail-patterns-1.err.exp diff --git a/tests/lang/parse-fail-patterns-1.nix b/tests/functional/lang/parse-fail-patterns-1.nix similarity index 100% rename from tests/lang/parse-fail-patterns-1.nix rename to tests/functional/lang/parse-fail-patterns-1.nix diff --git a/tests/lang/parse-fail-regression-20060610.err.exp b/tests/functional/lang/parse-fail-regression-20060610.err.exp similarity index 100% rename from tests/lang/parse-fail-regression-20060610.err.exp rename to tests/functional/lang/parse-fail-regression-20060610.err.exp diff --git a/tests/lang/parse-fail-regression-20060610.nix b/tests/functional/lang/parse-fail-regression-20060610.nix similarity index 100% rename from tests/lang/parse-fail-regression-20060610.nix rename to tests/functional/lang/parse-fail-regression-20060610.nix diff --git a/tests/lang/parse-fail-undef-var-2.err.exp b/tests/functional/lang/parse-fail-undef-var-2.err.exp similarity index 100% rename from tests/lang/parse-fail-undef-var-2.err.exp rename to tests/functional/lang/parse-fail-undef-var-2.err.exp diff --git a/tests/lang/parse-fail-undef-var-2.nix b/tests/functional/lang/parse-fail-undef-var-2.nix similarity index 100% rename from tests/lang/parse-fail-undef-var-2.nix rename to tests/functional/lang/parse-fail-undef-var-2.nix diff --git a/tests/lang/parse-fail-undef-var.err.exp b/tests/functional/lang/parse-fail-undef-var.err.exp similarity index 100% rename from tests/lang/parse-fail-undef-var.err.exp rename to tests/functional/lang/parse-fail-undef-var.err.exp diff --git a/tests/lang/parse-fail-undef-var.nix b/tests/functional/lang/parse-fail-undef-var.nix similarity index 100% rename from tests/lang/parse-fail-undef-var.nix rename to tests/functional/lang/parse-fail-undef-var.nix diff --git a/tests/lang/parse-fail-utf8.err.exp b/tests/functional/lang/parse-fail-utf8.err.exp similarity index 100% rename from tests/lang/parse-fail-utf8.err.exp rename to tests/functional/lang/parse-fail-utf8.err.exp diff --git a/tests/lang/parse-fail-utf8.nix b/tests/functional/lang/parse-fail-utf8.nix similarity index 100% rename from tests/lang/parse-fail-utf8.nix rename to tests/functional/lang/parse-fail-utf8.nix diff --git a/tests/lang/parse-okay-1.exp b/tests/functional/lang/parse-okay-1.exp similarity index 100% rename from tests/lang/parse-okay-1.exp rename to tests/functional/lang/parse-okay-1.exp diff --git a/tests/lang/parse-okay-1.nix b/tests/functional/lang/parse-okay-1.nix similarity index 100% rename from tests/lang/parse-okay-1.nix rename to tests/functional/lang/parse-okay-1.nix diff --git a/tests/lang/parse-okay-crlf.exp b/tests/functional/lang/parse-okay-crlf.exp similarity index 100% rename from tests/lang/parse-okay-crlf.exp rename to tests/functional/lang/parse-okay-crlf.exp diff --git a/tests/lang/parse-okay-crlf.nix b/tests/functional/lang/parse-okay-crlf.nix similarity index 100% rename from tests/lang/parse-okay-crlf.nix rename to tests/functional/lang/parse-okay-crlf.nix diff --git a/tests/lang/parse-okay-dup-attrs-5.exp b/tests/functional/lang/parse-okay-dup-attrs-5.exp similarity index 100% rename from tests/lang/parse-okay-dup-attrs-5.exp rename to tests/functional/lang/parse-okay-dup-attrs-5.exp diff --git a/tests/lang/parse-okay-dup-attrs-5.nix b/tests/functional/lang/parse-okay-dup-attrs-5.nix similarity index 100% rename from tests/lang/parse-okay-dup-attrs-5.nix rename to tests/functional/lang/parse-okay-dup-attrs-5.nix diff --git a/tests/lang/parse-okay-dup-attrs-6.exp b/tests/functional/lang/parse-okay-dup-attrs-6.exp similarity index 100% rename from tests/lang/parse-okay-dup-attrs-6.exp rename to tests/functional/lang/parse-okay-dup-attrs-6.exp diff --git a/tests/lang/parse-okay-dup-attrs-6.nix b/tests/functional/lang/parse-okay-dup-attrs-6.nix similarity index 100% rename from tests/lang/parse-okay-dup-attrs-6.nix rename to tests/functional/lang/parse-okay-dup-attrs-6.nix diff --git a/tests/lang/parse-okay-mixed-nested-attrs-1.exp b/tests/functional/lang/parse-okay-mixed-nested-attrs-1.exp similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-1.exp rename to tests/functional/lang/parse-okay-mixed-nested-attrs-1.exp diff --git a/tests/lang/parse-okay-mixed-nested-attrs-1.nix b/tests/functional/lang/parse-okay-mixed-nested-attrs-1.nix similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-1.nix rename to tests/functional/lang/parse-okay-mixed-nested-attrs-1.nix diff --git a/tests/lang/parse-okay-mixed-nested-attrs-2.exp b/tests/functional/lang/parse-okay-mixed-nested-attrs-2.exp similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-2.exp rename to tests/functional/lang/parse-okay-mixed-nested-attrs-2.exp diff --git a/tests/lang/parse-okay-mixed-nested-attrs-2.nix b/tests/functional/lang/parse-okay-mixed-nested-attrs-2.nix similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-2.nix rename to tests/functional/lang/parse-okay-mixed-nested-attrs-2.nix diff --git a/tests/lang/parse-okay-mixed-nested-attrs-3.exp b/tests/functional/lang/parse-okay-mixed-nested-attrs-3.exp similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-3.exp rename to tests/functional/lang/parse-okay-mixed-nested-attrs-3.exp diff --git a/tests/lang/parse-okay-mixed-nested-attrs-3.nix b/tests/functional/lang/parse-okay-mixed-nested-attrs-3.nix similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-3.nix rename to tests/functional/lang/parse-okay-mixed-nested-attrs-3.nix diff --git a/tests/lang/parse-okay-regression-20041027.exp b/tests/functional/lang/parse-okay-regression-20041027.exp similarity index 100% rename from tests/lang/parse-okay-regression-20041027.exp rename to tests/functional/lang/parse-okay-regression-20041027.exp diff --git a/tests/lang/parse-okay-regression-20041027.nix b/tests/functional/lang/parse-okay-regression-20041027.nix similarity index 100% rename from tests/lang/parse-okay-regression-20041027.nix rename to tests/functional/lang/parse-okay-regression-20041027.nix diff --git a/tests/lang/parse-okay-regression-751.exp b/tests/functional/lang/parse-okay-regression-751.exp similarity index 100% rename from tests/lang/parse-okay-regression-751.exp rename to tests/functional/lang/parse-okay-regression-751.exp diff --git a/tests/lang/parse-okay-regression-751.nix b/tests/functional/lang/parse-okay-regression-751.nix similarity index 100% rename from tests/lang/parse-okay-regression-751.nix rename to tests/functional/lang/parse-okay-regression-751.nix diff --git a/tests/lang/parse-okay-subversion.exp b/tests/functional/lang/parse-okay-subversion.exp similarity index 100% rename from tests/lang/parse-okay-subversion.exp rename to tests/functional/lang/parse-okay-subversion.exp diff --git a/tests/lang/parse-okay-subversion.nix b/tests/functional/lang/parse-okay-subversion.nix similarity index 100% rename from tests/lang/parse-okay-subversion.nix rename to tests/functional/lang/parse-okay-subversion.nix diff --git a/tests/lang/parse-okay-url.exp b/tests/functional/lang/parse-okay-url.exp similarity index 100% rename from tests/lang/parse-okay-url.exp rename to tests/functional/lang/parse-okay-url.exp diff --git a/tests/lang/parse-okay-url.nix b/tests/functional/lang/parse-okay-url.nix similarity index 100% rename from tests/lang/parse-okay-url.nix rename to tests/functional/lang/parse-okay-url.nix diff --git a/tests/lang/readDir/bar b/tests/functional/lang/readDir/bar similarity index 100% rename from tests/lang/readDir/bar rename to tests/functional/lang/readDir/bar diff --git a/tests/lang/readDir/foo/git-hates-directories b/tests/functional/lang/readDir/foo/git-hates-directories similarity index 100% rename from tests/lang/readDir/foo/git-hates-directories rename to tests/functional/lang/readDir/foo/git-hates-directories diff --git a/tests/lang/readDir/ldir b/tests/functional/lang/readDir/ldir similarity index 100% rename from tests/lang/readDir/ldir rename to tests/functional/lang/readDir/ldir diff --git a/tests/lang/readDir/linked b/tests/functional/lang/readDir/linked similarity index 100% rename from tests/lang/readDir/linked rename to tests/functional/lang/readDir/linked diff --git a/tests/functional/lang/symlink-resolution/foo/lib/default.nix b/tests/functional/lang/symlink-resolution/foo/lib/default.nix new file mode 100644 index 00000000000..8b8441b91da --- /dev/null +++ b/tests/functional/lang/symlink-resolution/foo/lib/default.nix @@ -0,0 +1 @@ +"test" diff --git a/tests/functional/lang/symlink-resolution/foo/overlays b/tests/functional/lang/symlink-resolution/foo/overlays new file mode 120000 index 00000000000..0d44a21c508 --- /dev/null +++ b/tests/functional/lang/symlink-resolution/foo/overlays @@ -0,0 +1 @@ +../overlays \ No newline at end of file diff --git a/tests/functional/lang/symlink-resolution/overlays/overlay.nix b/tests/functional/lang/symlink-resolution/overlays/overlay.nix new file mode 100644 index 00000000000..b0368308e29 --- /dev/null +++ b/tests/functional/lang/symlink-resolution/overlays/overlay.nix @@ -0,0 +1 @@ +import ../lib diff --git a/tests/functional/legacy-ssh-store.sh b/tests/functional/legacy-ssh-store.sh new file mode 100644 index 00000000000..894efccd46b --- /dev/null +++ b/tests/functional/legacy-ssh-store.sh @@ -0,0 +1,4 @@ +source common.sh + +# Check that store info trusted doesn't yet work with ssh:// +nix --store ssh://localhost?remote-store=$TEST_ROOT/other-store store info --json | jq -e 'has("trusted") | not' diff --git a/tests/linux-sandbox-cert-test.nix b/tests/functional/linux-sandbox-cert-test.nix similarity index 100% rename from tests/linux-sandbox-cert-test.nix rename to tests/functional/linux-sandbox-cert-test.nix diff --git a/tests/linux-sandbox.sh b/tests/functional/linux-sandbox.sh similarity index 100% rename from tests/linux-sandbox.sh rename to tests/functional/linux-sandbox.sh diff --git a/tests/local-store.sh b/tests/functional/local-store.sh similarity index 79% rename from tests/local-store.sh rename to tests/functional/local-store.sh index 89502f864b9..f7c8eb3f1a8 100644 --- a/tests/local-store.sh +++ b/tests/functional/local-store.sh @@ -18,5 +18,5 @@ PATH2=$(nix path-info --store "$PWD/x" $CORRECT_PATH) PATH3=$(nix path-info --store "local?root=$PWD/x" $CORRECT_PATH) [ $CORRECT_PATH == $PATH3 ] -# Ensure store ping trusted works with local store -nix --store ./x store ping --json | jq -e '.trusted' +# Ensure store info trusted works with local store +nix --store ./x store info --json | jq -e '.trusted' diff --git a/tests/local.mk b/tests/functional/local.mk similarity index 82% rename from tests/local.mk rename to tests/functional/local.mk index 4edf31303a9..21dabca888d 100644 --- a/tests/local.mk +++ b/tests/functional/local.mk @@ -12,6 +12,7 @@ nix_tests = \ flakes/check.sh \ flakes/unlocked-override.sh \ flakes/absolute-paths.sh \ + flakes/absolute-attr-paths.sh \ flakes/build-paths.sh \ flakes/flake-in-submodule.sh \ gc.sh \ @@ -54,6 +55,7 @@ nix_tests = \ secure-drv-outputs.sh \ restricted.sh \ fetchGitSubmodules.sh \ + fetchGitVerification.sh \ flakes/search-root.sh \ readfile-context.sh \ nix-channel.sh \ @@ -103,7 +105,6 @@ nix_tests = \ case-hack.sh \ placeholders.sh \ ssh-relay.sh \ - plugins.sh \ build.sh \ build-delete.sh \ output-normalization.sh \ @@ -113,32 +114,40 @@ nix_tests = \ pass-as-file.sh \ nix-profile.sh \ suggestions.sh \ - store-ping.sh \ + store-info.sh \ fetchClosure.sh \ completions.sh \ flakes/show.sh \ impure-derivations.sh \ path-from-hash-part.sh \ - test-libstoreconsumer.sh \ + path-info.sh \ toString-path.sh \ read-only-store.sh \ - nested-sandboxing.sh + nested-sandboxing.sh \ + impure-env.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh endif +ifeq ($(ENABLE_BUILD), yes) + nix_tests += test-libstoreconsumer.sh + + ifeq ($(BUILD_SHARED_LIBS), 1) + nix_tests += plugins.sh + endif +endif + +$(d)/test-libstoreconsumer.sh.test $(d)/test-libstoreconsumer.sh.test-debug: \ + $(d)/test-libstoreconsumer/test-libstoreconsumer +$(d)/plugins.sh.test $(d)/plugins.sh.test-debug: \ + $(d)/plugins/libplugintest.$(SO_EXT) + install-tests += $(foreach x, $(nix_tests), $(d)/$(x)) -clean-files += \ +test-clean-files := \ $(d)/common/vars-and-functions.sh \ $(d)/config.nix -test-deps += \ - tests/common/vars-and-functions.sh \ - tests/config.nix \ - tests/test-libstoreconsumer/test-libstoreconsumer - -ifeq ($(BUILD_SHARED_LIBS), 1) - test-deps += tests/plugins/libplugintest.$(SO_EXT) -endif +clean-files += $(test-clean-files) +test-deps += $(test-clean-files) diff --git a/tests/logging.sh b/tests/functional/logging.sh similarity index 100% rename from tests/logging.sh rename to tests/functional/logging.sh diff --git a/tests/misc.sh b/tests/functional/misc.sh similarity index 100% rename from tests/misc.sh rename to tests/functional/misc.sh diff --git a/tests/multiple-outputs.nix b/tests/functional/multiple-outputs.nix similarity index 100% rename from tests/multiple-outputs.nix rename to tests/functional/multiple-outputs.nix diff --git a/tests/multiple-outputs.sh b/tests/functional/multiple-outputs.sh similarity index 100% rename from tests/multiple-outputs.sh rename to tests/functional/multiple-outputs.sh diff --git a/tests/nar-access.nix b/tests/functional/nar-access.nix similarity index 100% rename from tests/nar-access.nix rename to tests/functional/nar-access.nix diff --git a/tests/nar-access.sh b/tests/functional/nar-access.sh similarity index 90% rename from tests/nar-access.sh rename to tests/functional/nar-access.sh index d487d58d239..87981e7d902 100644 --- a/tests/nar-access.sh +++ b/tests/functional/nar-access.sh @@ -25,6 +25,11 @@ diff -u baz.cat-nar $storePath/foo/baz nix store cat $storePath/foo/baz > baz.cat-nar diff -u baz.cat-nar $storePath/foo/baz +# Check that 'nix store cat' fails on invalid store paths. +invalidPath="$(dirname $storePath)/99999999999999999999999999999999-foo" +cp -r $storePath $invalidPath +expect 1 nix store cat $invalidPath/foo/baz + # Test --json. diff -u \ <(nix nar ls --json $narFile / | jq -S) \ @@ -46,7 +51,7 @@ diff -u \ <(echo '{"type":"regular","size":0}' | jq -S) # Test missing files. -expect 1 nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR' +expect 1 nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist' expect 1 nix store ls $storePath/xyzzy 2>&1 | grep 'does not exist' # Test failure to dump. diff --git a/tests/nested-sandboxing.sh b/tests/functional/nested-sandboxing.sh similarity index 75% rename from tests/nested-sandboxing.sh rename to tests/functional/nested-sandboxing.sh index d9fa788aaa4..61fe043c6a8 100644 --- a/tests/nested-sandboxing.sh +++ b/tests/functional/nested-sandboxing.sh @@ -1,5 +1,5 @@ source common.sh -# This test is run by `tests/nested-sandboxing/runner.nix` in an extra layer of sandboxing. +# This test is run by `tests/functional/nested-sandboxing/runner.nix` in an extra layer of sandboxing. [[ -d /nix/store ]] || skipTest "running this test without Nix's deps being drawn from /nix/store is not yet supported" requireSandboxSupport diff --git a/tests/nested-sandboxing/command.sh b/tests/functional/nested-sandboxing/command.sh similarity index 100% rename from tests/nested-sandboxing/command.sh rename to tests/functional/nested-sandboxing/command.sh diff --git a/tests/nested-sandboxing/runner.nix b/tests/functional/nested-sandboxing/runner.nix similarity index 100% rename from tests/nested-sandboxing/runner.nix rename to tests/functional/nested-sandboxing/runner.nix diff --git a/tests/nix-build-examples.nix b/tests/functional/nix-build-examples.nix similarity index 100% rename from tests/nix-build-examples.nix rename to tests/functional/nix-build-examples.nix diff --git a/tests/nix-build.sh b/tests/functional/nix-build.sh similarity index 100% rename from tests/nix-build.sh rename to tests/functional/nix-build.sh diff --git a/tests/nix-channel.sh b/tests/functional/nix-channel.sh similarity index 100% rename from tests/nix-channel.sh rename to tests/functional/nix-channel.sh diff --git a/tests/nix-collect-garbage-d.sh b/tests/functional/nix-collect-garbage-d.sh similarity index 100% rename from tests/nix-collect-garbage-d.sh rename to tests/functional/nix-collect-garbage-d.sh diff --git a/tests/nix-copy-ssh-ng.sh b/tests/functional/nix-copy-ssh-ng.sh similarity index 91% rename from tests/nix-copy-ssh-ng.sh rename to tests/functional/nix-copy-ssh-ng.sh index 45e53c9c0e3..463b5e0c4bb 100644 --- a/tests/nix-copy-ssh-ng.sh +++ b/tests/functional/nix-copy-ssh-ng.sh @@ -9,7 +9,7 @@ rm -rf "$remoteRoot" outPath=$(nix-build --no-out-link dependencies.nix) -nix store ping --store "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" +nix store info --store "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" # Regression test for https://github.com/NixOS/nix/issues/6253 nix copy --to "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath --no-check-sigs & diff --git a/tests/nix-copy-ssh.sh b/tests/functional/nix-copy-ssh.sh similarity index 100% rename from tests/nix-copy-ssh.sh rename to tests/functional/nix-copy-ssh.sh diff --git a/tests/nix-daemon-untrusting.sh b/tests/functional/nix-daemon-untrusting.sh similarity index 100% rename from tests/nix-daemon-untrusting.sh rename to tests/functional/nix-daemon-untrusting.sh diff --git a/tests/nix-profile.sh b/tests/functional/nix-profile.sh similarity index 100% rename from tests/nix-profile.sh rename to tests/functional/nix-profile.sh diff --git a/tests/nix-shell.sh b/tests/functional/nix-shell.sh similarity index 96% rename from tests/nix-shell.sh rename to tests/functional/nix-shell.sh index edaa1249bce..13403fadb82 100644 --- a/tests/nix-shell.sh +++ b/tests/functional/nix-shell.sh @@ -84,6 +84,11 @@ chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby) [ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/spaced \'\''"shell.shebang.rb abc ruby' ] +# Test nix-shell shebang quoting +sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.nix > $TEST_ROOT/shell.shebang.nix +chmod a+rx $TEST_ROOT/shell.shebang.nix +$TEST_ROOT/shell.shebang.nix + # Test 'nix develop'. nix develop -f "$shellDotNix" shellDrv -c bash -c '[[ -n $stdenv ]]' diff --git a/tests/nix_path.sh b/tests/functional/nix_path.sh similarity index 100% rename from tests/nix_path.sh rename to tests/functional/nix_path.sh diff --git a/tests/optimise-store.sh b/tests/functional/optimise-store.sh similarity index 100% rename from tests/optimise-store.sh rename to tests/functional/optimise-store.sh diff --git a/tests/output-normalization.sh b/tests/functional/output-normalization.sh similarity index 100% rename from tests/output-normalization.sh rename to tests/functional/output-normalization.sh diff --git a/tests/parallel.builder.sh b/tests/functional/parallel.builder.sh similarity index 100% rename from tests/parallel.builder.sh rename to tests/functional/parallel.builder.sh diff --git a/tests/parallel.nix b/tests/functional/parallel.nix similarity index 100% rename from tests/parallel.nix rename to tests/functional/parallel.nix diff --git a/tests/parallel.sh b/tests/functional/parallel.sh similarity index 100% rename from tests/parallel.sh rename to tests/functional/parallel.sh diff --git a/tests/pass-as-file.sh b/tests/functional/pass-as-file.sh similarity index 100% rename from tests/pass-as-file.sh rename to tests/functional/pass-as-file.sh diff --git a/tests/path-from-hash-part.sh b/tests/functional/path-from-hash-part.sh similarity index 100% rename from tests/path-from-hash-part.sh rename to tests/functional/path-from-hash-part.sh diff --git a/tests/functional/path-info.sh b/tests/functional/path-info.sh new file mode 100644 index 00000000000..763935eb71e --- /dev/null +++ b/tests/functional/path-info.sh @@ -0,0 +1,23 @@ +source common.sh + +echo foo > $TEST_ROOT/foo +foo=$(nix store add-file $TEST_ROOT/foo) + +echo bar > $TEST_ROOT/bar +bar=$(nix store add-file $TEST_ROOT/bar) + +echo baz > $TEST_ROOT/baz +baz=$(nix store add-file $TEST_ROOT/baz) +nix-store --delete "$baz" + +diff --unified --color=always \ + <(nix path-info --json "$foo" "$bar" "$baz" | + jq --sort-keys 'map_values(.narHash)') \ + <(jq --sort-keys <<-EOF + { + "$foo": "sha256-QvtAMbUl/uvi+LCObmqOhvNOapHdA2raiI4xG5zI5pA=", + "$bar": "sha256-9fhYGu9fqxcQC2Kc81qh2RMo1QcLBUBo8U+pPn+jthQ=", + "$baz": null + } +EOF + ) diff --git a/tests/path.nix b/tests/functional/path.nix similarity index 100% rename from tests/path.nix rename to tests/functional/path.nix diff --git a/tests/placeholders.sh b/tests/functional/placeholders.sh similarity index 100% rename from tests/placeholders.sh rename to tests/functional/placeholders.sh diff --git a/tests/plugins.sh b/tests/functional/plugins.sh similarity index 100% rename from tests/plugins.sh rename to tests/functional/plugins.sh diff --git a/tests/plugins/local.mk b/tests/functional/plugins/local.mk similarity index 100% rename from tests/plugins/local.mk rename to tests/functional/plugins/local.mk diff --git a/tests/plugins/plugintest.cc b/tests/functional/plugins/plugintest.cc similarity index 100% rename from tests/plugins/plugintest.cc rename to tests/functional/plugins/plugintest.cc diff --git a/tests/post-hook.sh b/tests/functional/post-hook.sh similarity index 100% rename from tests/post-hook.sh rename to tests/functional/post-hook.sh diff --git a/tests/pure-eval.nix b/tests/functional/pure-eval.nix similarity index 100% rename from tests/pure-eval.nix rename to tests/functional/pure-eval.nix diff --git a/tests/pure-eval.sh b/tests/functional/pure-eval.sh similarity index 100% rename from tests/pure-eval.sh rename to tests/functional/pure-eval.sh diff --git a/tests/push-to-store-old.sh b/tests/functional/push-to-store-old.sh similarity index 100% rename from tests/push-to-store-old.sh rename to tests/functional/push-to-store-old.sh diff --git a/tests/push-to-store.sh b/tests/functional/push-to-store.sh similarity index 100% rename from tests/push-to-store.sh rename to tests/functional/push-to-store.sh diff --git a/tests/read-only-store.sh b/tests/functional/read-only-store.sh similarity index 100% rename from tests/read-only-store.sh rename to tests/functional/read-only-store.sh diff --git a/tests/readfile-context.nix b/tests/functional/readfile-context.nix similarity index 100% rename from tests/readfile-context.nix rename to tests/functional/readfile-context.nix diff --git a/tests/readfile-context.sh b/tests/functional/readfile-context.sh similarity index 100% rename from tests/readfile-context.sh rename to tests/functional/readfile-context.sh diff --git a/tests/recursive.nix b/tests/functional/recursive.nix similarity index 100% rename from tests/recursive.nix rename to tests/functional/recursive.nix diff --git a/tests/recursive.sh b/tests/functional/recursive.sh similarity index 100% rename from tests/recursive.sh rename to tests/functional/recursive.sh diff --git a/tests/referrers.sh b/tests/functional/referrers.sh similarity index 100% rename from tests/referrers.sh rename to tests/functional/referrers.sh diff --git a/tests/remote-store.sh b/tests/functional/remote-store.sh similarity index 78% rename from tests/remote-store.sh rename to tests/functional/remote-store.sh index 7649964ef40..dc80f8b5555 100644 --- a/tests/remote-store.sh +++ b/tests/functional/remote-store.sh @@ -5,17 +5,17 @@ clearStore # Ensure "fake ssh" remote store works just as legacy fake ssh would. nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store doctor -# Ensure that store ping trusted works with ssh-ng:// -nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store store ping --json | jq -e '.trusted' +# Ensure that store info trusted works with ssh-ng:// +nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store store info --json | jq -e '.trusted' startDaemon if isDaemonNewer "2.15pre0"; then # Ensure that ping works trusted with new daemon - nix store ping --json | jq -e '.trusted' + nix store info --json | jq -e '.trusted' else # And the the field is absent with the old daemon - nix store ping --json | jq -e 'has("trusted") | not' + nix store info --json | jq -e 'has("trusted") | not' fi # Test import-from-derivation through the daemon. diff --git a/tests/repair.sh b/tests/functional/repair.sh similarity index 100% rename from tests/repair.sh rename to tests/functional/repair.sh diff --git a/tests/repl.sh b/tests/functional/repl.sh similarity index 94% rename from tests/repl.sh rename to tests/functional/repl.sh index bb8b60e5013..1b779c1f56d 100644 --- a/tests/repl.sh +++ b/tests/functional/repl.sh @@ -105,18 +105,13 @@ testReplResponseNoRegex ' testReplResponse ' drvPath ' '".*-simple.drv"' \ -$testDir/simple.nix +--file $testDir/simple.nix testReplResponse ' drvPath ' '".*-simple.drv"' \ --file $testDir/simple.nix --experimental-features 'ca-derivations' -testReplResponse ' -drvPath -' '".*-simple.drv"' \ ---file $testDir/simple.nix --extra-experimental-features 'repl-flake ca-derivations' - mkdir -p flake && cat < flake/flake.nix { outputs = { self }: { @@ -130,7 +125,7 @@ EOF testReplResponse ' foo + baz ' "3" \ - ./flake ./flake\#bar --experimental-features 'flakes repl-flake' + ./flake ./flake\#bar --experimental-features 'flakes' # Test the `:reload` mechansim with flakes: # - Eval `./flake#changingThing` @@ -143,7 +138,7 @@ sleep 1 # Leave the repl the time to eval 'foo' sed -i 's/beforeChange/afterChange/' flake/flake.nix echo ":reload" echo "changingThing" -) | nix repl ./flake --experimental-features 'flakes repl-flake') +) | nix repl ./flake --experimental-features 'flakes') echo "$replResult" | grepQuiet -s beforeChange echo "$replResult" | grepQuiet -s afterChange diff --git a/tests/restricted.nix b/tests/functional/restricted.nix similarity index 100% rename from tests/restricted.nix rename to tests/functional/restricted.nix diff --git a/tests/restricted.sh b/tests/functional/restricted.sh similarity index 95% rename from tests/restricted.sh rename to tests/functional/restricted.sh index 17f310a4b5b..197ae7a10fb 100644 --- a/tests/restricted.sh +++ b/tests/functional/restricted.sh @@ -9,10 +9,10 @@ nix-instantiate --restrict-eval ./simple.nix -I src=. nix-instantiate --restrict-eval ./simple.nix -I src1=simple.nix -I src2=config.nix -I src3=./simple.builder.sh (! nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix') -nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=.. +nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=../.. -(! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel') -nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel' -I src=../src +(! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel') +nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel' -I src=../../src (! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ') nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ' -I src=. diff --git a/tests/search.nix b/tests/functional/search.nix similarity index 100% rename from tests/search.nix rename to tests/functional/search.nix diff --git a/tests/search.sh b/tests/functional/search.sh similarity index 100% rename from tests/search.sh rename to tests/functional/search.sh diff --git a/tests/secure-drv-outputs.nix b/tests/functional/secure-drv-outputs.nix similarity index 100% rename from tests/secure-drv-outputs.nix rename to tests/functional/secure-drv-outputs.nix diff --git a/tests/secure-drv-outputs.sh b/tests/functional/secure-drv-outputs.sh similarity index 100% rename from tests/secure-drv-outputs.sh rename to tests/functional/secure-drv-outputs.sh diff --git a/tests/selfref-gc.sh b/tests/functional/selfref-gc.sh similarity index 100% rename from tests/selfref-gc.sh rename to tests/functional/selfref-gc.sh diff --git a/tests/shell-hello.nix b/tests/functional/shell-hello.nix similarity index 100% rename from tests/shell-hello.nix rename to tests/functional/shell-hello.nix diff --git a/tests/shell.nix b/tests/functional/shell.nix similarity index 100% rename from tests/shell.nix rename to tests/functional/shell.nix diff --git a/tests/shell.sh b/tests/functional/shell.sh similarity index 100% rename from tests/shell.sh rename to tests/functional/shell.sh diff --git a/tests/functional/shell.shebang.nix b/tests/functional/shell.shebang.nix new file mode 100755 index 00000000000..08e43d53c46 --- /dev/null +++ b/tests/functional/shell.shebang.nix @@ -0,0 +1,10 @@ +#! @ENV_PROG@ nix-shell +#! nix-shell -I nixpkgs=shell.nix --no-substitute +#! nix-shell --argstr s1 'foo "bar" \baz'"'"'qux' --argstr s2 "foo 'bar' \"\baz" --argstr s3 \foo\ bar\'baz --argstr s4 '' +#! nix-shell shell.shebang.nix --command true +{ s1, s2, s3, s4 }: +assert s1 == ''foo "bar" \baz'qux''; +assert s2 == "foo 'bar' \"baz"; +assert s3 == "foo bar'baz"; +assert s4 == ""; +(import {}).runCommand "nix-shell" {} "" diff --git a/tests/shell.shebang.rb b/tests/functional/shell.shebang.rb similarity index 100% rename from tests/shell.shebang.rb rename to tests/functional/shell.shebang.rb diff --git a/tests/shell.shebang.sh b/tests/functional/shell.shebang.sh similarity index 100% rename from tests/shell.shebang.sh rename to tests/functional/shell.shebang.sh diff --git a/tests/signing.sh b/tests/functional/signing.sh similarity index 100% rename from tests/signing.sh rename to tests/functional/signing.sh diff --git a/tests/simple-failing.nix b/tests/functional/simple-failing.nix similarity index 100% rename from tests/simple-failing.nix rename to tests/functional/simple-failing.nix diff --git a/tests/simple.builder.sh b/tests/functional/simple.builder.sh similarity index 100% rename from tests/simple.builder.sh rename to tests/functional/simple.builder.sh diff --git a/tests/simple.nix b/tests/functional/simple.nix similarity index 100% rename from tests/simple.nix rename to tests/functional/simple.nix diff --git a/tests/simple.sh b/tests/functional/simple.sh similarity index 100% rename from tests/simple.sh rename to tests/functional/simple.sh diff --git a/tests/ssh-relay.sh b/tests/functional/ssh-relay.sh similarity index 100% rename from tests/ssh-relay.sh rename to tests/functional/ssh-relay.sh diff --git a/tests/store-ping.sh b/tests/functional/store-info.sh similarity index 69% rename from tests/store-ping.sh rename to tests/functional/store-info.sh index 9846c7d3d12..c002e50beba 100644 --- a/tests/store-ping.sh +++ b/tests/functional/store-info.sh @@ -1,7 +1,7 @@ source common.sh -STORE_INFO=$(nix store ping 2>&1) -STORE_INFO_JSON=$(nix store ping --json) +STORE_INFO=$(nix store info 2>&1) +STORE_INFO_JSON=$(nix store info --json) echo "$STORE_INFO" | grep "Store URL: ${NIX_REMOTE}" @@ -11,7 +11,7 @@ if [[ -v NIX_DAEMON_PACKAGE ]] && isDaemonNewer "2.7.0pre20220126"; then [[ "$(echo "$STORE_INFO_JSON" | jq -r ".version")" == "$DAEMON_VERSION" ]] fi -expect 127 NIX_REMOTE=unix:$PWD/store nix store ping || \ - fail "nix store ping on a non-existent store should fail" +expect 127 NIX_REMOTE=unix:$PWD/store nix store info || \ + fail "nix store info on a non-existent store should fail" [[ "$(echo "$STORE_INFO_JSON" | jq -r ".url")" == "${NIX_REMOTE:-local}" ]] diff --git a/tests/structured-attrs-shell.nix b/tests/functional/structured-attrs-shell.nix similarity index 100% rename from tests/structured-attrs-shell.nix rename to tests/functional/structured-attrs-shell.nix diff --git a/tests/structured-attrs.nix b/tests/functional/structured-attrs.nix similarity index 100% rename from tests/structured-attrs.nix rename to tests/functional/structured-attrs.nix diff --git a/tests/structured-attrs.sh b/tests/functional/structured-attrs.sh similarity index 50% rename from tests/structured-attrs.sh rename to tests/functional/structured-attrs.sh index 378dbc73548..f11992dcd44 100644 --- a/tests/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -15,9 +15,21 @@ nix-build structured-attrs.nix -A all -o $TEST_ROOT/result export NIX_BUILD_SHELL=$SHELL env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \ - --run 'test -e .attrs.json; test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' + --run 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' + +nix develop -f structured-attrs-shell.nix -c bash -c 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' # `nix develop` is a slightly special way of dealing with environment vars, it parses # these from a shell-file exported from a derivation. This is to test especially `outputs` # (which is an associative array in thsi case) being fine. nix develop -f structured-attrs-shell.nix -c bash -c 'test -n "$out"' + +nix print-dev-env -f structured-attrs-shell.nix | grepQuiet 'NIX_ATTRS_JSON_FILE=' +nix print-dev-env -f structured-attrs-shell.nix | grepQuiet 'NIX_ATTRS_SH_FILE=' +nix print-dev-env -f shell.nix shellDrv | grepQuietInverse 'NIX_ATTRS_SH_FILE' + +jsonOut="$(nix print-dev-env -f structured-attrs-shell.nix --json)" + +test "$(<<<"$jsonOut" jq '.structuredAttrs|keys|.[]' -r)" = "$(printf ".attrs.json\n.attrs.sh")" + +test "$(<<<"$jsonOut" jq '.variables.out.value' -r)" = "$(<<<"$jsonOut" jq '.structuredAttrs.".attrs.json"' -r | jq -r '.outputs.out')" diff --git a/tests/substitute-with-invalid-ca.sh b/tests/functional/substitute-with-invalid-ca.sh similarity index 100% rename from tests/substitute-with-invalid-ca.sh rename to tests/functional/substitute-with-invalid-ca.sh diff --git a/tests/suggestions.sh b/tests/functional/suggestions.sh similarity index 100% rename from tests/suggestions.sh rename to tests/functional/suggestions.sh diff --git a/tests/supplementary-groups.sh b/tests/functional/supplementary-groups.sh similarity index 100% rename from tests/supplementary-groups.sh rename to tests/functional/supplementary-groups.sh diff --git a/tests/tarball.sh b/tests/functional/tarball.sh similarity index 94% rename from tests/tarball.sh rename to tests/functional/tarball.sh index 6e621a28c65..e59ee400ee0 100644 --- a/tests/tarball.sh +++ b/tests/functional/tarball.sh @@ -18,7 +18,7 @@ test_tarball() { local compressor="$2" tarball=$TEST_ROOT/tarball.tar$ext - (cd $TEST_ROOT && tar cf - tarball) | $compressor > $tarball + (cd $TEST_ROOT && GNUTAR_REPRODUCIBLE= tar --mtime=$tarroot/default.nix --owner=0 --group=0 --numeric-owner --sort=name -c -f - tarball) | $compressor > $tarball nix-env -f file://$tarball -qa --out-path | grepQuiet dependencies diff --git a/tests/test-infra.sh b/tests/functional/test-infra.sh similarity index 100% rename from tests/test-infra.sh rename to tests/functional/test-infra.sh diff --git a/tests/test-libstoreconsumer.sh b/tests/functional/test-libstoreconsumer.sh similarity index 100% rename from tests/test-libstoreconsumer.sh rename to tests/functional/test-libstoreconsumer.sh diff --git a/tests/test-libstoreconsumer/README.md b/tests/functional/test-libstoreconsumer/README.md similarity index 100% rename from tests/test-libstoreconsumer/README.md rename to tests/functional/test-libstoreconsumer/README.md diff --git a/tests/test-libstoreconsumer/local.mk b/tests/functional/test-libstoreconsumer/local.mk similarity index 100% rename from tests/test-libstoreconsumer/local.mk rename to tests/functional/test-libstoreconsumer/local.mk diff --git a/tests/test-libstoreconsumer/main.cc b/tests/functional/test-libstoreconsumer/main.cc similarity index 100% rename from tests/test-libstoreconsumer/main.cc rename to tests/functional/test-libstoreconsumer/main.cc diff --git a/tests/timeout.nix b/tests/functional/timeout.nix similarity index 100% rename from tests/timeout.nix rename to tests/functional/timeout.nix diff --git a/tests/timeout.sh b/tests/functional/timeout.sh similarity index 100% rename from tests/timeout.sh rename to tests/functional/timeout.sh diff --git a/tests/toString-path.sh b/tests/functional/toString-path.sh similarity index 100% rename from tests/toString-path.sh rename to tests/functional/toString-path.sh diff --git a/tests/undefined-variable.nix b/tests/functional/undefined-variable.nix similarity index 100% rename from tests/undefined-variable.nix rename to tests/functional/undefined-variable.nix diff --git a/tests/user-envs-migration.sh b/tests/functional/user-envs-migration.sh similarity index 100% rename from tests/user-envs-migration.sh rename to tests/functional/user-envs-migration.sh diff --git a/tests/user-envs.builder.sh b/tests/functional/user-envs.builder.sh similarity index 100% rename from tests/user-envs.builder.sh rename to tests/functional/user-envs.builder.sh diff --git a/tests/user-envs.nix b/tests/functional/user-envs.nix similarity index 100% rename from tests/user-envs.nix rename to tests/functional/user-envs.nix diff --git a/tests/user-envs.sh b/tests/functional/user-envs.sh similarity index 100% rename from tests/user-envs.sh rename to tests/functional/user-envs.sh diff --git a/tests/why-depends.sh b/tests/functional/why-depends.sh similarity index 100% rename from tests/why-depends.sh rename to tests/functional/why-depends.sh diff --git a/tests/zstd.sh b/tests/functional/zstd.sh similarity index 100% rename from tests/zstd.sh rename to tests/functional/zstd.sh diff --git a/tests/installer/default.nix b/tests/installer/default.nix index 49cfd2bccba..238c6ac8e8d 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -213,7 +213,7 @@ let source /etc/bashrc || true nix-env --version - nix --extra-experimental-features nix-command store ping + nix --extra-experimental-features nix-command store info out=\$(nix-build --no-substitute -E 'derivation { name = "foo"; system = "x86_64-linux"; builder = "/bin/sh"; args = ["-c" "echo foobar > \$out"]; }') [[ \$(cat \$out) = foobar ]] diff --git a/tests/lang/eval-fail-antiquoted-path.err.exp b/tests/lang/eval-fail-antiquoted-path.err.exp deleted file mode 100644 index 425deba4225..00000000000 --- a/tests/lang/eval-fail-antiquoted-path.err.exp +++ /dev/null @@ -1 +0,0 @@ -error: getting attributes of path ‘PWD/lang/fnord’: No such file or directory diff --git a/tests/lang/eval-fail-bad-antiquote-1.err.exp b/tests/lang/eval-fail-bad-antiquote-1.err.exp deleted file mode 100644 index cf94f53bc55..00000000000 --- a/tests/lang/eval-fail-bad-antiquote-1.err.exp +++ /dev/null @@ -1,10 +0,0 @@ -error: - … while evaluating a path segment - - at /pwd/lang/eval-fail-bad-antiquote-1.nix:1:2: - - 1| "${x: x}" - | ^ - 2| - - error: cannot coerce a function to a string diff --git a/tests/lang/eval-fail-bad-antiquote-2.err.exp b/tests/lang/eval-fail-bad-antiquote-2.err.exp deleted file mode 100644 index c8fe39d1214..00000000000 --- a/tests/lang/eval-fail-bad-antiquote-2.err.exp +++ /dev/null @@ -1 +0,0 @@ -error: operation 'addToStoreFromDump' is not supported by store 'dummy' diff --git a/tests/lang/eval-fail-bad-antiquote-3.err.exp b/tests/lang/eval-fail-bad-antiquote-3.err.exp deleted file mode 100644 index fbefbc8267f..00000000000 --- a/tests/lang/eval-fail-bad-antiquote-3.err.exp +++ /dev/null @@ -1,10 +0,0 @@ -error: - … while evaluating a path segment - - at /pwd/lang/eval-fail-bad-antiquote-3.nix:1:3: - - 1| ''${x: x}'' - | ^ - 2| - - error: cannot coerce a function to a string diff --git a/tests/lang/eval-fail-bad-string-interpolation-2.err.exp b/tests/lang/eval-fail-bad-string-interpolation-2.err.exp deleted file mode 100644 index c8fe39d1214..00000000000 --- a/tests/lang/eval-fail-bad-string-interpolation-2.err.exp +++ /dev/null @@ -1 +0,0 @@ -error: operation 'addToStoreFromDump' is not supported by store 'dummy' diff --git a/tests/lang/eval-fail-dup-dynamic-attrs.err.exp b/tests/lang/eval-fail-dup-dynamic-attrs.err.exp deleted file mode 100644 index e01f8e6d0ea..00000000000 --- a/tests/lang/eval-fail-dup-dynamic-attrs.err.exp +++ /dev/null @@ -1,8 +0,0 @@ -error: dynamic attribute 'b' already defined at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:11 - - at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:3:11: - - 2| set = { "${"" + "b"}" = 1; }; - 3| set = { "${"b" + ""}" = 2; }; - | ^ - 4| } diff --git a/tests/lang/eval-fail-nonexist-path.err.exp b/tests/lang/eval-fail-nonexist-path.err.exp deleted file mode 100644 index c8fe39d1214..00000000000 --- a/tests/lang/eval-fail-nonexist-path.err.exp +++ /dev/null @@ -1 +0,0 @@ -error: operation 'addToStoreFromDump' is not supported by store 'dummy' diff --git a/tests/lang/eval-okay-context-introspection.exp b/tests/lang/eval-okay-context-introspection.exp deleted file mode 100644 index 03b400cc886..00000000000 --- a/tests/lang/eval-okay-context-introspection.exp +++ /dev/null @@ -1 +0,0 @@ -[ true true true true true true ] diff --git a/tests/lang/eval-okay-path.exp b/tests/lang/eval-okay-path.exp deleted file mode 100644 index 3ce7f828305..00000000000 --- a/tests/lang/eval-okay-path.exp +++ /dev/null @@ -1 +0,0 @@ -"/nix/store/ya937r4ydw0l6kayq8jkyqaips9c75jm-output" diff --git a/tests/lang/eval-okay-path.nix b/tests/lang/eval-okay-path.nix deleted file mode 100644 index e67168cf3ed..00000000000 --- a/tests/lang/eval-okay-path.nix +++ /dev/null @@ -1,7 +0,0 @@ -builtins.path - { path = ./.; - filter = path: _: baseNameOf path == "data"; - recursive = true; - sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw"; - name = "output"; - } diff --git a/tests/lang/eval-okay-pathexists.nix b/tests/lang/eval-okay-pathexists.nix deleted file mode 100644 index e1246e370a0..00000000000 --- a/tests/lang/eval-okay-pathexists.nix +++ /dev/null @@ -1,8 +0,0 @@ -builtins.pathExists (./lib.nix) -&& builtins.pathExists (builtins.toPath ./lib.nix) -&& builtins.pathExists (builtins.toString ./lib.nix) -&& !builtins.pathExists (builtins.toString ./lib.nix + "/") -&& builtins.pathExists (builtins.toPath (builtins.toString ./lib.nix)) -&& !builtins.pathExists (builtins.toPath (builtins.toString ./bla.nix)) -&& builtins.pathExists ./lib.nix -&& !builtins.pathExists ./bla.nix diff --git a/tests/lang/parse-fail-dup-attrs-6.err.exp b/tests/lang/parse-fail-dup-attrs-6.err.exp deleted file mode 100644 index 74823fc2513..00000000000 --- a/tests/lang/parse-fail-dup-attrs-6.err.exp +++ /dev/null @@ -1 +0,0 @@ -error: attribute ‘services.ssh’ at (string):3:3 already defined at (string):2:3 diff --git a/tests/legacy-ssh-store.sh b/tests/legacy-ssh-store.sh deleted file mode 100644 index 71b716b8453..00000000000 --- a/tests/legacy-ssh-store.sh +++ /dev/null @@ -1,4 +0,0 @@ -source common.sh - -# Check that store ping trusted doesn't yet work with ssh:// -nix --store ssh://localhost?remote-store=$TEST_ROOT/other-store store ping --json | jq -e 'has("trusted") | not' diff --git a/tests/nixos/containers/containers.nix b/tests/nixos/containers/containers.nix index e721be48f58..c8ee78a4a58 100644 --- a/tests/nixos/containers/containers.nix +++ b/tests/nixos/containers/containers.nix @@ -56,8 +56,8 @@ host.fail("nix build -v --auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-6 --arg uidRange true") # Run systemd-nspawn in a Nix build. - #host.succeed("nix build -v --auto-allocate-uids --sandbox -L --offline --impure --file ${./systemd-nspawn.nix} --argstr nixpkgs ${nixpkgs}") - #host.succeed("[[ $(cat ./result/msg) = 'Hello World' ]]") + host.succeed("nix build -v --auto-allocate-uids --sandbox -L --offline --impure --file ${./systemd-nspawn.nix} --argstr nixpkgs ${nixpkgs}") + host.succeed("[[ $(cat ./result/msg) = 'Hello World' ]]") ''; } diff --git a/tests/nixos/containers/systemd-nspawn.nix b/tests/nixos/containers/systemd-nspawn.nix index f54f32f2af5..1dad4ebd754 100644 --- a/tests/nixos/containers/systemd-nspawn.nix +++ b/tests/nixos/containers/systemd-nspawn.nix @@ -73,6 +73,8 @@ runCommand "test" --resolv-conf=off \ --bind-ro=/nix/store \ --bind=$out \ + --bind=/proc:/run/host/proc \ + --bind=/sys:/run/host/sys \ --private-network \ $toplevel/init '' diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix new file mode 100644 index 00000000000..4459aa66495 --- /dev/null +++ b/tests/nixos/default.nix @@ -0,0 +1,43 @@ +{ lib, nixpkgs, nixpkgsFor }: + +let + + nixos-lib = import (nixpkgs + "/nixos/lib") { }; + + # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests + runNixOSTestFor = system: test: nixos-lib.runTest { + imports = [ test ]; + hostPkgs = nixpkgsFor.${system}.native; + defaults = { + nixpkgs.pkgs = nixpkgsFor.${system}.native; + }; + _module.args.nixpkgs = nixpkgs; + }; + +in + +{ + authorization = runNixOSTestFor "x86_64-linux" ./authorization.nix; + + remoteBuilds = runNixOSTestFor "x86_64-linux" ./remote-builds.nix; + + remoteBuildsSshNg = runNixOSTestFor "x86_64-linux" ./remote-builds-ssh-ng.nix; + + nix-copy-closure = runNixOSTestFor "x86_64-linux" ./nix-copy-closure.nix; + + nix-copy = runNixOSTestFor "x86_64-linux" ./nix-copy.nix; + + nssPreload = runNixOSTestFor "x86_64-linux" ./nss-preload.nix; + + githubFlakes = runNixOSTestFor "x86_64-linux" ./github-flakes.nix; + + sourcehutFlakes = runNixOSTestFor "x86_64-linux" ./sourcehut-flakes.nix; + + tarballFlakes = runNixOSTestFor "x86_64-linux" ./tarball-flakes.nix; + + containers = runNixOSTestFor "x86_64-linux" ./containers/containers.nix; + + setuid = lib.genAttrs + ["i686-linux" "x86_64-linux"] + (system: runNixOSTestFor system ./setuid.nix); +} diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix index 6de702d17c3..62ae8871b87 100644 --- a/tests/nixos/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -186,6 +186,10 @@ in client.succeed("nix registry pin nixpkgs") client.succeed("nix flake metadata nixpkgs --tarball-ttl 0 >&2") + # Test fetchTree on a github URL. + hash = client.succeed(f"nix eval --raw --expr '(fetchTree {info['url']}).narHash'") + assert hash == info['locked']['narHash'] + # Shut down the web server. The flake should be cached on the client. github.succeed("systemctl stop httpd.service") diff --git a/tests/nixos/remote-builds-ssh-ng.nix b/tests/nixos/remote-builds-ssh-ng.nix new file mode 100644 index 00000000000..b59dde9bf1d --- /dev/null +++ b/tests/nixos/remote-builds-ssh-ng.nix @@ -0,0 +1,108 @@ +{ config, lib, hostPkgs, ... }: + +let + pkgs = config.nodes.client.nixpkgs.pkgs; + + # Trivial Nix expression to build remotely. + expr = config: nr: pkgs.writeText "expr.nix" + '' + let utils = builtins.storePath ${config.system.build.extraUtils}; in + derivation { + name = "hello-${toString nr}"; + system = "i686-linux"; + PATH = "''${utils}/bin"; + builder = "''${utils}/bin/sh"; + args = [ "-c" "${ + lib.concatStringsSep "; " [ + ''if [[ -n $NIX_LOG_FD ]]'' + ''then echo '@nix {\"action\":\"setPhase\",\"phase\":\"buildPhase\"}' >&''$NIX_LOG_FD'' + "fi" + "echo Hello" + "mkdir $out" + "cat /proc/sys/kernel/hostname > $out/host" + ] + }" ]; + outputs = [ "out" ]; + } + ''; +in + +{ + name = "remote-builds-ssh-ng"; + + nodes = + { builder = + { config, pkgs, ... }: + { services.openssh.enable = true; + virtualisation.writableStore = true; + nix.settings.sandbox = true; + nix.settings.substituters = lib.mkForce [ ]; + }; + + client = + { config, lib, pkgs, ... }: + { nix.settings.max-jobs = 0; # force remote building + nix.distributedBuilds = true; + nix.buildMachines = + [ { hostName = "builder"; + sshUser = "root"; + sshKey = "/root/.ssh/id_ed25519"; + system = "i686-linux"; + maxJobs = 1; + protocol = "ssh-ng"; + } + ]; + virtualisation.writableStore = true; + virtualisation.additionalPaths = [ config.system.build.extraUtils ]; + nix.settings.substituters = lib.mkForce [ ]; + programs.ssh.extraConfig = "ConnectTimeout 30"; + }; + }; + + testScript = { nodes }: '' + # fmt: off + import subprocess + + start_all() + + # Create an SSH key on the client. + subprocess.run([ + "${hostPkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) + client.succeed("mkdir -p -m 700 /root/.ssh") + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") + + # Install the SSH key on the builder. + client.wait_for_unit("network.target") + builder.succeed("mkdir -p -m 700 /root/.ssh") + builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + builder.wait_for_unit("sshd") + client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") + + # Perform a build + out = client.succeed("nix-build ${expr nodes.client.config 1} 2> build-output") + + # Verify that the build was done on the builder + builder.succeed(f"test -e {out.strip()}") + + # Print the build log, prefix the log lines to avoid nix intercepting lines starting with @nix + buildOutput = client.succeed("sed -e 's/^/build-output:/' build-output") + print(buildOutput) + + # Make sure that we get the expected build output + client.succeed("grep -qF Hello build-output") + + # We don't want phase reporting in the build output + client.fail("grep -qF '@nix' build-output") + + # Get the log file + client.succeed(f"nix-store --read-log {out.strip()} > log-output") + # Prefix the log lines to avoid nix intercepting lines starting with @nix + logOutput = client.succeed("sed -e 's/^/log-file:/' log-output") + print(logOutput) + + # Check that we get phase reporting in the log file + client.succeed("grep -q '@nix {\"action\":\"setPhase\",\"phase\":\"buildPhase\"}' log-output") + ''; +} diff --git a/tests/unit/libexpr-support/local.mk b/tests/unit/libexpr-support/local.mk new file mode 100644 index 00000000000..12a76206ace --- /dev/null +++ b/tests/unit/libexpr-support/local.mk @@ -0,0 +1,23 @@ +libraries += libexpr-test-support + +libexpr-test-support_NAME = libnixexpr-test-support + +libexpr-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libexpr-test-support_INSTALL_DIR := $(checklibdir) +else + libexpr-test-support_INSTALL_DIR := +endif + +libexpr-test-support_SOURCES := \ + $(wildcard $(d)/tests/*.cc) \ + $(wildcard $(d)/tests/value/*.cc) + +libexpr-test-support_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) + +libexpr-test-support_LIBS = \ + libstore-test-support libutil-test-support \ + libexpr libstore libutil + +libexpr-test-support_LDFLAGS := -pthread -lrapidcheck diff --git a/src/libexpr/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh similarity index 90% rename from src/libexpr/tests/libexpr.hh rename to tests/unit/libexpr-support/tests/libexpr.hh index b8e65aafe43..9684314469e 100644 --- a/src/libexpr/tests/libexpr.hh +++ b/tests/unit/libexpr-support/tests/libexpr.hh @@ -71,7 +71,7 @@ namespace nix { if (arg.type() != nString) { return false; } - return std::string_view(arg.string.s) == s; + return std::string_view(arg.c_str()) == s; } MATCHER_P(IsIntEq, v, fmt("The string is equal to \"%1%\"", v)) { @@ -103,14 +103,17 @@ namespace nix { } MATCHER_P(IsPathEq, p, fmt("Is a path equal to \"%1%\"", p)) { - if (arg.type() != nPath) { - *result_listener << "Expected a path got " << arg.type(); - return false; - } else if (std::string_view(arg.string.s) != p) { - *result_listener << "Expected a path that equals \"" << p << "\" but got: " << arg.string.s; + if (arg.type() != nPath) { + *result_listener << "Expected a path got " << arg.type(); + return false; + } else { + auto path = arg.path(); + if (path.path != CanonPath(p)) { + *result_listener << "Expected a path that equals \"" << p << "\" but got: " << path.path; return false; } - return true; + } + return true; } diff --git a/tests/unit/libexpr-support/tests/value/context.cc b/tests/unit/libexpr-support/tests/value/context.cc new file mode 100644 index 00000000000..8658bdaef16 --- /dev/null +++ b/tests/unit/libexpr-support/tests/value/context.cc @@ -0,0 +1,30 @@ +#include + +#include "tests/path.hh" +#include "tests/value/context.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + return gen::just(NixStringContextElem::DrvDeep { + .drvPath = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + case 2: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + +} diff --git a/src/libexpr/tests/value/context.hh b/tests/unit/libexpr-support/tests/value/context.hh similarity index 95% rename from src/libexpr/tests/value/context.hh rename to tests/unit/libexpr-support/tests/value/context.hh index c0bc97ba3c4..8c68c78bbd1 100644 --- a/src/libexpr/tests/value/context.hh +++ b/tests/unit/libexpr-support/tests/value/context.hh @@ -3,7 +3,7 @@ #include -#include +#include "value/context.hh" namespace rc { using namespace nix; diff --git a/src/libexpr/tests/derived-path.cc b/tests/unit/libexpr/derived-path.cc similarity index 100% rename from src/libexpr/tests/derived-path.cc rename to tests/unit/libexpr/derived-path.cc diff --git a/src/libexpr/tests/error_traces.cc b/tests/unit/libexpr/error_traces.cc similarity index 99% rename from src/libexpr/tests/error_traces.cc rename to tests/unit/libexpr/error_traces.cc index 285651256ca..81498f65a3f 100644 --- a/src/libexpr/tests/error_traces.cc +++ b/tests/unit/libexpr/error_traces.cc @@ -310,7 +310,7 @@ namespace nix { ASSERT_TRACE2("storePath true", TypeError, hintfmt("cannot coerce %s to a string", "a Boolean"), - hintfmt("while evaluating the first argument passed to builtins.storePath")); + hintfmt("while evaluating the first argument passed to 'builtins.storePath'")); } @@ -378,12 +378,12 @@ namespace nix { ASSERT_TRACE2("filterSource [] []", TypeError, hintfmt("cannot coerce %s to a string", "a list"), - hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource")); + hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); ASSERT_TRACE2("filterSource [] \"foo\"", EvalError, hintfmt("string '%s' doesn't represent an absolute path", "foo"), - hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource")); + hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); ASSERT_TRACE2("filterSource [] ./.", TypeError, @@ -906,12 +906,12 @@ namespace nix { ASSERT_TRACE2("concatMap (x: 1) [ \"foo\" ] # TODO", TypeError, hintfmt("value is %s while a list was expected", "an integer"), - hintfmt("while evaluating the return value of the function passed to buitlins.concatMap")); + hintfmt("while evaluating the return value of the function passed to builtins.concatMap")); ASSERT_TRACE2("concatMap (x: \"foo\") [ 1 2 ] # TODO", TypeError, hintfmt("value is %s while a list was expected", "a string"), - hintfmt("while evaluating the return value of the function passed to buitlins.concatMap")); + hintfmt("while evaluating the return value of the function passed to builtins.concatMap")); } @@ -1084,7 +1084,7 @@ namespace nix { ASSERT_TRACE1("hashString \"foo\" \"content\"", UsageError, - hintfmt("unknown hash algorithm '%s'", "foo")); + hintfmt("unknown hash algorithm '%s', expect 'md5', 'sha1', 'sha256', or 'sha512'", "foo")); ASSERT_TRACE2("hashString \"sha256\" {}", TypeError, diff --git a/tests/unit/libexpr/eval.cc b/tests/unit/libexpr/eval.cc new file mode 100644 index 00000000000..93d3f658f52 --- /dev/null +++ b/tests/unit/libexpr/eval.cc @@ -0,0 +1,141 @@ +#include +#include + +#include "eval.hh" +#include "tests/libexpr.hh" + +namespace nix { + +TEST(nix_isAllowedURI, http_example_com) { + Strings allowed; + allowed.push_back("http://example.com"); + + ASSERT_TRUE(isAllowedURI("http://example.com", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.co", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.como", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.org", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.org/foo", allowed)); +} + +TEST(nix_isAllowedURI, http_example_com_foo) { + Strings allowed; + allowed.push_back("http://example.com/foo"); + + ASSERT_TRUE(isAllowedURI("http://example.com/foo", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.como", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.org/foo", allowed)); + // Broken? + // ASSERT_TRUE(isAllowedURI("http://example.com/foo?ok=1", allowed)); +} + +TEST(nix_isAllowedURI, http) { + Strings allowed; + allowed.push_back("http://"); + + ASSERT_TRUE(isAllowedURI("http://", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo/", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("https://", allowed)); + ASSERT_FALSE(isAllowedURI("http:foo", allowed)); +} + +TEST(nix_isAllowedURI, https) { + Strings allowed; + allowed.push_back("https://"); + + ASSERT_TRUE(isAllowedURI("https://example.com", allowed)); + ASSERT_TRUE(isAllowedURI("https://example.com/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com/https:", allowed)); +} + +TEST(nix_isAllowedURI, absolute_path) { + Strings allowed; + allowed.push_back("/var/evil"); // bad idea + + ASSERT_TRUE(isAllowedURI("/var/evil", allowed)); + ASSERT_TRUE(isAllowedURI("/var/evil/", allowed)); + ASSERT_TRUE(isAllowedURI("/var/evil/foo", allowed)); + ASSERT_TRUE(isAllowedURI("/var/evil/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evi", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com/var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil/foo", allowed)); +} + +TEST(nix_isAllowedURI, file_url) { + Strings allowed; + allowed.push_back("file:///var/evil"); // bad idea + + ASSERT_TRUE(isAllowedURI("file:///var/evil", allowed)); + ASSERT_TRUE(isAllowedURI("file:///var/evil/", allowed)); + ASSERT_TRUE(isAllowedURI("file:///var/evil/foo", allowed)); + ASSERT_TRUE(isAllowedURI("file:///var/evil/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evi", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com/var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http:///var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://var/evil/", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evi", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evilo", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evilo/", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evilo/foo", allowed)); + ASSERT_FALSE(isAllowedURI("file:///", allowed)); + ASSERT_FALSE(isAllowedURI("file://", allowed)); +} + +TEST(nix_isAllowedURI, github_all) { + Strings allowed; + allowed.push_back("github:"); + ASSERT_TRUE(isAllowedURI("github:", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar/feat-multi-bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar?ref=refs/heads/feat-multi-bar", allowed)); + ASSERT_TRUE(isAllowedURI("github://foo/bar", allowed)); + ASSERT_FALSE(isAllowedURI("https://github:443/foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file://github:foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file:///github:foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("github", allowed)); +} + +TEST(nix_isAllowedURI, github_org) { + Strings allowed; + allowed.push_back("github:foo"); + ASSERT_FALSE(isAllowedURI("github:", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar/feat-multi-bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar?ref=refs/heads/feat-multi-bar", allowed)); + ASSERT_FALSE(isAllowedURI("github://foo/bar", allowed)); + ASSERT_FALSE(isAllowedURI("https://github:443/foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file://github:foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file:///github:foo/bar/archive/master.tar.gz", allowed)); +} + +TEST(nix_isAllowedURI, non_scheme_colon) { + Strings allowed; + allowed.push_back("https://foo/bar:"); + ASSERT_TRUE(isAllowedURI("https://foo/bar:", allowed)); + ASSERT_TRUE(isAllowedURI("https://foo/bar:/baz", allowed)); + ASSERT_FALSE(isAllowedURI("https://foo/bar:baz", allowed)); +} + +} // namespace nix \ No newline at end of file diff --git a/src/libexpr/tests/flakeref.cc b/tests/unit/libexpr/flakeref.cc similarity index 100% rename from src/libexpr/tests/flakeref.cc rename to tests/unit/libexpr/flakeref.cc diff --git a/src/libexpr/tests/json.cc b/tests/unit/libexpr/json.cc similarity index 97% rename from src/libexpr/tests/json.cc rename to tests/unit/libexpr/json.cc index 7586bdd9b34..f4cc118d664 100644 --- a/src/libexpr/tests/json.cc +++ b/tests/unit/libexpr/json.cc @@ -62,7 +62,7 @@ namespace nix { // not supported by store 'dummy'" thrown in the test body. TEST_F(JSONValueTest, DISABLED_Path) { Value v; - v.mkPath("test"); + v.mkPath(state.rootPath(CanonPath("/test"))); ASSERT_EQ(getJSONValue(v), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\""); } } /* namespace nix */ diff --git a/tests/unit/libexpr/local.mk b/tests/unit/libexpr/local.mk new file mode 100644 index 00000000000..5743880d7d9 --- /dev/null +++ b/tests/unit/libexpr/local.mk @@ -0,0 +1,36 @@ +check: libexpr-tests_RUN + +programs += libexpr-tests + +libexpr-tests_NAME := libnixexpr-tests + +libexpr-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data + +libexpr-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libexpr-tests_INSTALL_DIR := $(checkbindir) +else + libexpr-tests_INSTALL_DIR := +endif + +libexpr-tests_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/value/*.cc) + +libexpr-tests_EXTRA_INCLUDES = \ + -I tests/unit/libexpr-support \ + -I tests/unit/libstore-support \ + -I tests/unit/libutil-support \ + -I src/libexpr \ + -I src/libfetchers \ + -I src/libstore \ + -I src/libutil + +libexpr-tests_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) + +libexpr-tests_LIBS = \ + libexpr-test-support libstore-test-support libutils-test-support \ + libexpr libfetchers libstore libutil + +libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock diff --git a/src/libexpr/tests/primops.cc b/tests/unit/libexpr/primops.cc similarity index 99% rename from src/libexpr/tests/primops.cc rename to tests/unit/libexpr/primops.cc index ce3b5d11fae..d820b860e7c 100644 --- a/src/libexpr/tests/primops.cc +++ b/tests/unit/libexpr/primops.cc @@ -711,14 +711,14 @@ namespace nix { // FIXME: add a test that verifies the string context is as expected auto v = eval("builtins.replaceStrings [\"oo\" \"a\"] [\"a\" \"i\"] \"foobar\""); ASSERT_EQ(v.type(), nString); - ASSERT_EQ(v.string.s, std::string_view("fabir")); + ASSERT_EQ(v.string_view(), "fabir"); } TEST_F(PrimOpTest, concatStringsSep) { // FIXME: add a test that verifies the string context is as expected auto v = eval("builtins.concatStringsSep \"%\" [\"foo\" \"bar\" \"baz\"]"); ASSERT_EQ(v.type(), nString); - ASSERT_EQ(std::string_view(v.string.s), "foo%bar%baz"); + ASSERT_EQ(v.string_view(), "foo%bar%baz"); } TEST_F(PrimOpTest, split1) { diff --git a/src/libexpr/tests/search-path.cc b/tests/unit/libexpr/search-path.cc similarity index 100% rename from src/libexpr/tests/search-path.cc rename to tests/unit/libexpr/search-path.cc diff --git a/src/libexpr/tests/trivial.cc b/tests/unit/libexpr/trivial.cc similarity index 100% rename from src/libexpr/tests/trivial.cc rename to tests/unit/libexpr/trivial.cc diff --git a/src/libexpr/tests/value/context.cc b/tests/unit/libexpr/value/context.cc similarity index 83% rename from src/libexpr/tests/value/context.cc rename to tests/unit/libexpr/value/context.cc index 92d4889abde..761286dbdcc 100644 --- a/src/libexpr/tests/value/context.cc +++ b/tests/unit/libexpr/value/context.cc @@ -117,36 +117,6 @@ TEST(NixStringContextElemTest, built_built_xp) { NixStringContextElem::parse("!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"), MissingExperimentalFeature); } -} - -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - return gen::just(NixStringContextElem::DrvDeep { - .drvPath = *gen::arbitrary(), - }); -} - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just(*gen::arbitrary()); - case 1: - return gen::just(*gen::arbitrary()); - case 2: - return gen::just(*gen::arbitrary()); - default: - assert(false); - } -} - -} - -namespace nix { - #ifndef COVERAGE RC_GTEST_PROP( diff --git a/src/libexpr/tests/value/print.cc b/tests/unit/libexpr/value/print.cc similarity index 99% rename from src/libexpr/tests/value/print.cc rename to tests/unit/libexpr/value/print.cc index 5e96e12ec7d..a4f6fc014f3 100644 --- a/src/libexpr/tests/value/print.cc +++ b/tests/unit/libexpr/value/print.cc @@ -114,7 +114,8 @@ TEST_F(ValuePrintingTests, vLambda) TEST_F(ValuePrintingTests, vPrimOp) { Value vPrimOp; - vPrimOp.mkPrimOp(nullptr); + PrimOp primOp{}; + vPrimOp.mkPrimOp(&primOp); test(vPrimOp, ""); } diff --git a/tests/unit/libstore-support/local.mk b/tests/unit/libstore-support/local.mk new file mode 100644 index 00000000000..ff075c96a69 --- /dev/null +++ b/tests/unit/libstore-support/local.mk @@ -0,0 +1,21 @@ +libraries += libstore-test-support + +libstore-test-support_NAME = libnixstore-test-support + +libstore-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libstore-test-support_INSTALL_DIR := $(checklibdir) +else + libstore-test-support_INSTALL_DIR := +endif + +libstore-test-support_SOURCES := $(wildcard $(d)/tests/*.cc) + +libstore-test-support_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES) + +libstore-test-support_LIBS = \ + libutil-test-support \ + libstore libutil + +libstore-test-support_LDFLAGS := -pthread -lrapidcheck diff --git a/tests/unit/libstore-support/tests/derived-path.cc b/tests/unit/libstore-support/tests/derived-path.cc new file mode 100644 index 00000000000..091706dbab4 --- /dev/null +++ b/tests/unit/libstore-support/tests/derived-path.cc @@ -0,0 +1,57 @@ +#include + +#include + +#include "tests/derived-path.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + return gen::just(DerivedPath::Opaque { + .path = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(SingleDerivedPath::Built { + .drvPath = make_ref(*gen::arbitrary()), + .output = (*gen::arbitrary()).name, + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(DerivedPath::Built { + .drvPath = make_ref(*gen::arbitrary()), + .outputs = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + +} diff --git a/src/libstore/tests/derived-path.hh b/tests/unit/libstore-support/tests/derived-path.hh similarity index 100% rename from src/libstore/tests/derived-path.hh rename to tests/unit/libstore-support/tests/derived-path.hh diff --git a/src/libstore/tests/libstore.hh b/tests/unit/libstore-support/tests/libstore.hh similarity index 86% rename from src/libstore/tests/libstore.hh rename to tests/unit/libstore-support/tests/libstore.hh index ef93457b53c..78b162b9568 100644 --- a/src/libstore/tests/libstore.hh +++ b/tests/unit/libstore-support/tests/libstore.hh @@ -8,7 +8,7 @@ namespace nix { -class LibStoreTest : public ::testing::Test { +class LibStoreTest : public virtual ::testing::Test { public: static void SetUpTestSuite() { initLibStore(); diff --git a/tests/unit/libstore-support/tests/outputs-spec.cc b/tests/unit/libstore-support/tests/outputs-spec.cc new file mode 100644 index 00000000000..e9d6022037b --- /dev/null +++ b/tests/unit/libstore-support/tests/outputs-spec.cc @@ -0,0 +1,24 @@ +#include "tests/outputs-spec.hh" + +#include + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just((OutputsSpec) OutputsSpec::All { }); + case 1: + return gen::just((OutputsSpec) OutputsSpec::Names { + *gen::nonEmpty(gen::container(gen::map( + gen::arbitrary(), + [](StorePathName n) { return n.name; }))), + }); + default: + assert(false); + } +} + +} diff --git a/src/libstore/tests/outputs-spec.hh b/tests/unit/libstore-support/tests/outputs-spec.hh similarity index 89% rename from src/libstore/tests/outputs-spec.hh rename to tests/unit/libstore-support/tests/outputs-spec.hh index ded331b333e..f5bf9042d20 100644 --- a/src/libstore/tests/outputs-spec.hh +++ b/tests/unit/libstore-support/tests/outputs-spec.hh @@ -5,7 +5,7 @@ #include -#include +#include "tests/path.hh" namespace rc { using namespace nix; diff --git a/tests/unit/libstore-support/tests/path.cc b/tests/unit/libstore-support/tests/path.cc new file mode 100644 index 00000000000..e5f169e94c5 --- /dev/null +++ b/tests/unit/libstore-support/tests/path.cc @@ -0,0 +1,82 @@ +#include + +#include + +#include "path-regex.hh" +#include "store-api.hh" + +#include "tests/hash.hh" +#include "tests/path.hh" + +namespace nix { + +void showValue(const StorePath & p, std::ostream & os) +{ + os << p.to_string(); +} + +} + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + auto len = *gen::inRange( + 1, + StorePath::MaxPathLen - StorePath::HashLen); + + std::string pre; + pre.reserve(len); + + for (size_t c = 0; c < len; ++c) { + switch (auto i = *gen::inRange(0, 10 + 2 * 26 + 6)) { + case 0 ... 9: + pre += '0' + i; + case 10 ... 35: + pre += 'A' + (i - 10); + break; + case 36 ... 61: + pre += 'a' + (i - 36); + break; + case 62: + pre += '+'; + break; + case 63: + pre += '-'; + break; + case 64: + // names aren't permitted to start with a period, + // so just fall through to the next case here + if (c != 0) { + pre += '.'; + break; + } + case 65: + pre += '_'; + break; + case 66: + pre += '?'; + break; + case 67: + pre += '='; + break; + default: + assert(false); + } + } + + return gen::just(StorePathName { + .name = std::move(pre), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(StorePath { + *gen::arbitrary(), + (*gen::arbitrary()).name, + }); +} + +} // namespace rc diff --git a/src/libstore/tests/path.hh b/tests/unit/libstore-support/tests/path.hh similarity index 82% rename from src/libstore/tests/path.hh rename to tests/unit/libstore-support/tests/path.hh index 21cb62310dc..4751b3373a3 100644 --- a/src/libstore/tests/path.hh +++ b/tests/unit/libstore-support/tests/path.hh @@ -11,6 +11,9 @@ struct StorePathName { std::string name; }; +// For rapidcheck +void showValue(const StorePath & p, std::ostream & os); + } namespace rc { diff --git a/tests/unit/libstore-support/tests/protocol.hh b/tests/unit/libstore-support/tests/protocol.hh new file mode 100644 index 00000000000..3c9e52c117a --- /dev/null +++ b/tests/unit/libstore-support/tests/protocol.hh @@ -0,0 +1,75 @@ +#pragma once +///@file + +#include +#include + +#include "tests/libstore.hh" +#include "tests/characterization.hh" + +namespace nix { + +template +class ProtoTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/" + protocolDir; + + Path goldenMaster(std::string_view testStem) const override { + return unitTestData + "/" + testStem + ".bin"; + } +}; + +template +class VersionedProtoTest : public ProtoTest +{ +public: + /** + * Golden test for `T` reading + */ + template + void readProtoTest(PathView testStem, typename Proto::Version version, T expected) + { + CharacterizationTest::readTest(testStem, [&](const auto & encoded) { + T got = ({ + StringSource from { encoded }; + Proto::template Serialise::read( + *LibStoreTest::store, + typename Proto::ReadConn { + .from = from, + .version = version, + }); + }); + + ASSERT_EQ(got, expected); + }); + } + + /** + * Golden test for `T` write + */ + template + void writeProtoTest(PathView testStem, typename Proto::Version version, const T & decoded) + { + CharacterizationTest::writeTest(testStem, [&]() { + StringSink to; + Proto::template Serialise::write( + *LibStoreTest::store, + typename Proto::WriteConn { + .to = to, + .version = version, + }, + decoded); + return std::move(to.s); + }); + } +}; + +#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ + TEST_F(FIXTURE, NAME ## _read) { \ + readProtoTest(STEM, VERSION, VALUE); \ + } \ + TEST_F(FIXTURE, NAME ## _write) { \ + writeProtoTest(STEM, VERSION, VALUE); \ + } + +} diff --git a/tests/unit/libstore/common-protocol.cc b/tests/unit/libstore/common-protocol.cc new file mode 100644 index 00000000000..c09ac6a3eb0 --- /dev/null +++ b/tests/unit/libstore/common-protocol.cc @@ -0,0 +1,187 @@ +#include + +#include +#include + +#include "common-protocol.hh" +#include "common-protocol-impl.hh" +#include "build-result.hh" +#include "tests/protocol.hh" +#include "tests/characterization.hh" + +namespace nix { + +const char commonProtoDir[] = "common-protocol"; + +class CommonProtoTest : public ProtoTest +{ +public: + /** + * Golden test for `T` reading + */ + template + void readProtoTest(PathView testStem, const T & expected) + { + CharacterizationTest::readTest(testStem, [&](const auto & encoded) { + T got = ({ + StringSource from { encoded }; + CommonProto::Serialise::read( + *store, + CommonProto::ReadConn { .from = from }); + }); + + ASSERT_EQ(got, expected); + }); + } + + /** + * Golden test for `T` write + */ + template + void writeProtoTest(PathView testStem, const T & decoded) + { + CharacterizationTest::writeTest(testStem, [&]() -> std::string { + StringSink to; + CommonProto::Serialise::write( + *store, + CommonProto::WriteConn { .to = to }, + decoded); + return to.s; + }); + } +}; + +#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + TEST_F(CommonProtoTest, NAME ## _read) { \ + readProtoTest(STEM, VALUE); \ + } \ + TEST_F(CommonProtoTest, NAME ## _write) { \ + writeProtoTest(STEM, VALUE); \ + } + +CHARACTERIZATION_TEST( + string, + "string", + (std::tuple { + "", + "hi", + "white rabbit", + "大白兔", + "oh no \0\0\0 what was that!", + })) + +CHARACTERIZATION_TEST( + storePath, + "store-path", + (std::tuple { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + })) + +CHARACTERIZATION_TEST( + contentAddress, + "content-address", + (std::tuple { + ContentAddress { + .method = TextIngestionMethod {}, + .hash = hashString(HashType::htSHA256, "Derive(...)"), + }, + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + ContentAddress { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + }, + })) + +CHARACTERIZATION_TEST( + drvOutput, + "drv-output", + (std::tuple { + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + })) + +CHARACTERIZATION_TEST( + realisation, + "realisation", + (std::tuple { + Realisation { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + }, + Realisation { + .id = { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + .dependentRealisations = { + { + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + }, + })) + +CHARACTERIZATION_TEST( + vector, + "vector", + (std::tuple, std::vector, std::vector, std::vector>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +CHARACTERIZATION_TEST( + set, + "set", + (std::tuple, std::set, std::set, std::set>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +CHARACTERIZATION_TEST( + optionalStorePath, + "optional-store-path", + (std::tuple, std::optional> { + std::nullopt, + std::optional { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + }, + })) + +CHARACTERIZATION_TEST( + optionalContentAddress, + "optional-content-address", + (std::tuple, std::optional> { + std::nullopt, + std::optional { + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + }, + })) + +} diff --git a/tests/unit/libstore/data/common-protocol/content-address.bin b/tests/unit/libstore/data/common-protocol/content-address.bin new file mode 100644 index 00000000000..8f14bcdb3e5 Binary files /dev/null and b/tests/unit/libstore/data/common-protocol/content-address.bin differ diff --git a/tests/unit/libstore/data/common-protocol/drv-output.bin b/tests/unit/libstore/data/common-protocol/drv-output.bin new file mode 100644 index 00000000000..800a45fd875 Binary files /dev/null and b/tests/unit/libstore/data/common-protocol/drv-output.bin differ diff --git a/tests/unit/libstore/data/common-protocol/optional-content-address.bin b/tests/unit/libstore/data/common-protocol/optional-content-address.bin new file mode 100644 index 00000000000..f8cfe65ba27 Binary files /dev/null and b/tests/unit/libstore/data/common-protocol/optional-content-address.bin differ diff --git a/tests/unit/libstore/data/common-protocol/optional-store-path.bin b/tests/unit/libstore/data/common-protocol/optional-store-path.bin new file mode 100644 index 00000000000..4fbca5576b6 Binary files /dev/null and b/tests/unit/libstore/data/common-protocol/optional-store-path.bin differ diff --git a/tests/unit/libstore/data/common-protocol/realisation.bin b/tests/unit/libstore/data/common-protocol/realisation.bin new file mode 100644 index 00000000000..2176c6c4afd Binary files /dev/null and b/tests/unit/libstore/data/common-protocol/realisation.bin differ diff --git a/tests/unit/libstore/data/common-protocol/set.bin b/tests/unit/libstore/data/common-protocol/set.bin new file mode 100644 index 00000000000..ce11ede7fe7 Binary files /dev/null and b/tests/unit/libstore/data/common-protocol/set.bin differ diff --git a/tests/unit/libstore/data/common-protocol/store-path.bin b/tests/unit/libstore/data/common-protocol/store-path.bin new file mode 100644 index 00000000000..3fc05f2981d Binary files /dev/null and b/tests/unit/libstore/data/common-protocol/store-path.bin differ diff --git a/tests/unit/libstore/data/common-protocol/string.bin b/tests/unit/libstore/data/common-protocol/string.bin new file mode 100644 index 00000000000..aa7b5a60474 Binary files /dev/null and b/tests/unit/libstore/data/common-protocol/string.bin differ diff --git a/tests/unit/libstore/data/common-protocol/vector.bin b/tests/unit/libstore/data/common-protocol/vector.bin new file mode 100644 index 00000000000..7a37c8cd109 Binary files /dev/null and b/tests/unit/libstore/data/common-protocol/vector.bin differ diff --git a/tests/unit/libstore/data/derivation/bad-old-version-dyn-deps.drv b/tests/unit/libstore/data/derivation/bad-old-version-dyn-deps.drv new file mode 100644 index 00000000000..3cd1ded029d --- /dev/null +++ b/tests/unit/libstore/data/derivation/bad-old-version-dyn-deps.drv @@ -0,0 +1 @@ +Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/bad-version.drv b/tests/unit/libstore/data/derivation/bad-version.drv new file mode 100644 index 00000000000..bbf75c114de --- /dev/null +++ b/tests/unit/libstore/data/derivation/bad-version.drv @@ -0,0 +1 @@ +DrvWithVersion("invalid-version",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/dynDerivationDeps.drv b/tests/unit/libstore/data/derivation/dynDerivationDeps.drv new file mode 100644 index 00000000000..cfffe48ec90 --- /dev/null +++ b/tests/unit/libstore/data/derivation/dynDerivationDeps.drv @@ -0,0 +1 @@ +DrvWithVersion("xp-dyn-drv",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/dynDerivationDeps.json b/tests/unit/libstore/data/derivation/dynDerivationDeps.json new file mode 100644 index 00000000000..9dbeb1f15af --- /dev/null +++ b/tests/unit/libstore/data/derivation/dynDerivationDeps.json @@ -0,0 +1,38 @@ +{ + "args": [ + "bar", + "baz" + ], + "builder": "foo", + "env": { + "BIG_BAD": "WOLF" + }, + "inputDrvs": { + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": { + "cat": { + "dynamicOutputs": {}, + "outputs": [ + "kitten" + ] + }, + "goose": { + "dynamicOutputs": {}, + "outputs": [ + "gosling" + ] + } + }, + "outputs": [ + "cat", + "dog" + ] + } + }, + "inputSrcs": [ + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ], + "name": "dyn-dep-derivation", + "outputs": {}, + "system": "wasm-sel4" +} diff --git a/tests/unit/libstore/data/derivation/output-caFixedFlat.json b/tests/unit/libstore/data/derivation/output-caFixedFlat.json new file mode 100644 index 00000000000..fe000ea36e3 --- /dev/null +++ b/tests/unit/libstore/data/derivation/output-caFixedFlat.json @@ -0,0 +1,5 @@ +{ + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "hashAlgo": "sha256", + "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name" +} diff --git a/tests/unit/libstore/data/derivation/output-caFixedNAR.json b/tests/unit/libstore/data/derivation/output-caFixedNAR.json new file mode 100644 index 00000000000..1afd602236b --- /dev/null +++ b/tests/unit/libstore/data/derivation/output-caFixedNAR.json @@ -0,0 +1,5 @@ +{ + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "hashAlgo": "r:sha256", + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" +} diff --git a/tests/unit/libstore/data/derivation/output-caFixedText.json b/tests/unit/libstore/data/derivation/output-caFixedText.json new file mode 100644 index 00000000000..0b2cc8bbc42 --- /dev/null +++ b/tests/unit/libstore/data/derivation/output-caFixedText.json @@ -0,0 +1,5 @@ +{ + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "hashAlgo": "text:sha256", + "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" +} diff --git a/tests/unit/libstore/data/derivation/output-caFloating.json b/tests/unit/libstore/data/derivation/output-caFloating.json new file mode 100644 index 00000000000..9115de851a1 --- /dev/null +++ b/tests/unit/libstore/data/derivation/output-caFloating.json @@ -0,0 +1,3 @@ +{ + "hashAlgo": "r:sha256" +} diff --git a/tests/unit/libstore/data/derivation/output-deferred.json b/tests/unit/libstore/data/derivation/output-deferred.json new file mode 100644 index 00000000000..0967ef424bc --- /dev/null +++ b/tests/unit/libstore/data/derivation/output-deferred.json @@ -0,0 +1 @@ +{} diff --git a/tests/unit/libstore/data/derivation/output-impure.json b/tests/unit/libstore/data/derivation/output-impure.json new file mode 100644 index 00000000000..62b61cdcae7 --- /dev/null +++ b/tests/unit/libstore/data/derivation/output-impure.json @@ -0,0 +1,4 @@ +{ + "hashAlgo": "r:sha256", + "impure": true +} diff --git a/tests/unit/libstore/data/derivation/output-inputAddressed.json b/tests/unit/libstore/data/derivation/output-inputAddressed.json new file mode 100644 index 00000000000..86c7f3a05ce --- /dev/null +++ b/tests/unit/libstore/data/derivation/output-inputAddressed.json @@ -0,0 +1,3 @@ +{ + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" +} diff --git a/tests/unit/libstore/data/derivation/simple.drv b/tests/unit/libstore/data/derivation/simple.drv new file mode 100644 index 00000000000..bda74ad25c0 --- /dev/null +++ b/tests/unit/libstore/data/derivation/simple.drv @@ -0,0 +1 @@ +Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/simple.json b/tests/unit/libstore/data/derivation/simple.json new file mode 100644 index 00000000000..20d0f8933e6 --- /dev/null +++ b/tests/unit/libstore/data/derivation/simple.json @@ -0,0 +1,25 @@ +{ + "args": [ + "bar", + "baz" + ], + "builder": "foo", + "env": { + "BIG_BAD": "WOLF" + }, + "inputDrvs": { + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": {}, + "outputs": [ + "cat", + "dog" + ] + } + }, + "inputSrcs": [ + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ], + "name": "simple-derivation", + "outputs": {}, + "system": "wasm-sel4" +} diff --git a/tests/unit/libstore/data/nar-info/impure.json b/tests/unit/libstore/data/nar-info/impure.json new file mode 100644 index 00000000000..bb9791a6ace --- /dev/null +++ b/tests/unit/libstore/data/nar-info/impure.json @@ -0,0 +1,20 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "compression": "xz", + "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "downloadHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "downloadSize": 4029176, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "registrationTime": 23423, + "signatures": [ + "asdf", + "qwer" + ], + "ultimate": true, + "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz" +} diff --git a/tests/unit/libstore/data/nar-info/pure.json b/tests/unit/libstore/data/nar-info/pure.json new file mode 100644 index 00000000000..955baec3107 --- /dev/null +++ b/tests/unit/libstore/data/nar-info/pure.json @@ -0,0 +1,9 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ] +} diff --git a/tests/unit/libstore/data/path-info/impure.json b/tests/unit/libstore/data/path-info/impure.json new file mode 100644 index 00000000000..0c452cc4930 --- /dev/null +++ b/tests/unit/libstore/data/path-info/impure.json @@ -0,0 +1,16 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "registrationTime": 23423, + "signatures": [ + "asdf", + "qwer" + ], + "ultimate": true +} diff --git a/tests/unit/libstore/data/path-info/pure.json b/tests/unit/libstore/data/path-info/pure.json new file mode 100644 index 00000000000..955baec3107 --- /dev/null +++ b/tests/unit/libstore/data/path-info/pure.json @@ -0,0 +1,9 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ] +} diff --git a/tests/unit/libstore/data/serve-protocol/build-result-2.2.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.2.bin new file mode 100644 index 00000000000..ae684778bc2 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/build-result-2.2.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/build-result-2.3.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.3.bin new file mode 100644 index 00000000000..d51e08dfc0d Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/build-result-2.3.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/build-result-2.6.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.6.bin new file mode 100644 index 00000000000..b02c706eab8 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/build-result-2.6.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/content-address.bin b/tests/unit/libstore/data/serve-protocol/content-address.bin new file mode 100644 index 00000000000..8f14bcdb3e5 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/content-address.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/drv-output.bin b/tests/unit/libstore/data/serve-protocol/drv-output.bin new file mode 100644 index 00000000000..800a45fd875 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/drv-output.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/optional-content-address.bin b/tests/unit/libstore/data/serve-protocol/optional-content-address.bin new file mode 100644 index 00000000000..f8cfe65ba27 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/optional-content-address.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/optional-store-path.bin b/tests/unit/libstore/data/serve-protocol/optional-store-path.bin new file mode 100644 index 00000000000..4fbca5576b6 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/optional-store-path.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/realisation.bin b/tests/unit/libstore/data/serve-protocol/realisation.bin new file mode 100644 index 00000000000..2176c6c4afd Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/realisation.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/set.bin b/tests/unit/libstore/data/serve-protocol/set.bin new file mode 100644 index 00000000000..ce11ede7fe7 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/set.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/store-path.bin b/tests/unit/libstore/data/serve-protocol/store-path.bin new file mode 100644 index 00000000000..3fc05f2981d Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/store-path.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/string.bin b/tests/unit/libstore/data/serve-protocol/string.bin new file mode 100644 index 00000000000..aa7b5a60474 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/string.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/vector.bin b/tests/unit/libstore/data/serve-protocol/vector.bin new file mode 100644 index 00000000000..7a37c8cd109 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/vector.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/build-result-1.27.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.27.bin new file mode 100644 index 00000000000..ae684778bc2 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/build-result-1.27.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/build-result-1.28.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.28.bin new file mode 100644 index 00000000000..74bcd5cf98b Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/build-result-1.28.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/build-result-1.29.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.29.bin new file mode 100644 index 00000000000..b02c706eab8 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/build-result-1.29.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/content-address.bin b/tests/unit/libstore/data/worker-protocol/content-address.bin new file mode 100644 index 00000000000..8f14bcdb3e5 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/content-address.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/derived-path-1.29.bin b/tests/unit/libstore/data/worker-protocol/derived-path-1.29.bin new file mode 100644 index 00000000000..05ea7678aa0 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/derived-path-1.29.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/derived-path-1.30.bin b/tests/unit/libstore/data/worker-protocol/derived-path-1.30.bin new file mode 100644 index 00000000000..0729b2690e2 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/derived-path-1.30.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/drv-output.bin b/tests/unit/libstore/data/worker-protocol/drv-output.bin new file mode 100644 index 00000000000..800a45fd875 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/drv-output.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/keyed-build-result-1.29.bin b/tests/unit/libstore/data/worker-protocol/keyed-build-result-1.29.bin new file mode 100644 index 00000000000..c5b3c7f3669 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/keyed-build-result-1.29.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/optional-content-address.bin b/tests/unit/libstore/data/worker-protocol/optional-content-address.bin new file mode 100644 index 00000000000..f8cfe65ba27 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/optional-content-address.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/optional-store-path.bin b/tests/unit/libstore/data/worker-protocol/optional-store-path.bin new file mode 100644 index 00000000000..4fbca5576b6 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/optional-store-path.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/optional-trusted-flag.bin b/tests/unit/libstore/data/worker-protocol/optional-trusted-flag.bin new file mode 100644 index 00000000000..51b239409bc Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/optional-trusted-flag.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/realisation.bin b/tests/unit/libstore/data/worker-protocol/realisation.bin new file mode 100644 index 00000000000..2176c6c4afd Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/realisation.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/set.bin b/tests/unit/libstore/data/worker-protocol/set.bin new file mode 100644 index 00000000000..ce11ede7fe7 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/set.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/store-path.bin b/tests/unit/libstore/data/worker-protocol/store-path.bin new file mode 100644 index 00000000000..3fc05f2981d Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/store-path.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/string.bin b/tests/unit/libstore/data/worker-protocol/string.bin new file mode 100644 index 00000000000..aa7b5a60474 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/string.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/unkeyed-valid-path-info-1.15.bin b/tests/unit/libstore/data/worker-protocol/unkeyed-valid-path-info-1.15.bin new file mode 100644 index 00000000000..e69ccbe8386 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/unkeyed-valid-path-info-1.15.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/valid-path-info-1.15.bin b/tests/unit/libstore/data/worker-protocol/valid-path-info-1.15.bin new file mode 100644 index 00000000000..7adc8dd4472 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/valid-path-info-1.15.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/valid-path-info-1.16.bin b/tests/unit/libstore/data/worker-protocol/valid-path-info-1.16.bin new file mode 100644 index 00000000000..a72de6bd62a Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/valid-path-info-1.16.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/vector.bin b/tests/unit/libstore/data/worker-protocol/vector.bin new file mode 100644 index 00000000000..7a37c8cd109 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/vector.bin differ diff --git a/tests/unit/libstore/derivation.cc b/tests/unit/libstore/derivation.cc new file mode 100644 index 00000000000..a7f4488fa09 --- /dev/null +++ b/tests/unit/libstore/derivation.cc @@ -0,0 +1,298 @@ +#include +#include + +#include "experimental-features.hh" +#include "derivations.hh" + +#include "tests/libstore.hh" +#include "tests/characterization.hh" + +namespace nix { + +using nlohmann::json; + +class DerivationTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/derivation"; + +public: + Path goldenMaster(std::string_view testStem) const override { + return unitTestData + "/" + testStem; + } + + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; +}; + +class CaDerivationTest : public DerivationTest +{ + void SetUp() override + { + mockXpSettings.set("experimental-features", "ca-derivations"); + } +}; + +class DynDerivationTest : public DerivationTest +{ + void SetUp() override + { + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + } +}; + +class ImpureDerivationTest : public DerivationTest +{ + void SetUp() override + { + mockXpSettings.set("experimental-features", "impure-derivations"); + } +}; + +TEST_F(DerivationTest, BadATerm_version) { + ASSERT_THROW( + parseDerivation( + *store, + readFile(goldenMaster("bad-version.drv")), + "whatever", + mockXpSettings), + FormatError); +} + +TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) { + ASSERT_THROW( + parseDerivation( + *store, + readFile(goldenMaster("bad-old-version-dyn-deps.drv")), + "dyn-dep-derivation", + mockXpSettings), + FormatError); +} + +#define TEST_JSON(FIXTURE, NAME, VAL, DRV_NAME, OUTPUT_NAME) \ + TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) { \ + readTest("output-" #NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + DerivationOutput got = DerivationOutput::fromJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME, \ + encoded, \ + mockXpSettings); \ + DerivationOutput expected { VAL }; \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \ + writeTest("output-" #NAME ".json", [&]() -> json { \ + return DerivationOutput { (VAL) }.toJSON( \ + *store, \ + (DRV_NAME), \ + (OUTPUT_NAME)); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ + } + +TEST_JSON(DerivationTest, inputAddressed, + (DerivationOutput::InputAddressed { + .path = store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"), + }), + "drv-name", "output-name") + +TEST_JSON(DerivationTest, caFixedFlat, + (DerivationOutput::CAFixed { + .ca = { + .method = FileIngestionMethod::Flat, + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, + }), + "drv-name", "output-name") + +TEST_JSON(DerivationTest, caFixedNAR, + (DerivationOutput::CAFixed { + .ca = { + .method = FileIngestionMethod::Recursive, + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, + }), + "drv-name", "output-name") + +TEST_JSON(DynDerivationTest, caFixedText, + (DerivationOutput::CAFixed { + .ca = { + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, + }), + "drv-name", "output-name") + +TEST_JSON(CaDerivationTest, caFloating, + (DerivationOutput::CAFloating { + .method = FileIngestionMethod::Recursive, + .hashType = htSHA256, + }), + "drv-name", "output-name") + +TEST_JSON(DerivationTest, deferred, + DerivationOutput::Deferred { }, + "drv-name", "output-name") + +TEST_JSON(ImpureDerivationTest, impure, + (DerivationOutput::Impure { + .method = FileIngestionMethod::Recursive, + .hashType = htSHA256, + }), + "drv-name", "output-name") + +#undef TEST_JSON + +#define TEST_JSON(FIXTURE, NAME, VAL) \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) { \ + readTest(#NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + Derivation expected { VAL }; \ + Derivation got = Derivation::fromJSON( \ + *store, \ + encoded, \ + mockXpSettings); \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) { \ + writeTest(#NAME ".json", [&]() -> json { \ + return Derivation { VAL }.toJSON(*store); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ + } + +#define TEST_ATERM(FIXTURE, NAME, VAL, DRV_NAME) \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) { \ + readTest(#NAME ".drv", [&](auto encoded) { \ + Derivation expected { VAL }; \ + auto got = parseDerivation( \ + *store, \ + std::move(encoded), \ + DRV_NAME, \ + mockXpSettings); \ + ASSERT_EQ(got.toJSON(*store), expected.toJSON(*store)) ; \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) { \ + writeTest(#NAME ".drv", [&]() -> std::string { \ + return (VAL).unparse(*store, false); \ + }); \ + } + +Derivation makeSimpleDrv(const Store & store) { + Derivation drv; + drv.name = "simple-derivation"; + drv.inputSrcs = { + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), + }; + drv.inputDrvs = { + .map = { + { + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + { + .value = { + "cat", + "dog", + }, + }, + }, + }, + }; + drv.platform = "wasm-sel4"; + drv.builder = "foo"; + drv.args = { + "bar", + "baz", + }; + drv.env = { + { + "BIG_BAD", + "WOLF", + }, + }; + return drv; +} + +TEST_JSON(DerivationTest, simple, makeSimpleDrv(*store)) + +TEST_ATERM(DerivationTest, simple, + makeSimpleDrv(*store), + "simple-derivation") + +Derivation makeDynDepDerivation(const Store & store) { + Derivation drv; + drv.name = "dyn-dep-derivation"; + drv.inputSrcs = { + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), + }; + drv.inputDrvs = { + .map = { + { + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + DerivedPathMap::ChildNode { + .value = { + "cat", + "dog", + }, + .childMap = { + { + "cat", + DerivedPathMap::ChildNode { + .value = { + "kitten", + }, + }, + }, + { + "goose", + DerivedPathMap::ChildNode { + .value = { + "gosling", + }, + }, + }, + }, + }, + }, + }, + }; + drv.platform = "wasm-sel4"; + drv.builder = "foo"; + drv.args = { + "bar", + "baz", + }; + drv.env = { + { + "BIG_BAD", + "WOLF", + }, + }; + return drv; +} + +TEST_JSON(DynDerivationTest, dynDerivationDeps, makeDynDepDerivation(*store)) + +TEST_ATERM(DynDerivationTest, dynDerivationDeps, + makeDynDepDerivation(*store), + "dyn-dep-derivation") + +#undef TEST_JSON +#undef TEST_ATERM + +} diff --git a/src/libstore/tests/derived-path.cc b/tests/unit/libstore/derived-path.cc similarity index 66% rename from src/libstore/tests/derived-path.cc rename to tests/unit/libstore/derived-path.cc index 3fa3c080181..c62d79a78ca 100644 --- a/src/libstore/tests/derived-path.cc +++ b/tests/unit/libstore/derived-path.cc @@ -1,64 +1,11 @@ #include -#include #include #include #include "tests/derived-path.hh" #include "tests/libstore.hh" -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - return gen::just(DerivedPath::Opaque { - .path = *gen::arbitrary(), - }); -} - -Gen Arbitrary::arbitrary() -{ - return gen::just(SingleDerivedPath::Built { - .drvPath = make_ref(*gen::arbitrary()), - .output = (*gen::arbitrary()).name, - }); -} - -Gen Arbitrary::arbitrary() -{ - return gen::just(DerivedPath::Built { - .drvPath = make_ref(*gen::arbitrary()), - .outputs = *gen::arbitrary(), - }); -} - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just(*gen::arbitrary()); - case 1: - return gen::just(*gen::arbitrary()); - default: - assert(false); - } -} - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just(*gen::arbitrary()); - case 1: - return gen::just(*gen::arbitrary()); - default: - assert(false); - } -} - -} - namespace nix { class DerivedPathTest : public LibStoreTest diff --git a/src/libstore/tests/downstream-placeholder.cc b/tests/unit/libstore/downstream-placeholder.cc similarity index 100% rename from src/libstore/tests/downstream-placeholder.cc rename to tests/unit/libstore/downstream-placeholder.cc diff --git a/tests/unit/libstore/local.mk b/tests/unit/libstore/local.mk new file mode 100644 index 00000000000..63f6d011f12 --- /dev/null +++ b/tests/unit/libstore/local.mk @@ -0,0 +1,31 @@ +check: libstore-tests_RUN + +programs += libstore-tests + +libstore-tests_NAME = libnixstore-tests + +libstore-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data + +libstore-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libstore-tests_INSTALL_DIR := $(checkbindir) +else + libstore-tests_INSTALL_DIR := +endif + +libstore-tests_SOURCES := $(wildcard $(d)/*.cc) + +libstore-tests_EXTRA_INCLUDES = \ + -I tests/unit/libstore-support \ + -I tests/unit/libutil-support \ + -I src/libstore \ + -I src/libutil + +libstore-tests_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES) + +libstore-tests_LIBS = \ + libstore-test-support libutil-test-support \ + libstore libutil + +libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/src/libstore/tests/machines.cc b/tests/unit/libstore/machines.cc similarity index 97% rename from src/libstore/tests/machines.cc rename to tests/unit/libstore/machines.cc index f51052b1462..5b66e5a5be9 100644 --- a/src/libstore/tests/machines.cc +++ b/tests/unit/libstore/machines.cc @@ -1,5 +1,7 @@ #include "machines.hh" #include "globals.hh" +#include "file-system.hh" +#include "util.hh" #include @@ -137,7 +139,7 @@ TEST(machines, getMachinesWithIncorrectFormat) { } TEST(machines, getMachinesWithCorrectFileReference) { - auto path = absPath("src/libstore/tests/test-data/machines.valid"); + auto path = absPath("tests/unit/libstore/test-data/machines.valid"); ASSERT_TRUE(pathExists(path)); settings.builders = std::string("@") + path; @@ -164,6 +166,6 @@ TEST(machines, getMachinesWithIncorrectFileReference) { } TEST(machines, getMachinesWithCorrectFileReferenceToIncorrectFile) { - settings.builders = std::string("@") + absPath("src/libstore/tests/test-data/machines.bad_format"); + settings.builders = std::string("@") + absPath("tests/unit/libstore/test-data/machines.bad_format"); EXPECT_THROW(getMachines(), FormatError); } diff --git a/src/libstore/tests/nar-info-disk-cache.cc b/tests/unit/libstore/nar-info-disk-cache.cc similarity index 100% rename from src/libstore/tests/nar-info-disk-cache.cc rename to tests/unit/libstore/nar-info-disk-cache.cc diff --git a/tests/unit/libstore/nar-info.cc b/tests/unit/libstore/nar-info.cc new file mode 100644 index 00000000000..31e20ca140c --- /dev/null +++ b/tests/unit/libstore/nar-info.cc @@ -0,0 +1,85 @@ +#include +#include + +#include "path-info.hh" + +#include "tests/characterization.hh" +#include "tests/libstore.hh" + +namespace nix { + +using nlohmann::json; + +class NarInfoTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/nar-info"; + + Path goldenMaster(PathView testStem) const override { + return unitTestData + "/" + testStem + ".json"; + } +}; + +static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) { + NarInfo info = ValidPathInfo { + store, + "foo", + FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + + .references = { + .others = { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }, + .self = true, + }, + }, + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.narSize = 34878; + if (includeImpureInfo) { + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.registrationTime = 23423; + info.ultimate = true; + info.sigs = { "asdf", "qwer" }; + + info.url = "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz"; + info.compression = "xz"; + info.fileHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="); + info.fileSize = 4029176; + } + return info; +} + +#define JSON_TEST(STEM, PURE) \ + TEST_F(NarInfoTest, NarInfo_ ## STEM ## _from_json) { \ + readTest(#STEM, [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + auto expected = makeNarInfo(*store, PURE); \ + NarInfo got = NarInfo::fromJSON( \ + *store, \ + expected.path, \ + encoded); \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(NarInfoTest, NarInfo_ ## STEM ## _to_json) { \ + writeTest(#STEM, [&]() -> json { \ + return makeNarInfo(*store, PURE) \ + .toJSON(*store, PURE, HashFormat::SRI); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ + } + +JSON_TEST(pure, false) +JSON_TEST(impure, true) + +} diff --git a/src/libstore/tests/outputs-spec.cc b/tests/unit/libstore/outputs-spec.cc similarity index 92% rename from src/libstore/tests/outputs-spec.cc rename to tests/unit/libstore/outputs-spec.cc index 95294518544..456196be106 100644 --- a/src/libstore/tests/outputs-spec.cc +++ b/tests/unit/libstore/outputs-spec.cc @@ -1,4 +1,4 @@ -#include "outputs-spec.hh" +#include "tests/outputs-spec.hh" #include #include @@ -199,31 +199,6 @@ TEST_JSON(ExtendedOutputsSpec, names, R"(["a","b"])", (ExtendedOutputsSpec::Expl #undef TEST_JSON -} - -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just((OutputsSpec) OutputsSpec::All { }); - case 1: - return gen::just((OutputsSpec) OutputsSpec::Names { - *gen::nonEmpty(gen::container(gen::map( - gen::arbitrary(), - [](StorePathName n) { return n.name; }))), - }); - default: - assert(false); - } -} - -} - -namespace nix { - #ifndef COVERAGE RC_GTEST_PROP( diff --git a/tests/unit/libstore/path-info.cc b/tests/unit/libstore/path-info.cc new file mode 100644 index 00000000000..18f00ca1929 --- /dev/null +++ b/tests/unit/libstore/path-info.cc @@ -0,0 +1,79 @@ +#include +#include + +#include "path-info.hh" + +#include "tests/characterization.hh" +#include "tests/libstore.hh" + +namespace nix { + +using nlohmann::json; + +class PathInfoTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/path-info"; + + Path goldenMaster(PathView testStem) const override { + return unitTestData + "/" + testStem + ".json"; + } +}; + +static UnkeyedValidPathInfo makePathInfo(const Store & store, bool includeImpureInfo) { + UnkeyedValidPathInfo info = ValidPathInfo { + store, + "foo", + FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + + .references = { + .others = { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }, + .self = true, + }, + }, + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.narSize = 34878; + if (includeImpureInfo) { + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.registrationTime = 23423; + info.ultimate = true; + info.sigs = { "asdf", "qwer" }; + } + return info; +} + +#define JSON_TEST(STEM, PURE) \ + TEST_F(PathInfoTest, PathInfo_ ## STEM ## _from_json) { \ + readTest(#STEM, [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + UnkeyedValidPathInfo got = UnkeyedValidPathInfo::fromJSON( \ + *store, \ + encoded); \ + auto expected = makePathInfo(*store, PURE); \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(PathInfoTest, PathInfo_ ## STEM ## _to_json) { \ + writeTest(#STEM, [&]() -> json { \ + return makePathInfo(*store, PURE) \ + .toJSON(*store, PURE, HashFormat::SRI); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ + } + +JSON_TEST(pure, false) +JSON_TEST(impure, true) + +} diff --git a/src/libstore/tests/path.cc b/tests/unit/libstore/path.cc similarity index 61% rename from src/libstore/tests/path.cc rename to tests/unit/libstore/path.cc index efa35ef2b71..30631b5fd65 100644 --- a/src/libstore/tests/path.cc +++ b/tests/unit/libstore/path.cc @@ -39,6 +39,7 @@ TEST_DONT_PARSE(double_star, "**") TEST_DONT_PARSE(star_first, "*,foo") TEST_DONT_PARSE(star_second, "foo,*") TEST_DONT_PARSE(bang, "foo!o") +TEST_DONT_PARSE(dotfile, ".gitignore") #undef TEST_DONT_PARSE @@ -65,75 +66,6 @@ TEST_DO_PARSE(equals_sign, "foo=foo") #undef TEST_DO_PARSE -// For rapidcheck -void showValue(const StorePath & p, std::ostream & os) { - os << p.to_string(); -} - -} - -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - auto len = *gen::inRange( - 1, - StorePath::MaxPathLen - std::string_view { HASH_PART }.size()); - - std::string pre; - pre.reserve(len); - - for (size_t c = 0; c < len; ++c) { - switch (auto i = *gen::inRange(0, 10 + 2 * 26 + 6)) { - case 0 ... 9: - pre += '0' + i; - case 10 ... 35: - pre += 'A' + (i - 10); - break; - case 36 ... 61: - pre += 'a' + (i - 36); - break; - case 62: - pre += '+'; - break; - case 63: - pre += '-'; - break; - case 64: - pre += '.'; - break; - case 65: - pre += '_'; - break; - case 66: - pre += '?'; - break; - case 67: - pre += '='; - break; - default: - assert(false); - } - } - - return gen::just(StorePathName { - .name = std::move(pre), - }); -} - -Gen Arbitrary::arbitrary() -{ - return gen::just(StorePath { - *gen::arbitrary(), - (*gen::arbitrary()).name, - }); -} - -} // namespace rc - -namespace nix { - #ifndef COVERAGE RC_GTEST_FIXTURE_PROP( diff --git a/src/libstore/tests/references.cc b/tests/unit/libstore/references.cc similarity index 100% rename from src/libstore/tests/references.cc rename to tests/unit/libstore/references.cc diff --git a/tests/unit/libstore/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc new file mode 100644 index 00000000000..c8ac87a04ce --- /dev/null +++ b/tests/unit/libstore/serve-protocol.cc @@ -0,0 +1,279 @@ +#include + +#include +#include + +#include "serve-protocol.hh" +#include "serve-protocol-impl.hh" +#include "build-result.hh" +#include "tests/protocol.hh" +#include "tests/characterization.hh" + +namespace nix { + +const char serveProtoDir[] = "serve-protocol"; + +struct ServeProtoTest : VersionedProtoTest +{ + /** + * For serializers that don't care about the minimum version, we + * used the oldest one: 1.0. + */ + ServeProto::Version defaultVersion = 2 << 8 | 0; +}; + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + string, + "string", + defaultVersion, + (std::tuple { + "", + "hi", + "white rabbit", + "大白兔", + "oh no \0\0\0 what was that!", + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + storePath, + "store-path", + defaultVersion, + (std::tuple { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + contentAddress, + "content-address", + defaultVersion, + (std::tuple { + ContentAddress { + .method = TextIngestionMethod {}, + .hash = hashString(HashType::htSHA256, "Derive(...)"), + }, + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + ContentAddress { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + drvOutput, + "drv-output", + defaultVersion, + (std::tuple { + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + realisation, + "realisation", + defaultVersion, + (std::tuple { + Realisation { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + }, + Realisation { + .id = { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + .dependentRealisations = { + { + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_2, + "build-result-2.2", + 2 << 8 | 2, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::Built, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_3, + "build-result-2.3", + 2 << 8 | 3, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + BuildResult { + .status = BuildResult::Built, + .startTime = 30, + .stopTime = 50, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_6, + "build-result-2.6", + 2 << 8 | 6, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + BuildResult { + .status = BuildResult::Built, + .timesBuilt = 1, + .builtOutputs = { + { + "foo", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + { + "bar", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, + }, + }, + }, + .startTime = 30, + .stopTime = 50, +#if 0 + // These fields are not yet serialized. + // FIXME Include in next version of protocol or document + // why they are skipped. + .cpuUser = std::chrono::milliseconds(500s), + .cpuSystem = std::chrono::milliseconds(604s), +#endif + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + vector, + "vector", + defaultVersion, + (std::tuple, std::vector, std::vector, std::vector>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + set, + "set", + defaultVersion, + (std::tuple, std::set, std::set, std::set>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + optionalStorePath, + "optional-store-path", + defaultVersion, + (std::tuple, std::optional> { + std::nullopt, + std::optional { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + optionalContentAddress, + "optional-content-address", + defaultVersion, + (std::tuple, std::optional> { + std::nullopt, + std::optional { + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + }, + })) + +} diff --git a/src/libstore/tests/test-data/machines.bad_format b/tests/unit/libstore/test-data/machines.bad_format similarity index 100% rename from src/libstore/tests/test-data/machines.bad_format rename to tests/unit/libstore/test-data/machines.bad_format diff --git a/src/libstore/tests/test-data/machines.valid b/tests/unit/libstore/test-data/machines.valid similarity index 100% rename from src/libstore/tests/test-data/machines.valid rename to tests/unit/libstore/test-data/machines.valid diff --git a/tests/unit/libstore/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc new file mode 100644 index 00000000000..ad5943c69bc --- /dev/null +++ b/tests/unit/libstore/worker-protocol.cc @@ -0,0 +1,547 @@ +#include + +#include +#include + +#include "worker-protocol.hh" +#include "worker-protocol-impl.hh" +#include "derived-path.hh" +#include "build-result.hh" +#include "tests/protocol.hh" +#include "tests/characterization.hh" + +namespace nix { + +const char workerProtoDir[] = "worker-protocol"; + +struct WorkerProtoTest : VersionedProtoTest +{ + /** + * For serializers that don't care about the minimum version, we + * used the oldest one: 1.0. + */ + WorkerProto::Version defaultVersion = 1 << 8 | 0; +}; + + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + string, + "string", + defaultVersion, + (std::tuple { + "", + "hi", + "white rabbit", + "大白兔", + "oh no \0\0\0 what was that!", + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + storePath, + "store-path", + defaultVersion, + (std::tuple { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + contentAddress, + "content-address", + defaultVersion, + (std::tuple { + ContentAddress { + .method = TextIngestionMethod {}, + .hash = hashString(HashType::htSHA256, "Derive(...)"), + }, + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + ContentAddress { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + derivedPath_1_29, + "derived-path-1.29", + 1 << 8 | 29, + (std::tuple { + DerivedPath::Opaque { + .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::All { }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::Names { "x", "y" }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + derivedPath_1_30, + "derived-path-1.30", + 1 << 8 | 30, + (std::tuple { + DerivedPath::Opaque { + .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + DerivedPath::Opaque { + .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::All { }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::Names { "x", "y" }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + drvOutput, + "drv-output", + defaultVersion, + (std::tuple { + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + realisation, + "realisation", + defaultVersion, + (std::tuple { + Realisation { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + }, + Realisation { + .id = { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + .dependentRealisations = { + { + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_27, + "build-result-1.27", + 1 << 8 | 27, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::Built, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_28, + "build-result-1.28", + 1 << 8 | 28, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::Built, + .builtOutputs = { + { + "foo", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + { + "bar", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, + }, + }, + }, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_29, + "build-result-1.29", + 1 << 8 | 29, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + BuildResult { + .status = BuildResult::Built, + .timesBuilt = 1, + .builtOutputs = { + { + "foo", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + { + "bar", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, + }, + }, + }, + .startTime = 30, + .stopTime = 50, +#if 0 + // These fields are not yet serialized. + // FIXME Include in next version of protocol or document + // why they are skipped. + .cpuUser = std::chrono::milliseconds(500s), + .cpuSystem = std::chrono::milliseconds(604s), +#endif + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + keyedBuildResult_1_29, + "keyed-build-result-1.29", + 1 << 8 | 29, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + KeyedBuildResult { + { + .status = KeyedBuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + /* .path = */ DerivedPath::Opaque { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-xxx" }, + }, + }, + KeyedBuildResult { + { + .status = KeyedBuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + /* .path = */ DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::Names { "out" }, + }, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + unkeyedValidPathInfo_1_15, + "unkeyed-valid-path-info-1.15", + 1 << 8 | 15, + (std::tuple { + ({ + UnkeyedValidPathInfo info { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + ({ + UnkeyedValidPathInfo info { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv", + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + validPathInfo_1_15, + "valid-path-info-1.15", + 1 << 8 | 15, + (std::tuple { + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + // other reference + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo", + }, + // self reference + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + validPathInfo_1_16, + "valid-path-info-1.16", + 1 << 8 | 16, + (std::tuple { + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info.ultimate = true; + info; + }), + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + // other reference + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo", + }, + // self reference + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info.sigs = { + "fake-sig-1", + "fake-sig-2", + }, + info; + }), + ({ + ValidPathInfo info { + *LibStoreTest::store, + "foo", + FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + .references = { + .others = { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }, + .self = true, + }, + }, + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + optionalTrustedFlag, + "optional-trusted-flag", + defaultVersion, + (std::tuple, std::optional, std::optional> { + std::nullopt, + std::optional { Trusted }, + std::optional { NotTrusted }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + vector, + "vector", + defaultVersion, + (std::tuple, std::vector, std::vector, std::vector>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + set, + "set", + defaultVersion, + (std::tuple, std::set, std::set, std::set>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + optionalStorePath, + "optional-store-path", + defaultVersion, + (std::tuple, std::optional> { + std::nullopt, + std::optional { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + optionalContentAddress, + "optional-content-address", + defaultVersion, + (std::tuple, std::optional> { + std::nullopt, + std::optional { + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + }, + })) + +} diff --git a/tests/unit/libutil-support/local.mk b/tests/unit/libutil-support/local.mk new file mode 100644 index 00000000000..2ee2cdb6cd7 --- /dev/null +++ b/tests/unit/libutil-support/local.mk @@ -0,0 +1,19 @@ +libraries += libutil-test-support + +libutil-test-support_NAME = libnixutil-test-support + +libutil-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libutil-test-support_INSTALL_DIR := $(checklibdir) +else + libutil-test-support_INSTALL_DIR := +endif + +libutil-test-support_SOURCES := $(wildcard $(d)/tests/*.cc) + +libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) + +libutil-test-support_LIBS = libutil + +libutil-test-support_LDFLAGS := -pthread -lrapidcheck diff --git a/tests/unit/libutil-support/tests/characterization.hh b/tests/unit/libutil-support/tests/characterization.hh new file mode 100644 index 00000000000..9d6c850f017 --- /dev/null +++ b/tests/unit/libutil-support/tests/characterization.hh @@ -0,0 +1,108 @@ +#pragma once +///@file + +#include + +#include "types.hh" +#include "environment-variables.hh" + +namespace nix { + +/** + * The path to the unit test data directory. See the contributing guide + * in the manual for further details. + */ +static Path getUnitTestData() { + return getEnv("_NIX_TEST_UNIT_DATA").value(); +} + +/** + * Whether we should update "golden masters" instead of running tests + * against them. See the contributing guide in the manual for further + * details. + */ +static bool testAccept() { + return getEnv("_NIX_TEST_ACCEPT") == "1"; +} + +/** + * Mixin class for writing characterization tests + */ +class CharacterizationTest : public virtual ::testing::Test +{ +protected: + /** + * While the "golden master" for this characterization test is + * located. It should not be shared with any other test. + */ + virtual Path goldenMaster(PathView testStem) const = 0; + +public: + /** + * Golden test for reading + * + * @param test hook that takes the contents of the file and does the + * actual work + */ + void readTest(PathView testStem, auto && test) + { + auto file = goldenMaster(testStem); + + if (testAccept()) + { + GTEST_SKIP() + << "Cannot read golden master " + << file + << "because another test is also updating it"; + } + else + { + test(readFile(file)); + } + } + + /** + * Golden test for writing + * + * @param test hook that produces contents of the file and does the + * actual work + */ + void writeTest( + PathView testStem, auto && test, auto && readFile2, auto && writeFile2) + { + auto file = goldenMaster(testStem); + + auto got = test(); + + if (testAccept()) + { + createDirs(dirOf(file)); + writeFile2(file, got); + GTEST_SKIP() + << "Updating golden master " + << file; + } + else + { + decltype(got) expected = readFile2(file); + ASSERT_EQ(got, expected); + } + } + + /** + * Specialize to `std::string` + */ + void writeTest(PathView testStem, auto && test) + { + writeTest( + testStem, test, + [](const Path & f) -> std::string { + return readFile(f); + }, + [](const Path & f, const std::string & c) { + return writeFile(f, c); + }); + } +}; + +} diff --git a/tests/unit/libutil-support/tests/hash.cc b/tests/unit/libutil-support/tests/hash.cc new file mode 100644 index 00000000000..577e9890e59 --- /dev/null +++ b/tests/unit/libutil-support/tests/hash.cc @@ -0,0 +1,20 @@ +#include + +#include + +#include "hash.hh" + +#include "tests/hash.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + Hash hash(htSHA1); + for (size_t i = 0; i < hash.hashSize; ++i) + hash.hash[i] = *gen::arbitrary(); + return gen::just(hash); +} + +} diff --git a/src/libutil/tests/hash.hh b/tests/unit/libutil-support/tests/hash.hh similarity index 100% rename from src/libutil/tests/hash.hh rename to tests/unit/libutil-support/tests/hash.hh diff --git a/tests/unit/libutil/args.cc b/tests/unit/libutil/args.cc new file mode 100644 index 00000000000..95022443006 --- /dev/null +++ b/tests/unit/libutil/args.cc @@ -0,0 +1,168 @@ +#include "args.hh" +#include "fs-sink.hh" +#include + +#include +#include + +namespace nix { + + TEST(parseShebangContent, basic) { + std::list r = parseShebangContent("hi there"); + ASSERT_EQ(r.size(), 2); + auto i = r.begin(); + ASSERT_EQ(*i++, "hi"); + ASSERT_EQ(*i++, "there"); + } + + TEST(parseShebangContent, empty) { + std::list r = parseShebangContent(""); + ASSERT_EQ(r.size(), 0); + } + + TEST(parseShebangContent, doubleBacktick) { + std::list r = parseShebangContent("``\"ain't that nice\"``"); + ASSERT_EQ(r.size(), 1); + auto i = r.begin(); + ASSERT_EQ(*i++, "\"ain't that nice\""); + } + + TEST(parseShebangContent, doubleBacktickEmpty) { + std::list r = parseShebangContent("````"); + ASSERT_EQ(r.size(), 1); + auto i = r.begin(); + ASSERT_EQ(*i++, ""); + } + + TEST(parseShebangContent, doubleBacktickMarkdownInlineCode) { + std::list r = parseShebangContent("``# I'm markdown section about `coolFunction` ``"); + ASSERT_EQ(r.size(), 1); + auto i = r.begin(); + ASSERT_EQ(*i++, "# I'm markdown section about `coolFunction`"); + } + + TEST(parseShebangContent, doubleBacktickMarkdownCodeBlockNaive) { + std::list r = parseShebangContent("``Example 1\n```nix\na: a\n``` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "Example 1\n``nix\na: a\n``"); + } + + TEST(parseShebangContent, doubleBacktickMarkdownCodeBlockCorrect) { + std::list r = parseShebangContent("``Example 1\n````nix\na: a\n```` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "Example 1\n```nix\na: a\n```"); + } + + TEST(parseShebangContent, doubleBacktickMarkdownCodeBlock2) { + std::list r = parseShebangContent("``Example 1\n````nix\na: a\n````\nExample 2\n````nix\na: a\n```` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "Example 1\n```nix\na: a\n```\nExample 2\n```nix\na: a\n```"); + } + + TEST(parseShebangContent, singleBacktickInDoubleBacktickQuotes) { + std::list r = parseShebangContent("``` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "`"); + } + + TEST(parseShebangContent, singleBacktickAndSpaceInDoubleBacktickQuotes) { + std::list r = parseShebangContent("``` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "` "); + } + + TEST(parseShebangContent, doubleBacktickInDoubleBacktickQuotes) { + std::list r = parseShebangContent("````` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "``"); + } + + TEST(parseShebangContent, increasingQuotes) { + std::list r = parseShebangContent("```` ``` `` ````` `` `````` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 4); + ASSERT_EQ(*i++, ""); + ASSERT_EQ(*i++, "`"); + ASSERT_EQ(*i++, "``"); + ASSERT_EQ(*i++, "```"); + } + + +#ifndef COVERAGE + +// quick and dirty +static inline std::string escape(std::string_view s_) { + + std::string_view s = s_; + std::string r = "``"; + + // make a guess to allocate ahead of time + r.reserve( + // plain chars + s.size() + // quotes + + 5 + // some "escape" backticks + + s.size() / 8); + + while (!s.empty()) { + if (s[0] == '`' && s.size() >= 2 && s[1] == '`') { + // escape it + r += "`"; + while (!s.empty() && s[0] == '`') { + r += "`"; + s = s.substr(1); + } + } else { + r += s[0]; + s = s.substr(1); + } + } + + if (!r.empty() + && ( + r[r.size() - 1] == '`' + || r[r.size() - 1] == ' ' + )) { + r += " "; + } + + r += "``"; + + return r; +}; + +RC_GTEST_PROP( + parseShebangContent, + prop_round_trip_single, + (const std::string & orig)) +{ + auto escaped = escape(orig); + // RC_LOG() << "escaped: <[[" << escaped << "]]>" << std::endl; + auto ss = parseShebangContent(escaped); + RC_ASSERT(ss.size() == 1); + RC_ASSERT(*ss.begin() == orig); +} + +RC_GTEST_PROP( + parseShebangContent, + prop_round_trip_two, + (const std::string & one, const std::string & two)) +{ + auto ss = parseShebangContent(escape(one) + " " + escape(two)); + RC_ASSERT(ss.size() == 2); + auto i = ss.begin(); + RC_ASSERT(*i++ == one); + RC_ASSERT(*i++ == two); +} + + +#endif + +} diff --git a/src/libutil/tests/canon-path.cc b/tests/unit/libutil/canon-path.cc similarity index 100% rename from src/libutil/tests/canon-path.cc rename to tests/unit/libutil/canon-path.cc diff --git a/src/libutil/tests/chunked-vector.cc b/tests/unit/libutil/chunked-vector.cc similarity index 100% rename from src/libutil/tests/chunked-vector.cc rename to tests/unit/libutil/chunked-vector.cc diff --git a/src/libutil/tests/closure.cc b/tests/unit/libutil/closure.cc similarity index 100% rename from src/libutil/tests/closure.cc rename to tests/unit/libutil/closure.cc diff --git a/src/libutil/tests/compression.cc b/tests/unit/libutil/compression.cc similarity index 100% rename from src/libutil/tests/compression.cc rename to tests/unit/libutil/compression.cc diff --git a/src/libutil/tests/config.cc b/tests/unit/libutil/config.cc similarity index 100% rename from src/libutil/tests/config.cc rename to tests/unit/libutil/config.cc diff --git a/tests/unit/libutil/data/git/check-data.sh b/tests/unit/libutil/data/git/check-data.sh new file mode 100644 index 00000000000..b3f59c4f12a --- /dev/null +++ b/tests/unit/libutil/data/git/check-data.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/git-hashing/check-data +mkdir -p $TEST_ROOT + +repo="$TEST_ROOT/scratch" +git init "$repo" + +git -C "$repo" config user.email "you@example.com" +git -C "$repo" config user.name "Your Name" + +# `-w` to write for tree test +freshlyAddedHash=$(git -C "$repo" hash-object -w -t blob --stdin < "./hello-world.bin") +encodingHash=$(sha1sum -b < "./hello-world-blob.bin" | head -c 40) + +# If the hashes match, then `hello-world-blob.bin` must be the encoding +# of `hello-world.bin`. +[[ "$encodingHash" == "$freshlyAddedHash" ]] + +# Create empty directory object for tree test +echo -n | git -C "$repo" hash-object -w -t tree --stdin + +# Relies on both child hashes already existing in the git store +freshlyAddedHash=$(git -C "$repo" mktree < "./tree.txt") +encodingHash=$(sha1sum -b < "./tree.bin" | head -c 40) + +# If the hashes match, then `tree.bin` must be the encoding of the +# directory denoted by `tree.txt` interpreted as git directory listing. +[[ "$encodingHash" == "$freshlyAddedHash" ]] diff --git a/tests/unit/libutil/data/git/hello-world-blob.bin b/tests/unit/libutil/data/git/hello-world-blob.bin new file mode 100644 index 00000000000..255f5df55cc Binary files /dev/null and b/tests/unit/libutil/data/git/hello-world-blob.bin differ diff --git a/tests/unit/libutil/data/git/hello-world.bin b/tests/unit/libutil/data/git/hello-world.bin new file mode 100644 index 00000000000..63ddb340119 Binary files /dev/null and b/tests/unit/libutil/data/git/hello-world.bin differ diff --git a/tests/unit/libutil/data/git/tree.bin b/tests/unit/libutil/data/git/tree.bin new file mode 100644 index 00000000000..5256ec14070 Binary files /dev/null and b/tests/unit/libutil/data/git/tree.bin differ diff --git a/tests/unit/libutil/data/git/tree.txt b/tests/unit/libutil/data/git/tree.txt new file mode 100644 index 00000000000..be3d02920c9 --- /dev/null +++ b/tests/unit/libutil/data/git/tree.txt @@ -0,0 +1,3 @@ +100644 blob 63ddb340119baf8492d2da53af47e8c7cfcd5eb2 Foo +100755 blob 63ddb340119baf8492d2da53af47e8c7cfcd5eb2 bAr +040000 tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904 baZ diff --git a/tests/unit/libutil/git.cc b/tests/unit/libutil/git.cc new file mode 100644 index 00000000000..551a2d10522 --- /dev/null +++ b/tests/unit/libutil/git.cc @@ -0,0 +1,236 @@ +#include + +#include "git.hh" +#include "memory-source-accessor.hh" + +#include "tests/characterization.hh" + +namespace nix { + +using namespace git; + +class GitTest : public CharacterizationTest +{ + Path unitTestData = getUnitTestData() + "/git"; + +public: + + Path goldenMaster(std::string_view testStem) const override { + return unitTestData + "/" + testStem; + } + + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + +private: + + void SetUp() override + { + mockXpSettings.set("experimental-features", "git-hashing"); + } +}; + +TEST(GitMode, gitMode_directory) { + Mode m = Mode::Directory; + RawMode r = 0040000; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST(GitMode, gitMode_executable) { + Mode m = Mode::Executable; + RawMode r = 0100755; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST(GitMode, gitMode_regular) { + Mode m = Mode::Regular; + RawMode r = 0100644; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST(GitMode, gitMode_symlink) { + Mode m = Mode::Symlink; + RawMode r = 0120000; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST_F(GitTest, blob_read) { + readTest("hello-world-blob.bin", [&](const auto & encoded) { + StringSource in { encoded }; + StringSink out; + RegularFileSink out2 { out }; + parse(out2, "", in, [](auto &, auto) {}, mockXpSettings); + + auto expected = readFile(goldenMaster("hello-world.bin")); + + ASSERT_EQ(out.s, expected); + }); +} + +TEST_F(GitTest, blob_write) { + writeTest("hello-world-blob.bin", [&]() { + auto decoded = readFile(goldenMaster("hello-world.bin")); + StringSink s; + dumpBlobPrefix(decoded.size(), s, mockXpSettings); + s(decoded); + return s.s; + }); +} + +/** + * This data is for "shallow" tree tests. However, we use "real" hashes + * so that we can check our test data in a small shell script test test + * (`tests/unit/libutil/data/git/check-data.sh`). + */ +const static Tree tree = { + { + "Foo", + { + .mode = Mode::Regular, + // hello world with special chars from above + .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", htSHA1), + }, + }, + { + "bAr", + { + .mode = Mode::Executable, + // ditto + .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", htSHA1), + }, + }, + { + "baZ/", + { + .mode = Mode::Directory, + // Empty directory hash + .hash = Hash::parseAny("4b825dc642cb6eb9a060e54bf8d69288fbee4904", htSHA1), + }, + }, +}; + +TEST_F(GitTest, tree_read) { + readTest("tree.bin", [&](const auto & encoded) { + StringSource in { encoded }; + NullParseSink out; + Tree got; + parse(out, "", in, [&](auto & name, auto entry) { + auto name2 = name; + if (entry.mode == Mode::Directory) + name2 += '/'; + got.insert_or_assign(name2, std::move(entry)); + }, mockXpSettings); + + ASSERT_EQ(got, tree); + }); +} + +TEST_F(GitTest, tree_write) { + writeTest("tree.bin", [&]() { + StringSink s; + dumpTree(tree, s, mockXpSettings); + return s.s; + }); +} + +TEST_F(GitTest, both_roundrip) { + using File = MemorySourceAccessor::File; + + MemorySourceAccessor files; + files.root = File::Directory { + .contents { + { + "foo", + File::Regular { + .contents = "hello\n\0\n\tworld!", + }, + }, + { + "bar", + File::Directory { + .contents = { + { + "baz", + File::Regular { + .executable = true, + .contents = "good day,\n\0\n\tworld!", + }, + }, + }, + }, + }, + }, + }; + + std::map cas; + + std::function dumpHook; + dumpHook = [&](const CanonPath & path) { + StringSink s; + HashSink hashSink { htSHA1 }; + TeeSink s2 { s, hashSink }; + auto mode = dump( + files, path, s2, dumpHook, + defaultPathFilter, mockXpSettings); + auto hash = hashSink.finish().first; + cas.insert_or_assign(hash, std::move(s.s)); + return TreeEntry { + .mode = mode, + .hash = hash, + }; + }; + + auto root = dumpHook(CanonPath::root); + + MemorySourceAccessor files2; + + MemorySink sinkFiles2 { files2 }; + + std::function mkSinkHook; + mkSinkHook = [&](const Path prefix, const Hash & hash) { + StringSource in { cas[hash] }; + parse(sinkFiles2, prefix, in, [&](const Path & name, const auto & entry) { + mkSinkHook(prefix + "/" + name, entry.hash); + }, mockXpSettings); + }; + + mkSinkHook("", root.hash); + + ASSERT_EQ(files, files2); +} + +TEST(GitLsRemote, parseSymrefLineWithReference) { + auto line = "ref: refs/head/main HEAD"; + auto res = parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, LsRemoteRefLine::Kind::Symbolic); + ASSERT_EQ(res->target, "refs/head/main"); + ASSERT_EQ(res->reference, "HEAD"); +} + +TEST(GitLsRemote, parseSymrefLineWithNoReference) { + auto line = "ref: refs/head/main"; + auto res = parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, LsRemoteRefLine::Kind::Symbolic); + ASSERT_EQ(res->target, "refs/head/main"); + ASSERT_EQ(res->reference, std::nullopt); +} + +TEST(GitLsRemote, parseObjectRefLine) { + auto line = "abc123 refs/head/main"; + auto res = parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, LsRemoteRefLine::Kind::Object); + ASSERT_EQ(res->target, "abc123"); + ASSERT_EQ(res->reference, "refs/head/main"); +} + +} diff --git a/src/libutil/tests/hash.cc b/tests/unit/libutil/hash.cc similarity index 65% rename from src/libutil/tests/hash.cc rename to tests/unit/libutil/hash.cc index e4e928b3b40..92291afce28 100644 --- a/src/libutil/tests/hash.cc +++ b/tests/unit/libutil/hash.cc @@ -1,12 +1,8 @@ #include -#include #include -#include -#include - -#include "tests/hash.hh" +#include "hash.hh" namespace nix { @@ -18,28 +14,28 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc1321 auto s1 = ""; auto hash = hashString(HashType::htMD5, s1); - ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e"); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e"); } TEST(hashString, testKnownMD5Hashes2) { // values taken from: https://tools.ietf.org/html/rfc1321 auto s2 = "abc"; auto hash = hashString(HashType::htMD5, s2); - ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72"); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72"); } TEST(hashString, testKnownSHA1Hashes1) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abc"; auto hash = hashString(HashType::htSHA1, s); - ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); } TEST(hashString, testKnownSHA1Hashes2) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; auto hash = hashString(HashType::htSHA1, s); - ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); } TEST(hashString, testKnownSHA256Hashes1) { @@ -47,7 +43,7 @@ namespace nix { auto s = "abc"; auto hash = hashString(HashType::htSHA256, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); } @@ -55,7 +51,7 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; auto hash = hashString(HashType::htSHA256, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha256:248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); } @@ -63,33 +59,34 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abc"; auto hash = hashString(HashType::htSHA512, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha512:ddaf35a193617abacc417349ae20413112e6fa4e89a9" "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd" "454d4423643ce80e2a9ac94fa54ca49f"); } - TEST(hashString, testKnownSHA512Hashes2) { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; auto hash = hashString(HashType::htSHA512, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha512:8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa1" "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a" "c7d329eeb6dd26545e96e55b874be909"); } -} -namespace rc { -using namespace nix; + /* ---------------------------------------------------------------------------- + * parseHashFormat, parseHashFormatOpt, printHashFormat + * --------------------------------------------------------------------------*/ -Gen Arbitrary::arbitrary() -{ - Hash hash(htSHA1); - for (size_t i = 0; i < hash.hashSize; ++i) - hash.hash[i] = *gen::arbitrary(); - return gen::just(hash); -} + TEST(hashFormat, testRoundTripPrintParse) { + for (const HashFormat hashFormat: { HashFormat::Base64, HashFormat::Base32, HashFormat::Base16, HashFormat::SRI}) { + ASSERT_EQ(parseHashFormat(printHashFormat(hashFormat)), hashFormat); + ASSERT_EQ(*parseHashFormatOpt(printHashFormat(hashFormat)), hashFormat); + } + } + TEST(hashFormat, testParseHashFormatOptException) { + ASSERT_EQ(parseHashFormatOpt("sha0042"), std::nullopt); + } } diff --git a/src/libutil/tests/hilite.cc b/tests/unit/libutil/hilite.cc similarity index 100% rename from src/libutil/tests/hilite.cc rename to tests/unit/libutil/hilite.cc diff --git a/tests/unit/libutil/local.mk b/tests/unit/libutil/local.mk new file mode 100644 index 00000000000..930efb90bbd --- /dev/null +++ b/tests/unit/libutil/local.mk @@ -0,0 +1,31 @@ +check: libutil-tests_RUN + +programs += libutil-tests + +libutil-tests_NAME = libnixutil-tests + +libutil-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data + +libutil-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libutil-tests_INSTALL_DIR := $(checkbindir) +else + libutil-tests_INSTALL_DIR := +endif + +libutil-tests_SOURCES := $(wildcard $(d)/*.cc) + +libutil-tests_EXTRA_INCLUDES = \ + -I tests/unit/libutil-support \ + -I src/libutil + +libutil-tests_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) + +libutil-tests_LIBS = libutil-test-support libutil + +libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) + +check: $(d)/data/git/check-data.sh.test + +$(eval $(call run-test,$(d)/data/git/check-data.sh)) diff --git a/src/libutil/tests/logging.cc b/tests/unit/libutil/logging.cc similarity index 99% rename from src/libutil/tests/logging.cc rename to tests/unit/libutil/logging.cc index 2ffdc2e9b88..c6dfe63d39b 100644 --- a/src/libutil/tests/logging.cc +++ b/tests/unit/libutil/logging.cc @@ -2,7 +2,6 @@ #include "logging.hh" #include "nixexpr.hh" -#include "util.hh" #include #include diff --git a/src/libutil/tests/lru-cache.cc b/tests/unit/libutil/lru-cache.cc similarity index 100% rename from src/libutil/tests/lru-cache.cc rename to tests/unit/libutil/lru-cache.cc diff --git a/src/libutil/tests/pool.cc b/tests/unit/libutil/pool.cc similarity index 100% rename from src/libutil/tests/pool.cc rename to tests/unit/libutil/pool.cc diff --git a/src/libutil/tests/references.cc b/tests/unit/libutil/references.cc similarity index 100% rename from src/libutil/tests/references.cc rename to tests/unit/libutil/references.cc diff --git a/src/libutil/tests/suggestions.cc b/tests/unit/libutil/suggestions.cc similarity index 100% rename from src/libutil/tests/suggestions.cc rename to tests/unit/libutil/suggestions.cc diff --git a/src/libutil/tests/tests.cc b/tests/unit/libutil/tests.cc similarity index 99% rename from src/libutil/tests/tests.cc rename to tests/unit/libutil/tests.cc index f3c1e8248a0..568f03f702d 100644 --- a/src/libutil/tests/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -1,5 +1,8 @@ #include "util.hh" #include "types.hh" +#include "file-system.hh" +#include "processes.hh" +#include "terminal.hh" #include #include diff --git a/src/libutil/tests/url.cc b/tests/unit/libutil/url.cc similarity index 89% rename from src/libutil/tests/url.cc rename to tests/unit/libutil/url.cc index a908631e641..7d08f467eac 100644 --- a/src/libutil/tests/url.cc +++ b/tests/unit/libutil/url.cc @@ -335,4 +335,36 @@ namespace nix { ASSERT_EQ(d, s); } + TEST(percentEncode, yen) { + // https://en.wikipedia.org/wiki/Percent-encoding#Character_data + std::string s = reinterpret_cast(u8"円"); + std::string e = "%E5%86%86"; + + ASSERT_EQ(percentEncode(s), e); + ASSERT_EQ(percentDecode(e), s); + } + +TEST(nix, isValidSchemeName) { + ASSERT_TRUE(isValidSchemeName("http")); + ASSERT_TRUE(isValidSchemeName("https")); + ASSERT_TRUE(isValidSchemeName("file")); + ASSERT_TRUE(isValidSchemeName("file+https")); + ASSERT_TRUE(isValidSchemeName("fi.le")); + ASSERT_TRUE(isValidSchemeName("file-ssh")); + ASSERT_TRUE(isValidSchemeName("file+")); + ASSERT_TRUE(isValidSchemeName("file.")); + ASSERT_TRUE(isValidSchemeName("file1")); + ASSERT_FALSE(isValidSchemeName("file:")); + ASSERT_FALSE(isValidSchemeName("file/")); + ASSERT_FALSE(isValidSchemeName("+file")); + ASSERT_FALSE(isValidSchemeName(".file")); + ASSERT_FALSE(isValidSchemeName("-file")); + ASSERT_FALSE(isValidSchemeName("1file")); + // regex ok? + ASSERT_FALSE(isValidSchemeName("\nhttp")); + ASSERT_FALSE(isValidSchemeName("\nhttp\n")); + ASSERT_FALSE(isValidSchemeName("http\n")); + ASSERT_FALSE(isValidSchemeName("http ")); +} + } diff --git a/src/libutil/tests/xml-writer.cc b/tests/unit/libutil/xml-writer.cc similarity index 100% rename from src/libutil/tests/xml-writer.cc rename to tests/unit/libutil/xml-writer.cc