diff --git a/.dir-locals.el b/.dir-locals.el index 2d1117f4bda..8b21d4e403a 100644 --- a/.dir-locals.el +++ b/.dir-locals.el @@ -13,4 +13,5 @@ (eval . (c-set-offset 'arglist-cont-nonempty '+)) (eval . (c-set-offset 'substatement-open 0)) (eval . (c-set-offset 'access-label '-)) + (eval . (c-set-offset 'inlambda 0)) ))) diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000000..887ecadba59 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,26 @@ +# EditorConfig configuration for nix +# http://EditorConfig.org + +# Top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file, utf-8 charset +[*] +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +charset = utf-8 + +# Match nix files, set indent to spaces with width of two +[*.nix] +indent_style = space +indent_size = 2 + +# Match c++/shell/perl, set indent to spaces with width of four +[*.{hpp,cc,hh,sh,pl}] +indent_style = space +indent_size = 4 + +# Match diffs, avoid to trim trailing whitespace +[*.{diff,patch}] +trim_trailing_whitespace = false diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000000..e6d346bc1cf --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,32 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +**Describe the bug** + +A clear and concise description of what the bug is. + +If you have a problem with a specific package or NixOS, +you probably want to file an issue at https://github.com/NixOS/nixpkgs/issues. + +**Steps To Reproduce** + +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** + +A clear and concise description of what you expected to happen. + +**`nix-env --version` output** + +**Additional context** + +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000000..392ed30c66c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: improvement +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000000..7feefc85588 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,14 @@ +name: "Test" +on: + pull_request: + push: +jobs: + tests: + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - uses: cachix/install-nix-action@v8 + - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings diff --git a/.gitignore b/.gitignore index 92f95fe1fcb..9830265701d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,20 +1,19 @@ Makefile.config +perl/Makefile.config # / /aclocal.m4 /autom4te.cache +/precompiled-headers.h.gch +/precompiled-headers.h.pch /config.* /configure -/nix.spec /stamp-h1 /svn-revision /libtool /corepkgs/config.nix -# /corepkgs/buildenv/ -/corepkgs/buildenv/builder.pl - # /corepkgs/channels/ /corepkgs/channels/unpack.sh @@ -35,9 +34,9 @@ Makefile.config # /scripts/ /scripts/nix-profile.sh /scripts/nix-copy-closure -/scripts/build-remote.pl /scripts/nix-reduce-build /scripts/nix-http-export.cgi +/scripts/nix-profile-daemon.sh # /src/libexpr/ /src/libexpr/lexer-tab.cc @@ -48,8 +47,10 @@ Makefile.config /src/libexpr/nix.tbl # /src/libstore/ -/src/libstore/schema.sql.hh -/src/libstore/sandbox-defaults.sb +*.gen.* + +# /src/libutil/ +/src/libutil/tests/libutil-tests /src/nix/nix @@ -72,15 +73,13 @@ Makefile.config # /src/nix-channel/ /src/nix-channel/nix-channel -# /src/download-via-ssh/ -/src/download-via-ssh/download-via-ssh - -# /src/buildenv/ -/src/buildenv/buildenv - # /src/nix-build/ /src/nix-build/nix-build +/src/nix-copy-closure/nix-copy-closure + +/src/error-demo/error-demo + /src/build-remote/build-remote # /tests/ @@ -88,6 +87,10 @@ Makefile.config /tests/common.sh /tests/dummy /tests/result* +/tests/restricted-innocent +/tests/shell +/tests/shell.drv +/tests/config.nix # /tests/lang/ /tests/lang/*.out @@ -101,19 +104,25 @@ Makefile.config /misc/systemd/nix-daemon.socket /misc/upstart/nix-daemon.conf +/src/resolve-system-dependencies/resolve-system-dependencies + inst/ *.a *.o *.so +*.dylib *.dll *.exe *.dep *~ *.pc +*.plist # GNU Global GPATH GRTAGS GSYMS GTAGS + +nix-rust/target diff --git a/.version b/.version new file mode 100644 index 00000000000..7208c218290 --- /dev/null +++ b/.version @@ -0,0 +1 @@ +2.4 \ No newline at end of file diff --git a/Makefile b/Makefile index 14be271bb10..0ba011e2ad0 100644 --- a/Makefile +++ b/Makefile @@ -1,42 +1,34 @@ makefiles = \ + mk/precompiled-headers.mk \ local.mk \ - src/boost/format/local.mk \ + nix-rust/local.mk \ src/libutil/local.mk \ + src/libutil/tests/local.mk \ src/libstore/local.mk \ + src/libfetchers/local.mk \ src/libmain/local.mk \ src/libexpr/local.mk \ src/nix/local.mk \ - src/nix-store/local.mk \ - src/nix-instantiate/local.mk \ - src/nix-env/local.mk \ - src/nix-daemon/local.mk \ - src/nix-collect-garbage/local.mk \ - src/nix-prefetch-url/local.mk \ - src/buildenv/local.mk \ src/resolve-system-dependencies/local.mk \ - src/nix-channel/local.mk \ - src/nix-build/local.mk \ - src/build-remote/local.mk \ - perl/local.mk \ scripts/local.mk \ corepkgs/local.mk \ misc/systemd/local.mk \ misc/launchd/local.mk \ misc/upstart/local.mk \ - misc/emacs/local.mk \ doc/manual/local.mk \ - tests/local.mk - #src/download-via-ssh/local.mk \ - -GLOBAL_CXXFLAGS += -std=c++14 -g -Wall + tests/local.mk \ + tests/plugins/local.mk -include Makefile.config OPTIMIZE = 1 ifeq ($(OPTIMIZE), 1) - GLOBAL_CFLAGS += -O3 GLOBAL_CXXFLAGS += -O3 +else + GLOBAL_CXXFLAGS += -O0 endif include mk/lib.mk + +GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++17 diff --git a/Makefile.config.in b/Makefile.config.in index 15e94380477..b632444e8ab 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -1,36 +1,44 @@ +AR = @AR@ BDW_GC_LIBS = @BDW_GC_LIBS@ +BOOST_LDFLAGS = @BOOST_LDFLAGS@ +BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@ CC = @CC@ CFLAGS = @CFLAGS@ CXX = @CXX@ CXXFLAGS = @CXXFLAGS@ +EDITLINE_LIBS = @EDITLINE_LIBS@ ENABLE_S3 = @ENABLE_S3@ +GTEST_LIBS = @GTEST_LIBS@ +HAVE_SECCOMP = @HAVE_SECCOMP@ HAVE_SODIUM = @HAVE_SODIUM@ +LDFLAGS = @LDFLAGS@ +LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@ +LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ LIBCURL_LIBS = @LIBCURL_LIBS@ +LIBLZMA_LIBS = @LIBLZMA_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ PACKAGE_NAME = @PACKAGE_NAME@ PACKAGE_VERSION = @PACKAGE_VERSION@ SODIUM_LIBS = @SODIUM_LIBS@ -LIBLZMA_LIBS = @LIBLZMA_LIBS@ SQLITE3_LIBS = @SQLITE3_LIBS@ bash = @bash@ bindir = @bindir@ -curl = @curl@ datadir = @datadir@ datarootdir = @datarootdir@ +doc_generate = @doc_generate@ docdir = @docdir@ exec_prefix = @exec_prefix@ includedir = @includedir@ libdir = @libdir@ libexecdir = @libexecdir@ localstatedir = @localstatedir@ +lsof = @lsof@ mandir = @mandir@ -perl = @perl@ -perlbindings = @perlbindings@ -perllibdir = @perllibdir@ pkglibdir = $(libdir)/$(PACKAGE_NAME) prefix = @prefix@ +sandbox_shell = @sandbox_shell@ storedir = @storedir@ sysconfdir = @sysconfdir@ -doc_generate = @doc_generate@ +system = @system@ xmllint = @xmllint@ xsltproc = @xsltproc@ diff --git a/README.md b/README.md index 1eb73b256f5..a1588284dcf 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,54 @@ -Nix, the purely functional package manager ------------------------------------------- +# Nix -Nix is a new take on package management that is fairly unique. Because of it's -purity aspects, a lot of issues found in traditional package managers don't -appear with Nix. +[![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos) +[![Test](https://github.com/NixOS/nix/workflows/Test/badge.svg)](https://github.com/NixOS/nix/actions) -To find out more about the tool, usage and installation instructions, please -read the manual, which is available on the Nix website at -. +Nix is a powerful package manager for Linux and other Unix systems that makes package +management reliable and reproducible. Please refer to the [Nix manual](https://nixos.org/nix/manual) +for more details. -## Contributing +## Installation -Take a look at the [Hacking Section](http://nixos.org/nix/manual/#chap-hacking) -of the manual. It helps you to get started with building Nix from source. +On Linux and macOS the easiest way to Install Nix is to run the following shell command +(as a user other than root): -## License +``` +$ curl -L https://nixos.org/nix/install | sh +``` + +Information on additional installation methods is available on the [Nix download page](https://nixos.org/download.html). + +## Building And Developing + +### Building Nix + +You can build Nix using one of the targets provided by [release.nix](./release.nix): + +``` +$ nix-build ./release.nix -A build.aarch64-linux +$ nix-build ./release.nix -A build.x86_64-darwin +$ nix-build ./release.nix -A build.i686-linux +$ nix-build ./release.nix -A build.x86_64-linux +``` -Nix is released under the LGPL v2.1 +### Development Environment + +You can use the provided `shell.nix` to get a working development environment: + +``` +$ nix-shell +$ ./bootstrap.sh +$ ./configure +$ make +``` + +## Additional Resources + +- [Nix manual](https://nixos.org/nix/manual) +- [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix) +- [NixOS Discourse](https://discourse.nixos.org/) +- [IRC - #nixos on freenode.net](irc://irc.freenode.net/#nixos) + +## License -This product includes software developed by the OpenSSL Project for -use in the [OpenSSL Toolkit](http://www.OpenSSL.org/). +Nix is released under the [LGPL v2.1](./COPYING). diff --git a/config/config.guess b/config/config.guess index 137bedf2e28..d4fb3213ec7 100755 --- a/config/config.guess +++ b/config/config.guess @@ -1,14 +1,12 @@ #! /bin/sh # Attempt to guess a canonical system name. -# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, -# 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -# 2011, 2012 Free Software Foundation, Inc. +# Copyright 1992-2018 Free Software Foundation, Inc. -timestamp='2012-08-14' +timestamp='2018-08-02' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or +# the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, but @@ -17,24 +15,22 @@ timestamp='2012-08-14' # General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program; if not, see . +# along with this program; if not, see . # # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a # configuration script generated by Autoconf, you may include it under -# the same distribution terms that you use for the rest of that program. - - -# Originally written by Per Bothner. Please send patches (context -# diff format) to and include a ChangeLog -# entry. +# the same distribution terms that you use for the rest of that +# program. This Exception is an additional permission under section 7 +# of the GNU General Public License, version 3 ("GPLv3"). # -# This script attempts to guess a canonical system name similar to -# config.sub. If it succeeds, it prints the system name on stdout, and -# exits with 0. Otherwise, it exits with 1. +# Originally written by Per Bothner; maintained since 2000 by Ben Elliston. # # You can get the latest version of this script from: -# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD +# https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess +# +# Please send patches to . + me=`echo "$0" | sed -e 's,.*/,,'` @@ -43,7 +39,7 @@ Usage: $0 [OPTION] Output the configuration name of the system \`$me' is run on. -Operation modes: +Options: -h, --help print this help, then exit -t, --time-stamp print date of last modification, then exit -v, --version print version number, then exit @@ -54,9 +50,7 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, -2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 -Free Software Foundation, Inc. +Copyright 1992-2018 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -90,8 +84,6 @@ if test $# != 0; then exit 1 fi -trap 'exit 1' 1 2 15 - # CC_FOR_BUILD -- compiler used by this script. Note that the use of a # compiler to aid in system detection is discouraged as it requires # temporary files to be created and, as you can see below, it is a @@ -102,34 +94,39 @@ trap 'exit 1' 1 2 15 # Portable tmp directory creation inspired by the Autoconf team. -set_cc_for_build=' -trap "exitcode=\$?; (rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null) && exit \$exitcode" 0 ; -trap "rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null; exit 1" 1 2 13 15 ; -: ${TMPDIR=/tmp} ; - { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || - { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir $tmp) ; } || - { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir $tmp) && echo "Warning: creating insecure temp directory" >&2 ; } || - { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } ; -dummy=$tmp/dummy ; -tmpfiles="$dummy.c $dummy.o $dummy.rel $dummy" ; -case $CC_FOR_BUILD,$HOST_CC,$CC in - ,,) echo "int x;" > $dummy.c ; - for c in cc gcc c89 c99 ; do - if ($c -c -o $dummy.o $dummy.c) >/dev/null 2>&1 ; then - CC_FOR_BUILD="$c"; break ; - fi ; - done ; - if test x"$CC_FOR_BUILD" = x ; then - CC_FOR_BUILD=no_compiler_found ; - fi - ;; - ,,*) CC_FOR_BUILD=$CC ;; - ,*,*) CC_FOR_BUILD=$HOST_CC ;; -esac ; set_cc_for_build= ;' +tmp= +# shellcheck disable=SC2172 +trap 'test -z "$tmp" || rm -fr "$tmp"' 1 2 13 15 +trap 'exitcode=$?; test -z "$tmp" || rm -fr "$tmp"; exit $exitcode' 0 + +set_cc_for_build() { + : "${TMPDIR=/tmp}" + # shellcheck disable=SC2039 + { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || + { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir "$tmp" 2>/dev/null) ; } || + { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir "$tmp" 2>/dev/null) && echo "Warning: creating insecure temp directory" >&2 ; } || + { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } + dummy=$tmp/dummy + case ${CC_FOR_BUILD-},${HOST_CC-},${CC-} in + ,,) echo "int x;" > "$dummy.c" + for driver in cc gcc c89 c99 ; do + if ($driver -c -o "$dummy.o" "$dummy.c") >/dev/null 2>&1 ; then + CC_FOR_BUILD="$driver" + break + fi + done + if test x"$CC_FOR_BUILD" = x ; then + CC_FOR_BUILD=no_compiler_found + fi + ;; + ,,*) CC_FOR_BUILD=$CC ;; + ,*,*) CC_FOR_BUILD=$HOST_CC ;; + esac +} # This is needed to find uname on a Pyramid OSx when run in the BSD universe. # (ghazi@noc.rutgers.edu 1994-08-24) -if (test -f /.attbin/uname) >/dev/null 2>&1 ; then +if test -f /.attbin/uname ; then PATH=$PATH:/.attbin ; export PATH fi @@ -138,9 +135,37 @@ UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown +case "$UNAME_SYSTEM" in +Linux|GNU|GNU/*) + # If the system lacks a compiler, then just pick glibc. + # We could probably try harder. + LIBC=gnu + + set_cc_for_build + cat <<-EOF > "$dummy.c" + #include + #if defined(__UCLIBC__) + LIBC=uclibc + #elif defined(__dietlibc__) + LIBC=dietlibc + #else + LIBC=gnu + #endif + EOF + eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g'`" + + # If ldd exists, use it to detect musl libc. + if command -v ldd >/dev/null && \ + ldd --version 2>&1 | grep -q ^musl + then + LIBC=musl + fi + ;; +esac + # Note: order is significant - the case branches are not exclusive. -case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in +case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in *:NetBSD:*:*) # NetBSD (nbsd) targets should (where applicable) match one or # more of the tuples: *-*-netbsdelf*, *-*-netbsdaout*, @@ -153,21 +178,31 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # Note: NetBSD doesn't particularly care about the vendor # portion of the name. We always set it to "unknown". sysctl="sysctl -n hw.machine_arch" - UNAME_MACHINE_ARCH=`(/sbin/$sysctl 2>/dev/null || \ - /usr/sbin/$sysctl 2>/dev/null || echo unknown)` - case "${UNAME_MACHINE_ARCH}" in + UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \ + "/sbin/$sysctl" 2>/dev/null || \ + "/usr/sbin/$sysctl" 2>/dev/null || \ + echo unknown)` + case "$UNAME_MACHINE_ARCH" in armeb) machine=armeb-unknown ;; arm*) machine=arm-unknown ;; sh3el) machine=shl-unknown ;; sh3eb) machine=sh-unknown ;; sh5el) machine=sh5le-unknown ;; - *) machine=${UNAME_MACHINE_ARCH}-unknown ;; + earmv*) + arch=`echo "$UNAME_MACHINE_ARCH" | sed -e 's,^e\(armv[0-9]\).*$,\1,'` + endian=`echo "$UNAME_MACHINE_ARCH" | sed -ne 's,^.*\(eb\)$,\1,p'` + machine="${arch}${endian}"-unknown + ;; + *) machine="$UNAME_MACHINE_ARCH"-unknown ;; esac # The Operating System including object format, if it has switched - # to ELF recently, or will in the future. - case "${UNAME_MACHINE_ARCH}" in + # to ELF recently (or will in the future) and ABI. + case "$UNAME_MACHINE_ARCH" in + earm*) + os=netbsdelf + ;; arm*|i386|m68k|ns32k|sh3*|sparc|vax) - eval $set_cc_for_build + set_cc_for_build if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ELF__ then @@ -182,44 +217,67 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in os=netbsd ;; esac + # Determine ABI tags. + case "$UNAME_MACHINE_ARCH" in + earm*) + expr='s/^earmv[0-9]/-eabi/;s/eb$//' + abi=`echo "$UNAME_MACHINE_ARCH" | sed -e "$expr"` + ;; + esac # The OS release # Debian GNU/NetBSD machines have a different userland, and # thus, need a distinct triplet. However, they do not need # kernel version information, so it can be replaced with a # suitable tag, in the style of linux-gnu. - case "${UNAME_VERSION}" in + case "$UNAME_VERSION" in Debian*) release='-gnu' ;; *) - release=`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'` + release=`echo "$UNAME_RELEASE" | sed -e 's/[-_].*//' | cut -d. -f1,2` ;; esac # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: # contains redundant information, the shorter form: # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. - echo "${machine}-${os}${release}" + echo "$machine-${os}${release}${abi-}" exit ;; *:Bitrig:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'` - echo ${UNAME_MACHINE_ARCH}-unknown-bitrig${UNAME_RELEASE} + echo "$UNAME_MACHINE_ARCH"-unknown-bitrig"$UNAME_RELEASE" exit ;; *:OpenBSD:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'` - echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE} + echo "$UNAME_MACHINE_ARCH"-unknown-openbsd"$UNAME_RELEASE" + exit ;; + *:LibertyBSD:*:*) + UNAME_MACHINE_ARCH=`arch | sed 's/^.*BSD\.//'` + echo "$UNAME_MACHINE_ARCH"-unknown-libertybsd"$UNAME_RELEASE" + exit ;; + *:MidnightBSD:*:*) + echo "$UNAME_MACHINE"-unknown-midnightbsd"$UNAME_RELEASE" exit ;; *:ekkoBSD:*:*) - echo ${UNAME_MACHINE}-unknown-ekkobsd${UNAME_RELEASE} + echo "$UNAME_MACHINE"-unknown-ekkobsd"$UNAME_RELEASE" exit ;; *:SolidBSD:*:*) - echo ${UNAME_MACHINE}-unknown-solidbsd${UNAME_RELEASE} + echo "$UNAME_MACHINE"-unknown-solidbsd"$UNAME_RELEASE" exit ;; macppc:MirBSD:*:*) - echo powerpc-unknown-mirbsd${UNAME_RELEASE} + echo powerpc-unknown-mirbsd"$UNAME_RELEASE" exit ;; *:MirBSD:*:*) - echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE} + echo "$UNAME_MACHINE"-unknown-mirbsd"$UNAME_RELEASE" + exit ;; + *:Sortix:*:*) + echo "$UNAME_MACHINE"-unknown-sortix + exit ;; + *:Redox:*:*) + echo "$UNAME_MACHINE"-unknown-redox exit ;; + mips:OSF1:*.*) + echo mips-dec-osf1 + exit ;; alpha:OSF1:*:*) case $UNAME_RELEASE in *4.0) @@ -236,63 +294,54 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1` case "$ALPHA_CPU_TYPE" in "EV4 (21064)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "EV4.5 (21064)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "LCA4 (21066/21068)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "EV5 (21164)") - UNAME_MACHINE="alphaev5" ;; + UNAME_MACHINE=alphaev5 ;; "EV5.6 (21164A)") - UNAME_MACHINE="alphaev56" ;; + UNAME_MACHINE=alphaev56 ;; "EV5.6 (21164PC)") - UNAME_MACHINE="alphapca56" ;; + UNAME_MACHINE=alphapca56 ;; "EV5.7 (21164PC)") - UNAME_MACHINE="alphapca57" ;; + UNAME_MACHINE=alphapca57 ;; "EV6 (21264)") - UNAME_MACHINE="alphaev6" ;; + UNAME_MACHINE=alphaev6 ;; "EV6.7 (21264A)") - UNAME_MACHINE="alphaev67" ;; + UNAME_MACHINE=alphaev67 ;; "EV6.8CB (21264C)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.8AL (21264B)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.8CX (21264D)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.9A (21264/EV69A)") - UNAME_MACHINE="alphaev69" ;; + UNAME_MACHINE=alphaev69 ;; "EV7 (21364)") - UNAME_MACHINE="alphaev7" ;; + UNAME_MACHINE=alphaev7 ;; "EV7.9 (21364A)") - UNAME_MACHINE="alphaev79" ;; + UNAME_MACHINE=alphaev79 ;; esac # A Pn.n version is a patched version. # A Vn.n version is a released version. # A Tn.n version is a released field test version. # A Xn.n version is an unreleased experimental baselevel. # 1.2 uses "1.2" for uname -r. - echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` + echo "$UNAME_MACHINE"-dec-osf"`echo "$UNAME_RELEASE" | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`" # Reset EXIT trap before exiting to avoid spurious non-zero exit code. exitcode=$? trap '' 0 exit $exitcode ;; - Alpha\ *:Windows_NT*:*) - # How do we know it's Interix rather than the generic POSIX subsystem? - # Should we change UNAME_MACHINE based on the output of uname instead - # of the specific Alpha model? - echo alpha-pc-interix - exit ;; - 21064:Windows_NT:50:3) - echo alpha-dec-winnt3.5 - exit ;; Amiga*:UNIX_System_V:4.0:*) echo m68k-unknown-sysv4 exit ;; *:[Aa]miga[Oo][Ss]:*:*) - echo ${UNAME_MACHINE}-unknown-amigaos + echo "$UNAME_MACHINE"-unknown-amigaos exit ;; *:[Mm]orph[Oo][Ss]:*:*) - echo ${UNAME_MACHINE}-unknown-morphos + echo "$UNAME_MACHINE"-unknown-morphos exit ;; *:OS/390:*:*) echo i370-ibm-openedition @@ -304,9 +353,9 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in echo powerpc-ibm-os400 exit ;; arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*) - echo arm-acorn-riscix${UNAME_RELEASE} + echo arm-acorn-riscix"$UNAME_RELEASE" exit ;; - arm:riscos:*:*|arm:RISCOS:*:*) + arm*:riscos:*:*|arm*:RISCOS:*:*) echo arm-unknown-riscos exit ;; SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*) @@ -331,38 +380,33 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in sparc) echo sparc-icl-nx7; exit ;; esac ;; s390x:SunOS:*:*) - echo ${UNAME_MACHINE}-ibm-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo "$UNAME_MACHINE"-ibm-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`" exit ;; sun4H:SunOS:5.*:*) - echo sparc-hal-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo sparc-hal-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" exit ;; sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*) - echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo sparc-sun-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`" exit ;; i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*) - echo i386-pc-auroraux${UNAME_RELEASE} + echo i386-pc-auroraux"$UNAME_RELEASE" exit ;; i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) - eval $set_cc_for_build - SUN_ARCH="i386" - # If there is a compiler, see if it is configured for 64-bit objects. - # Note that the Sun cc does not turn __LP64__ into 1 like gcc does. - # This test works for both compilers. - if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then - if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_64BIT_ARCH >/dev/null - then - SUN_ARCH="x86_64" - fi - fi - echo ${SUN_ARCH}-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + UNAME_REL="`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`" + case `isainfo -b` in + 32) + echo i386-pc-solaris2"$UNAME_REL" + ;; + 64) + echo x86_64-pc-solaris2"$UNAME_REL" + ;; + esac exit ;; sun4*:SunOS:6*:*) # According to config.sub, this is the proper way to canonicalize # SunOS6. Hard to guess exactly what SunOS6 will be like, but # it's likely to be more like Solaris than SunOS4. - echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo sparc-sun-solaris3"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" exit ;; sun4*:SunOS:*:*) case "`/usr/bin/arch -k`" in @@ -371,25 +415,25 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in ;; esac # Japanese Language versions have a version number like `4.1.3-JL'. - echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'` + echo sparc-sun-sunos"`echo "$UNAME_RELEASE"|sed -e 's/-/_/'`" exit ;; sun3*:SunOS:*:*) - echo m68k-sun-sunos${UNAME_RELEASE} + echo m68k-sun-sunos"$UNAME_RELEASE" exit ;; sun*:*:4.2BSD:*) UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` - test "x${UNAME_RELEASE}" = "x" && UNAME_RELEASE=3 + test "x$UNAME_RELEASE" = x && UNAME_RELEASE=3 case "`/bin/arch`" in sun3) - echo m68k-sun-sunos${UNAME_RELEASE} + echo m68k-sun-sunos"$UNAME_RELEASE" ;; sun4) - echo sparc-sun-sunos${UNAME_RELEASE} + echo sparc-sun-sunos"$UNAME_RELEASE" ;; esac exit ;; aushp:SunOS:*:*) - echo sparc-auspex-sunos${UNAME_RELEASE} + echo sparc-auspex-sunos"$UNAME_RELEASE" exit ;; # The situation for MiNT is a little confusing. The machine name # can be virtually everything (everything which is not @@ -400,44 +444,44 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # MiNT. But MiNT is downward compatible to TOS, so this should # be no problem. atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} + echo m68k-atari-mint"$UNAME_RELEASE" exit ;; atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} + echo m68k-atari-mint"$UNAME_RELEASE" exit ;; *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} + echo m68k-atari-mint"$UNAME_RELEASE" exit ;; milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*) - echo m68k-milan-mint${UNAME_RELEASE} + echo m68k-milan-mint"$UNAME_RELEASE" exit ;; hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*) - echo m68k-hades-mint${UNAME_RELEASE} + echo m68k-hades-mint"$UNAME_RELEASE" exit ;; *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*) - echo m68k-unknown-mint${UNAME_RELEASE} + echo m68k-unknown-mint"$UNAME_RELEASE" exit ;; m68k:machten:*:*) - echo m68k-apple-machten${UNAME_RELEASE} + echo m68k-apple-machten"$UNAME_RELEASE" exit ;; powerpc:machten:*:*) - echo powerpc-apple-machten${UNAME_RELEASE} + echo powerpc-apple-machten"$UNAME_RELEASE" exit ;; RISC*:Mach:*:*) echo mips-dec-mach_bsd4.3 exit ;; RISC*:ULTRIX:*:*) - echo mips-dec-ultrix${UNAME_RELEASE} + echo mips-dec-ultrix"$UNAME_RELEASE" exit ;; VAX*:ULTRIX*:*:*) - echo vax-dec-ultrix${UNAME_RELEASE} + echo vax-dec-ultrix"$UNAME_RELEASE" exit ;; 2020:CLIX:*:* | 2430:CLIX:*:*) - echo clipper-intergraph-clix${UNAME_RELEASE} + echo clipper-intergraph-clix"$UNAME_RELEASE" exit ;; mips:*:*:UMIPS | mips:*:*:RISCos) - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + set_cc_for_build + sed 's/^ //' << EOF > "$dummy.c" #ifdef __cplusplus #include /* for printf() prototype */ int main (int argc, char *argv[]) { @@ -446,23 +490,23 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in #endif #if defined (host_mips) && defined (MIPSEB) #if defined (SYSTYPE_SYSV) - printf ("mips-mips-riscos%ssysv\n", argv[1]); exit (0); + printf ("mips-mips-riscos%ssysv\\n", argv[1]); exit (0); #endif #if defined (SYSTYPE_SVR4) - printf ("mips-mips-riscos%ssvr4\n", argv[1]); exit (0); + printf ("mips-mips-riscos%ssvr4\\n", argv[1]); exit (0); #endif #if defined (SYSTYPE_BSD43) || defined(SYSTYPE_BSD) - printf ("mips-mips-riscos%sbsd\n", argv[1]); exit (0); + printf ("mips-mips-riscos%sbsd\\n", argv[1]); exit (0); #endif #endif exit (-1); } EOF - $CC_FOR_BUILD -o $dummy $dummy.c && - dummyarg=`echo "${UNAME_RELEASE}" | sed -n 's/\([0-9]*\).*/\1/p'` && - SYSTEM_NAME=`$dummy $dummyarg` && + $CC_FOR_BUILD -o "$dummy" "$dummy.c" && + dummyarg=`echo "$UNAME_RELEASE" | sed -n 's/\([0-9]*\).*/\1/p'` && + SYSTEM_NAME=`"$dummy" "$dummyarg"` && { echo "$SYSTEM_NAME"; exit; } - echo mips-mips-riscos${UNAME_RELEASE} + echo mips-mips-riscos"$UNAME_RELEASE" exit ;; Motorola:PowerMAX_OS:*:*) echo powerpc-motorola-powermax @@ -488,17 +532,17 @@ EOF AViiON:dgux:*:*) # DG/UX returns AViiON for all architectures UNAME_PROCESSOR=`/usr/bin/uname -p` - if [ $UNAME_PROCESSOR = mc88100 ] || [ $UNAME_PROCESSOR = mc88110 ] + if [ "$UNAME_PROCESSOR" = mc88100 ] || [ "$UNAME_PROCESSOR" = mc88110 ] then - if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx ] || \ - [ ${TARGET_BINARY_INTERFACE}x = x ] + if [ "$TARGET_BINARY_INTERFACE"x = m88kdguxelfx ] || \ + [ "$TARGET_BINARY_INTERFACE"x = x ] then - echo m88k-dg-dgux${UNAME_RELEASE} + echo m88k-dg-dgux"$UNAME_RELEASE" else - echo m88k-dg-dguxbcs${UNAME_RELEASE} + echo m88k-dg-dguxbcs"$UNAME_RELEASE" fi else - echo i586-dg-dgux${UNAME_RELEASE} + echo i586-dg-dgux"$UNAME_RELEASE" fi exit ;; M88*:DolphinOS:*:*) # DolphinOS (SVR3) @@ -515,7 +559,7 @@ EOF echo m68k-tektronix-bsd exit ;; *:IRIX*:*:*) - echo mips-sgi-irix`echo ${UNAME_RELEASE}|sed -e 's/-/_/g'` + echo mips-sgi-irix"`echo "$UNAME_RELEASE"|sed -e 's/-/_/g'`" exit ;; ????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX. echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id @@ -527,14 +571,14 @@ EOF if [ -x /usr/bin/oslevel ] ; then IBM_REV=`/usr/bin/oslevel` else - IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} + IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" fi - echo ${UNAME_MACHINE}-ibm-aix${IBM_REV} + echo "$UNAME_MACHINE"-ibm-aix"$IBM_REV" exit ;; *:AIX:2:3) if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + set_cc_for_build + sed 's/^ //' << EOF > "$dummy.c" #include main() @@ -545,7 +589,7 @@ EOF exit(0); } EOF - if $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` + if $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"` then echo "$SYSTEM_NAME" else @@ -559,26 +603,27 @@ EOF exit ;; *:AIX:*:[4567]) IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'` - if /usr/sbin/lsattr -El ${IBM_CPU_ID} | grep ' POWER' >/dev/null 2>&1; then + if /usr/sbin/lsattr -El "$IBM_CPU_ID" | grep ' POWER' >/dev/null 2>&1; then IBM_ARCH=rs6000 else IBM_ARCH=powerpc fi - if [ -x /usr/bin/oslevel ] ; then - IBM_REV=`/usr/bin/oslevel` + if [ -x /usr/bin/lslpp ] ; then + IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc | + awk -F: '{ print $3 }' | sed s/[0-9]*$/0/` else - IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} + IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" fi - echo ${IBM_ARCH}-ibm-aix${IBM_REV} + echo "$IBM_ARCH"-ibm-aix"$IBM_REV" exit ;; *:AIX:*:*) echo rs6000-ibm-aix exit ;; - ibmrt:4.4BSD:*|romp-ibm:BSD:*) + ibmrt:4.4BSD:*|romp-ibm:4.4BSD:*) echo romp-ibm-bsd4.4 exit ;; ibmrt:*BSD:*|romp-ibm:BSD:*) # covers RT/PC BSD and - echo romp-ibm-bsd${UNAME_RELEASE} # 4.3 with uname added to + echo romp-ibm-bsd"$UNAME_RELEASE" # 4.3 with uname added to exit ;; # report: romp-ibm BSD 4.3 *:BOSX:*:*) echo rs6000-bull-bosx @@ -593,28 +638,28 @@ EOF echo m68k-hp-bsd4.4 exit ;; 9000/[34678]??:HP-UX:*:*) - HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` - case "${UNAME_MACHINE}" in - 9000/31? ) HP_ARCH=m68000 ;; - 9000/[34]?? ) HP_ARCH=m68k ;; + HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'` + case "$UNAME_MACHINE" in + 9000/31?) HP_ARCH=m68000 ;; + 9000/[34]??) HP_ARCH=m68k ;; 9000/[678][0-9][0-9]) if [ -x /usr/bin/getconf ]; then sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` - case "${sc_cpu_version}" in - 523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0 - 528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1 + case "$sc_cpu_version" in + 523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0 + 528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1 532) # CPU_PA_RISC2_0 - case "${sc_kernel_bits}" in - 32) HP_ARCH="hppa2.0n" ;; - 64) HP_ARCH="hppa2.0w" ;; - '') HP_ARCH="hppa2.0" ;; # HP-UX 10.20 + case "$sc_kernel_bits" in + 32) HP_ARCH=hppa2.0n ;; + 64) HP_ARCH=hppa2.0w ;; + '') HP_ARCH=hppa2.0 ;; # HP-UX 10.20 esac ;; esac fi - if [ "${HP_ARCH}" = "" ]; then - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + if [ "$HP_ARCH" = "" ]; then + set_cc_for_build + sed 's/^ //' << EOF > "$dummy.c" #define _HPUX_SOURCE #include @@ -647,13 +692,13 @@ EOF exit (0); } EOF - (CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy` + (CCOPTS="" $CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null) && HP_ARCH=`"$dummy"` test -z "$HP_ARCH" && HP_ARCH=hppa fi ;; esac - if [ ${HP_ARCH} = "hppa2.0w" ] + if [ "$HP_ARCH" = hppa2.0w ] then - eval $set_cc_for_build + set_cc_for_build # hppa2.0w-hp-hpux* has a 64-bit kernel and a compiler generating # 32-bit code. hppa64-hp-hpux* has the same kernel and a compiler @@ -664,23 +709,23 @@ EOF # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess # => hppa64-hp-hpux11.23 - if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | + if echo __LP64__ | (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | grep -q __LP64__ then - HP_ARCH="hppa2.0w" + HP_ARCH=hppa2.0w else - HP_ARCH="hppa64" + HP_ARCH=hppa64 fi fi - echo ${HP_ARCH}-hp-hpux${HPUX_REV} + echo "$HP_ARCH"-hp-hpux"$HPUX_REV" exit ;; ia64:HP-UX:*:*) - HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` - echo ia64-hp-hpux${HPUX_REV} + HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'` + echo ia64-hp-hpux"$HPUX_REV" exit ;; 3050*:HI-UX:*:*) - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + set_cc_for_build + sed 's/^ //' << EOF > "$dummy.c" #include int main () @@ -705,11 +750,11 @@ EOF exit (0); } EOF - $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` && + $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"` && { echo "$SYSTEM_NAME"; exit; } echo unknown-hitachi-hiuxwe2 exit ;; - 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:* ) + 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:*) echo hppa1.1-hp-bsd exit ;; 9000/8??:4.3bsd:*:*) @@ -718,7 +763,7 @@ EOF *9??*:MPE/iX:*:* | *3000*:MPE/iX:*:*) echo hppa1.0-hp-mpeix exit ;; - hp7??:OSF1:*:* | hp8?[79]:OSF1:*:* ) + hp7??:OSF1:*:* | hp8?[79]:OSF1:*:*) echo hppa1.1-hp-osf exit ;; hp8??:OSF1:*:*) @@ -726,9 +771,9 @@ EOF exit ;; i*86:OSF1:*:*) if [ -x /usr/sbin/sysversion ] ; then - echo ${UNAME_MACHINE}-unknown-osf1mk + echo "$UNAME_MACHINE"-unknown-osf1mk else - echo ${UNAME_MACHINE}-unknown-osf1 + echo "$UNAME_MACHINE"-unknown-osf1 fi exit ;; parisc*:Lites*:*:*) @@ -753,127 +798,120 @@ EOF echo c4-convex-bsd exit ;; CRAY*Y-MP:*:*:*) - echo ymp-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo ymp-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; CRAY*[A-Z]90:*:*:*) - echo ${UNAME_MACHINE}-cray-unicos${UNAME_RELEASE} \ + echo "$UNAME_MACHINE"-cray-unicos"$UNAME_RELEASE" \ | sed -e 's/CRAY.*\([A-Z]90\)/\1/' \ -e y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/ \ -e 's/\.[^.]*$/.X/' exit ;; CRAY*TS:*:*:*) - echo t90-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo t90-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; CRAY*T3E:*:*:*) - echo alphaev5-cray-unicosmk${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo alphaev5-cray-unicosmk"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; CRAY*SV1:*:*:*) - echo sv1-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo sv1-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; *:UNICOS/mp:*:*) - echo craynv-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo craynv-cray-unicosmp"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) - FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'` + FUJITSU_PROC=`uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` + FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` + FUJITSU_REL=`echo "$UNAME_RELEASE" | sed -e 's/ /_/'` echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; 5000:UNIX_System_V:4.*:*) - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'` + FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` + FUJITSU_REL=`echo "$UNAME_RELEASE" | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'` echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) - echo ${UNAME_MACHINE}-pc-bsdi${UNAME_RELEASE} + echo "$UNAME_MACHINE"-pc-bsdi"$UNAME_RELEASE" exit ;; sparc*:BSD/OS:*:*) - echo sparc-unknown-bsdi${UNAME_RELEASE} + echo sparc-unknown-bsdi"$UNAME_RELEASE" exit ;; *:BSD/OS:*:*) - echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE} + echo "$UNAME_MACHINE"-unknown-bsdi"$UNAME_RELEASE" + exit ;; + arm*:FreeBSD:*:*) + UNAME_PROCESSOR=`uname -p` + set_cc_for_build + if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep -q __ARM_PCS_VFP + then + echo "${UNAME_PROCESSOR}"-unknown-freebsd"`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`"-gnueabi + else + echo "${UNAME_PROCESSOR}"-unknown-freebsd"`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`"-gnueabihf + fi exit ;; *:FreeBSD:*:*) UNAME_PROCESSOR=`/usr/bin/uname -p` - case ${UNAME_PROCESSOR} in + case "$UNAME_PROCESSOR" in amd64) - echo x86_64-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; - *) - echo ${UNAME_PROCESSOR}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; + UNAME_PROCESSOR=x86_64 ;; + i386) + UNAME_PROCESSOR=i586 ;; esac + echo "$UNAME_PROCESSOR"-unknown-freebsd"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`" exit ;; i*:CYGWIN*:*) - echo ${UNAME_MACHINE}-pc-cygwin + echo "$UNAME_MACHINE"-pc-cygwin exit ;; *:MINGW64*:*) - echo ${UNAME_MACHINE}-pc-mingw64 + echo "$UNAME_MACHINE"-pc-mingw64 exit ;; *:MINGW*:*) - echo ${UNAME_MACHINE}-pc-mingw32 + echo "$UNAME_MACHINE"-pc-mingw32 exit ;; - i*:MSYS*:*) - echo ${UNAME_MACHINE}-pc-msys - exit ;; - i*:windows32*:*) - # uname -m includes "-pc" on this system. - echo ${UNAME_MACHINE}-mingw32 + *:MSYS*:*) + echo "$UNAME_MACHINE"-pc-msys exit ;; i*:PW*:*) - echo ${UNAME_MACHINE}-pc-pw32 + echo "$UNAME_MACHINE"-pc-pw32 exit ;; *:Interix*:*) - case ${UNAME_MACHINE} in + case "$UNAME_MACHINE" in x86) - echo i586-pc-interix${UNAME_RELEASE} + echo i586-pc-interix"$UNAME_RELEASE" exit ;; authenticamd | genuineintel | EM64T) - echo x86_64-unknown-interix${UNAME_RELEASE} + echo x86_64-unknown-interix"$UNAME_RELEASE" exit ;; IA64) - echo ia64-unknown-interix${UNAME_RELEASE} + echo ia64-unknown-interix"$UNAME_RELEASE" exit ;; esac ;; - [345]86:Windows_95:* | [345]86:Windows_98:* | [345]86:Windows_NT:*) - echo i${UNAME_MACHINE}-pc-mks - exit ;; - 8664:Windows_NT:*) - echo x86_64-pc-mks - exit ;; - i*:Windows_NT*:* | Pentium*:Windows_NT*:*) - # How do we know it's Interix rather than the generic POSIX subsystem? - # It also conflicts with pre-2.0 versions of AT&T UWIN. Should we - # UNAME_MACHINE based on the output of uname instead of i386? - echo i586-pc-interix - exit ;; i*:UWIN*:*) - echo ${UNAME_MACHINE}-pc-uwin + echo "$UNAME_MACHINE"-pc-uwin exit ;; amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*) echo x86_64-unknown-cygwin exit ;; - p*:CYGWIN*:*) - echo powerpcle-unknown-cygwin - exit ;; prep*:SunOS:5.*:*) - echo powerpcle-unknown-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo powerpcle-unknown-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" exit ;; *:GNU:*:*) # the GNU system - echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-gnu`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'` + echo "`echo "$UNAME_MACHINE"|sed -e 's,[-/].*$,,'`-unknown-$LIBC`echo "$UNAME_RELEASE"|sed -e 's,/.*$,,'`" exit ;; *:GNU/*:*:*) # other systems with GNU libc and userland - echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-gnu + echo "$UNAME_MACHINE-unknown-`echo "$UNAME_SYSTEM" | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`-$LIBC" exit ;; - i*86:Minix:*:*) - echo ${UNAME_MACHINE}-pc-minix + *:Minix:*:*) + echo "$UNAME_MACHINE"-unknown-minix exit ;; aarch64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; aarch64_be:Linux:*:*) UNAME_MACHINE=aarch64_be - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; alpha:Linux:*:*) case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in @@ -886,63 +924,64 @@ EOF EV68*) UNAME_MACHINE=alphaev68 ;; esac objdump --private-headers /bin/sh | grep -q ld.so.1 - if test "$?" = 0 ; then LIBC="libc1" ; else LIBC="" ; fi - echo ${UNAME_MACHINE}-unknown-linux-gnu${LIBC} + if test "$?" = 0 ; then LIBC=gnulibc1 ; fi + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + arc:Linux:*:* | arceb:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; arm*:Linux:*:*) - eval $set_cc_for_build + set_cc_for_build if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ARM_EABI__ then - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" else if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ARM_PCS_VFP then - echo ${UNAME_MACHINE}-unknown-linux-gnueabi + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"eabi else - echo ${UNAME_MACHINE}-unknown-linux-gnueabihf + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"eabihf fi fi exit ;; avr32*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; cris:Linux:*:*) - echo ${UNAME_MACHINE}-axis-linux-gnu + echo "$UNAME_MACHINE"-axis-linux-"$LIBC" exit ;; crisv32:Linux:*:*) - echo ${UNAME_MACHINE}-axis-linux-gnu + echo "$UNAME_MACHINE"-axis-linux-"$LIBC" + exit ;; + e2k:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; frv:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; hexagon:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; i*86:Linux:*:*) - LIBC=gnu - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c - #ifdef __dietlibc__ - LIBC=dietlibc - #endif -EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC'` - echo "${UNAME_MACHINE}-pc-linux-${LIBC}" + echo "$UNAME_MACHINE"-pc-linux-"$LIBC" exit ;; ia64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + k1om:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; m32r*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; m68*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; mips:Linux:*:* | mips64:Linux:*:*) - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + set_cc_for_build + sed 's/^ //' << EOF > "$dummy.c" #undef CPU #undef ${UNAME_MACHINE} #undef ${UNAME_MACHINE}el @@ -956,55 +995,70 @@ EOF #endif #endif EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'` - test x"${CPU}" != x && { echo "${CPU}-unknown-linux-gnu"; exit; } + eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^CPU'`" + test "x$CPU" != x && { echo "$CPU-unknown-linux-$LIBC"; exit; } ;; - or32:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + mips64el:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + openrisc*:Linux:*:*) + echo or1k-unknown-linux-"$LIBC" + exit ;; + or32:Linux:*:* | or1k*:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; padre:Linux:*:*) - echo sparc-unknown-linux-gnu + echo sparc-unknown-linux-"$LIBC" exit ;; parisc64:Linux:*:* | hppa64:Linux:*:*) - echo hppa64-unknown-linux-gnu + echo hppa64-unknown-linux-"$LIBC" exit ;; parisc:Linux:*:* | hppa:Linux:*:*) # Look for CPU level case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in - PA7*) echo hppa1.1-unknown-linux-gnu ;; - PA8*) echo hppa2.0-unknown-linux-gnu ;; - *) echo hppa-unknown-linux-gnu ;; + PA7*) echo hppa1.1-unknown-linux-"$LIBC" ;; + PA8*) echo hppa2.0-unknown-linux-"$LIBC" ;; + *) echo hppa-unknown-linux-"$LIBC" ;; esac exit ;; ppc64:Linux:*:*) - echo powerpc64-unknown-linux-gnu + echo powerpc64-unknown-linux-"$LIBC" exit ;; ppc:Linux:*:*) - echo powerpc-unknown-linux-gnu + echo powerpc-unknown-linux-"$LIBC" + exit ;; + ppc64le:Linux:*:*) + echo powerpc64le-unknown-linux-"$LIBC" + exit ;; + ppcle:Linux:*:*) + echo powerpcle-unknown-linux-"$LIBC" + exit ;; + riscv32:Linux:*:* | riscv64:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; s390:Linux:*:* | s390x:Linux:*:*) - echo ${UNAME_MACHINE}-ibm-linux + echo "$UNAME_MACHINE"-ibm-linux-"$LIBC" exit ;; sh64*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; sh*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; sparc:Linux:*:* | sparc64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; tile*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; vax:Linux:*:*) - echo ${UNAME_MACHINE}-dec-linux-gnu + echo "$UNAME_MACHINE"-dec-linux-"$LIBC" exit ;; x86_64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-pc-linux-"$LIBC" exit ;; xtensa*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; i*86:DYNIX/ptx:4*:*) # ptx 4.0 does uname -s correctly, with DYNIX/ptx in there. @@ -1018,34 +1072,34 @@ EOF # I am not positive that other SVR4 systems won't match this, # I just have to hope. -- rms. # Use sysv4.2uw... so that sysv4* matches it. - echo ${UNAME_MACHINE}-pc-sysv4.2uw${UNAME_VERSION} + echo "$UNAME_MACHINE"-pc-sysv4.2uw"$UNAME_VERSION" exit ;; i*86:OS/2:*:*) # If we were able to find `uname', then EMX Unix compatibility # is probably installed. - echo ${UNAME_MACHINE}-pc-os2-emx + echo "$UNAME_MACHINE"-pc-os2-emx exit ;; i*86:XTS-300:*:STOP) - echo ${UNAME_MACHINE}-unknown-stop + echo "$UNAME_MACHINE"-unknown-stop exit ;; i*86:atheos:*:*) - echo ${UNAME_MACHINE}-unknown-atheos + echo "$UNAME_MACHINE"-unknown-atheos exit ;; i*86:syllable:*:*) - echo ${UNAME_MACHINE}-pc-syllable + echo "$UNAME_MACHINE"-pc-syllable exit ;; i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.[02]*:*) - echo i386-unknown-lynxos${UNAME_RELEASE} + echo i386-unknown-lynxos"$UNAME_RELEASE" exit ;; i*86:*DOS:*:*) - echo ${UNAME_MACHINE}-pc-msdosdjgpp + echo "$UNAME_MACHINE"-pc-msdosdjgpp exit ;; - i*86:*:4.*:* | i*86:SYSTEM_V:4.*:*) - UNAME_REL=`echo ${UNAME_RELEASE} | sed 's/\/MP$//'` + i*86:*:4.*:*) + UNAME_REL=`echo "$UNAME_RELEASE" | sed 's/\/MP$//'` if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then - echo ${UNAME_MACHINE}-univel-sysv${UNAME_REL} + echo "$UNAME_MACHINE"-univel-sysv"$UNAME_REL" else - echo ${UNAME_MACHINE}-pc-sysv${UNAME_REL} + echo "$UNAME_MACHINE"-pc-sysv"$UNAME_REL" fi exit ;; i*86:*:5:[678]*) @@ -1055,12 +1109,12 @@ EOF *Pentium) UNAME_MACHINE=i586 ;; *Pent*|*Celeron) UNAME_MACHINE=i686 ;; esac - echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION} + echo "$UNAME_MACHINE-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}{$UNAME_VERSION}" exit ;; i*86:*:3.2:*) if test -f /usr/options/cb.name; then UNAME_REL=`sed -n 's/.*Version //p' /dev/null >/dev/null ; then UNAME_REL=`(/bin/uname -X|grep Release|sed -e 's/.*= //')` (/bin/uname -X|grep i80486 >/dev/null) && UNAME_MACHINE=i486 @@ -1070,9 +1124,9 @@ EOF && UNAME_MACHINE=i686 (/bin/uname -X|grep '^Machine.*Pentium Pro' >/dev/null) \ && UNAME_MACHINE=i686 - echo ${UNAME_MACHINE}-pc-sco$UNAME_REL + echo "$UNAME_MACHINE"-pc-sco"$UNAME_REL" else - echo ${UNAME_MACHINE}-pc-sysv32 + echo "$UNAME_MACHINE"-pc-sysv32 fi exit ;; pc:*:*:*) @@ -1080,7 +1134,7 @@ EOF # uname -m prints for DJGPP always 'pc', but it prints nothing about # the processor, so we play safe by assuming i586. # Note: whatever this is, it MUST be the same as what config.sub - # prints for the "djgpp" host, or else GDB configury will decide that + # prints for the "djgpp" host, or else GDB configure will decide that # this is a cross-build. echo i586-pc-msdosdjgpp exit ;; @@ -1092,9 +1146,9 @@ EOF exit ;; i860:*:4.*:*) # i860-SVR4 if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then - echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4 + echo i860-stardent-sysv"$UNAME_RELEASE" # Stardent Vistra i860-SVR4 else # Add other i860-SVR4 vendors below as they are discovered. - echo i860-unknown-sysv${UNAME_RELEASE} # Unknown i860-SVR4 + echo i860-unknown-sysv"$UNAME_RELEASE" # Unknown i860-SVR4 fi exit ;; mini*:CTIX:SYS*5:*) @@ -1114,9 +1168,9 @@ EOF test -r /etc/.relid \ && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && { echo i486-ncr-sysv4.3${OS_REL}; exit; } + && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ - && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; + && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } ;; 3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*) /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ && { echo i486-ncr-sysv4; exit; } ;; @@ -1125,28 +1179,28 @@ EOF test -r /etc/.relid \ && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && { echo i486-ncr-sysv4.3${OS_REL}; exit; } + && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ - && { echo i586-ncr-sysv4.3${OS_REL}; exit; } + && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } /bin/uname -p 2>/dev/null | /bin/grep pteron >/dev/null \ - && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; + && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } ;; m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*) - echo m68k-unknown-lynxos${UNAME_RELEASE} + echo m68k-unknown-lynxos"$UNAME_RELEASE" exit ;; mc68030:UNIX_System_V:4.*:*) echo m68k-atari-sysv4 exit ;; TSUNAMI:LynxOS:2.*:*) - echo sparc-unknown-lynxos${UNAME_RELEASE} + echo sparc-unknown-lynxos"$UNAME_RELEASE" exit ;; rs6000:LynxOS:2.*:*) - echo rs6000-unknown-lynxos${UNAME_RELEASE} + echo rs6000-unknown-lynxos"$UNAME_RELEASE" exit ;; PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.[02]*:*) - echo powerpc-unknown-lynxos${UNAME_RELEASE} + echo powerpc-unknown-lynxos"$UNAME_RELEASE" exit ;; SM[BE]S:UNIX_SV:*:*) - echo mips-dde-sysv${UNAME_RELEASE} + echo mips-dde-sysv"$UNAME_RELEASE" exit ;; RM*:ReliantUNIX-*:*:*) echo mips-sni-sysv4 @@ -1157,7 +1211,7 @@ EOF *:SINIX-*:*:*) if uname -p 2>/dev/null >/dev/null ; then UNAME_MACHINE=`(uname -p) 2>/dev/null` - echo ${UNAME_MACHINE}-sni-sysv4 + echo "$UNAME_MACHINE"-sni-sysv4 else echo ns32k-sni-sysv fi @@ -1177,23 +1231,23 @@ EOF exit ;; i*86:VOS:*:*) # From Paul.Green@stratus.com. - echo ${UNAME_MACHINE}-stratus-vos + echo "$UNAME_MACHINE"-stratus-vos exit ;; *:VOS:*:*) # From Paul.Green@stratus.com. echo hppa1.1-stratus-vos exit ;; mc68*:A/UX:*:*) - echo m68k-apple-aux${UNAME_RELEASE} + echo m68k-apple-aux"$UNAME_RELEASE" exit ;; news*:NEWS-OS:6*:*) echo mips-sony-newsos6 exit ;; R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*) if [ -d /usr/nec ]; then - echo mips-nec-sysv${UNAME_RELEASE} + echo mips-nec-sysv"$UNAME_RELEASE" else - echo mips-unknown-sysv${UNAME_RELEASE} + echo mips-unknown-sysv"$UNAME_RELEASE" fi exit ;; BeBox:BeOS:*:*) # BeOS running on hardware made by Be, PPC only. @@ -1212,65 +1266,93 @@ EOF echo x86_64-unknown-haiku exit ;; SX-4:SUPER-UX:*:*) - echo sx4-nec-superux${UNAME_RELEASE} + echo sx4-nec-superux"$UNAME_RELEASE" exit ;; SX-5:SUPER-UX:*:*) - echo sx5-nec-superux${UNAME_RELEASE} + echo sx5-nec-superux"$UNAME_RELEASE" exit ;; SX-6:SUPER-UX:*:*) - echo sx6-nec-superux${UNAME_RELEASE} + echo sx6-nec-superux"$UNAME_RELEASE" exit ;; SX-7:SUPER-UX:*:*) - echo sx7-nec-superux${UNAME_RELEASE} + echo sx7-nec-superux"$UNAME_RELEASE" exit ;; SX-8:SUPER-UX:*:*) - echo sx8-nec-superux${UNAME_RELEASE} + echo sx8-nec-superux"$UNAME_RELEASE" exit ;; SX-8R:SUPER-UX:*:*) - echo sx8r-nec-superux${UNAME_RELEASE} + echo sx8r-nec-superux"$UNAME_RELEASE" + exit ;; + SX-ACE:SUPER-UX:*:*) + echo sxace-nec-superux"$UNAME_RELEASE" exit ;; Power*:Rhapsody:*:*) - echo powerpc-apple-rhapsody${UNAME_RELEASE} + echo powerpc-apple-rhapsody"$UNAME_RELEASE" exit ;; *:Rhapsody:*:*) - echo ${UNAME_MACHINE}-apple-rhapsody${UNAME_RELEASE} + echo "$UNAME_MACHINE"-apple-rhapsody"$UNAME_RELEASE" exit ;; *:Darwin:*:*) UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown - case $UNAME_PROCESSOR in - i386) - eval $set_cc_for_build - if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then - if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_64BIT_ARCH >/dev/null - then - UNAME_PROCESSOR="x86_64" - fi - fi ;; - unknown) UNAME_PROCESSOR=powerpc ;; - esac - echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE} + set_cc_for_build + if test "$UNAME_PROCESSOR" = unknown ; then + UNAME_PROCESSOR=powerpc + fi + if test "`echo "$UNAME_RELEASE" | sed -e 's/\..*//'`" -le 10 ; then + if [ "$CC_FOR_BUILD" != no_compiler_found ]; then + if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null + then + case $UNAME_PROCESSOR in + i386) UNAME_PROCESSOR=x86_64 ;; + powerpc) UNAME_PROCESSOR=powerpc64 ;; + esac + fi + # On 10.4-10.6 one might compile for PowerPC via gcc -arch ppc + if (echo '#ifdef __POWERPC__'; echo IS_PPC; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_PPC >/dev/null + then + UNAME_PROCESSOR=powerpc + fi + fi + elif test "$UNAME_PROCESSOR" = i386 ; then + # Avoid executing cc on OS X 10.9, as it ships with a stub + # that puts up a graphical alert prompting to install + # developer tools. Any system running Mac OS X 10.7 or + # later (Darwin 11 and later) is required to have a 64-bit + # processor. This is not true of the ARM version of Darwin + # that Apple uses in portable devices. + UNAME_PROCESSOR=x86_64 + fi + echo "$UNAME_PROCESSOR"-apple-darwin"$UNAME_RELEASE" exit ;; *:procnto*:*:* | *:QNX:[0123456789]*:*) UNAME_PROCESSOR=`uname -p` - if test "$UNAME_PROCESSOR" = "x86"; then + if test "$UNAME_PROCESSOR" = x86; then UNAME_PROCESSOR=i386 UNAME_MACHINE=pc fi - echo ${UNAME_PROCESSOR}-${UNAME_MACHINE}-nto-qnx${UNAME_RELEASE} + echo "$UNAME_PROCESSOR"-"$UNAME_MACHINE"-nto-qnx"$UNAME_RELEASE" exit ;; *:QNX:*:4*) echo i386-pc-qnx exit ;; - NEO-?:NONSTOP_KERNEL:*:*) - echo neo-tandem-nsk${UNAME_RELEASE} + NEO-*:NONSTOP_KERNEL:*:*) + echo neo-tandem-nsk"$UNAME_RELEASE" exit ;; NSE-*:NONSTOP_KERNEL:*:*) - echo nse-tandem-nsk${UNAME_RELEASE} + echo nse-tandem-nsk"$UNAME_RELEASE" + exit ;; + NSR-*:NONSTOP_KERNEL:*:*) + echo nsr-tandem-nsk"$UNAME_RELEASE" exit ;; - NSR-?:NONSTOP_KERNEL:*:*) - echo nsr-tandem-nsk${UNAME_RELEASE} + NSV-*:NONSTOP_KERNEL:*:*) + echo nsv-tandem-nsk"$UNAME_RELEASE" + exit ;; + NSX-*:NONSTOP_KERNEL:*:*) + echo nsx-tandem-nsk"$UNAME_RELEASE" exit ;; *:NonStop-UX:*:*) echo mips-compaq-nonstopux @@ -1279,18 +1361,19 @@ EOF echo bs2000-siemens-sysv exit ;; DS/*:UNIX_System_V:*:*) - echo ${UNAME_MACHINE}-${UNAME_SYSTEM}-${UNAME_RELEASE} + echo "$UNAME_MACHINE"-"$UNAME_SYSTEM"-"$UNAME_RELEASE" exit ;; *:Plan9:*:*) # "uname -m" is not consistent, so use $cputype instead. 386 # is converted to i386 for consistency with other x86 # operating systems. - if test "$cputype" = "386"; then + # shellcheck disable=SC2154 + if test "$cputype" = 386; then UNAME_MACHINE=i386 else UNAME_MACHINE="$cputype" fi - echo ${UNAME_MACHINE}-unknown-plan9 + echo "$UNAME_MACHINE"-unknown-plan9 exit ;; *:TOPS-10:*:*) echo pdp10-unknown-tops10 @@ -1311,14 +1394,14 @@ EOF echo pdp10-unknown-its exit ;; SEI:*:*:SEIUX) - echo mips-sei-seiux${UNAME_RELEASE} + echo mips-sei-seiux"$UNAME_RELEASE" exit ;; *:DragonFly:*:*) - echo ${UNAME_MACHINE}-unknown-dragonfly`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` + echo "$UNAME_MACHINE"-unknown-dragonfly"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`" exit ;; *:*VMS:*:*) UNAME_MACHINE=`(uname -p) 2>/dev/null` - case "${UNAME_MACHINE}" in + case "$UNAME_MACHINE" in A*) echo alpha-dec-vms ; exit ;; I*) echo ia64-dec-vms ; exit ;; V*) echo vax-dec-vms ; exit ;; @@ -1327,182 +1410,48 @@ EOF echo i386-pc-xenix exit ;; i*86:skyos:*:*) - echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE}` | sed -e 's/ .*$//' + echo "$UNAME_MACHINE"-pc-skyos"`echo "$UNAME_RELEASE" | sed -e 's/ .*$//'`" exit ;; i*86:rdos:*:*) - echo ${UNAME_MACHINE}-pc-rdos + echo "$UNAME_MACHINE"-pc-rdos exit ;; i*86:AROS:*:*) - echo ${UNAME_MACHINE}-pc-aros + echo "$UNAME_MACHINE"-pc-aros exit ;; x86_64:VMkernel:*:*) - echo ${UNAME_MACHINE}-unknown-esx + echo "$UNAME_MACHINE"-unknown-esx + exit ;; + amd64:Isilon\ OneFS:*:*) + echo x86_64-unknown-onefs exit ;; esac -eval $set_cc_for_build -cat >$dummy.c < -# include -#endif -main () -{ -#if defined (sony) -#if defined (MIPSEB) - /* BFD wants "bsd" instead of "newsos". Perhaps BFD should be changed, - I don't know.... */ - printf ("mips-sony-bsd\n"); exit (0); -#else -#include - printf ("m68k-sony-newsos%s\n", -#ifdef NEWSOS4 - "4" -#else - "" -#endif - ); exit (0); -#endif -#endif - -#if defined (__arm) && defined (__acorn) && defined (__unix) - printf ("arm-acorn-riscix\n"); exit (0); -#endif +echo "$0: unable to guess system type" >&2 -#if defined (hp300) && !defined (hpux) - printf ("m68k-hp-bsd\n"); exit (0); -#endif +case "$UNAME_MACHINE:$UNAME_SYSTEM" in + mips:Linux | mips64:Linux) + # If we got here on MIPS GNU/Linux, output extra information. + cat >&2 </dev/null`; - if (version < 4) - printf ("%s-next-nextstep%d\n", __ARCHITECTURE__, version); - else - printf ("%s-next-openstep%d\n", __ARCHITECTURE__, version); - exit (0); -#endif - -#if defined (MULTIMAX) || defined (n16) -#if defined (UMAXV) - printf ("ns32k-encore-sysv\n"); exit (0); -#else -#if defined (CMU) - printf ("ns32k-encore-mach\n"); exit (0); -#else - printf ("ns32k-encore-bsd\n"); exit (0); -#endif -#endif -#endif - -#if defined (__386BSD__) - printf ("i386-pc-bsd\n"); exit (0); -#endif - -#if defined (sequent) -#if defined (i386) - printf ("i386-sequent-dynix\n"); exit (0); -#endif -#if defined (ns32000) - printf ("ns32k-sequent-dynix\n"); exit (0); -#endif -#endif - -#if defined (_SEQUENT_) - struct utsname un; - - uname(&un); - - if (strncmp(un.version, "V2", 2) == 0) { - printf ("i386-sequent-ptx2\n"); exit (0); - } - if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */ - printf ("i386-sequent-ptx1\n"); exit (0); - } - printf ("i386-sequent-ptx\n"); exit (0); - -#endif - -#if defined (vax) -# if !defined (ultrix) -# include -# if defined (BSD) -# if BSD == 43 - printf ("vax-dec-bsd4.3\n"); exit (0); -# else -# if BSD == 199006 - printf ("vax-dec-bsd4.3reno\n"); exit (0); -# else - printf ("vax-dec-bsd\n"); exit (0); -# endif -# endif -# else - printf ("vax-dec-bsd\n"); exit (0); -# endif -# else - printf ("vax-dec-ultrix\n"); exit (0); -# endif -#endif - -#if defined (alliant) && defined (i860) - printf ("i860-alliant-bsd\n"); exit (0); -#endif - - exit (1); -} +NOTE: MIPS GNU/Linux systems require a C compiler to fully recognize +the system type. Please install a C compiler and try again. EOF - -$CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null && SYSTEM_NAME=`$dummy` && - { echo "$SYSTEM_NAME"; exit; } - -# Apollos put the system type in the environment. - -test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit; } - -# Convex versions that predate uname can use getsysinfo(1) - -if [ -x /usr/convex/getsysinfo ] -then - case `getsysinfo -f cpu_type` in - c1*) - echo c1-convex-bsd - exit ;; - c2*) - if getsysinfo -f scalar_acc - then echo c32-convex-bsd - else echo c2-convex-bsd - fi - exit ;; - c34*) - echo c34-convex-bsd - exit ;; - c38*) - echo c38-convex-bsd - exit ;; - c4*) - echo c4-convex-bsd - exit ;; - esac -fi + ;; +esac cat >&2 < in order to provide the needed -information to handle your system. +If $0 has already been updated, send the following data and any +information you think might be pertinent to config-patches@gnu.org to +provide the necessary information to handle your system. config.guess timestamp = $timestamp @@ -1521,16 +1470,16 @@ hostinfo = `(hostinfo) 2>/dev/null` /usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null` /usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null` -UNAME_MACHINE = ${UNAME_MACHINE} -UNAME_RELEASE = ${UNAME_RELEASE} -UNAME_SYSTEM = ${UNAME_SYSTEM} -UNAME_VERSION = ${UNAME_VERSION} +UNAME_MACHINE = "$UNAME_MACHINE" +UNAME_RELEASE = "$UNAME_RELEASE" +UNAME_SYSTEM = "$UNAME_SYSTEM" +UNAME_VERSION = "$UNAME_VERSION" EOF exit 1 # Local variables: -# eval: (add-hook 'write-file-hooks 'time-stamp) +# eval: (add-hook 'before-save-hook 'time-stamp) # time-stamp-start: "timestamp='" # time-stamp-format: "%:y-%02m-%02d" # time-stamp-end: "'" diff --git a/config/config.sub b/config/config.sub index bdda9e4a32c..c19e671805a 100755 --- a/config/config.sub +++ b/config/config.sub @@ -1,36 +1,31 @@ #! /bin/sh # Configuration validation subroutine script. -# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, -# 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -# 2011, 2012 Free Software Foundation, Inc. +# Copyright 1992-2018 Free Software Foundation, Inc. -timestamp='2012-08-18' +timestamp='2018-08-13' -# This file is (in principle) common to ALL GNU software. -# The presence of a machine in this file suggests that SOME GNU software -# can handle that machine. It does not imply ALL GNU software can. -# -# This file is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program; if not, see . +# along with this program; if not, see . # # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a # configuration script generated by Autoconf, you may include it under -# the same distribution terms that you use for the rest of that program. +# the same distribution terms that you use for the rest of that +# program. This Exception is an additional permission under section 7 +# of the GNU General Public License, version 3 ("GPLv3"). -# Please send patches to . Submit a context -# diff and a properly formatted GNU ChangeLog entry. +# Please send patches to . # # Configuration subroutine to validate and canonicalize a configuration type. # Supply the specified configuration type as an argument. @@ -38,7 +33,7 @@ timestamp='2012-08-18' # Otherwise, we print the canonical config type on stdout and succeed. # You can get the latest version of this script from: -# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD +# https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub # This file is supposed to be the same for all GNU packages # and recognize all the CPU types, system types and aliases @@ -58,12 +53,11 @@ timestamp='2012-08-18' me=`echo "$0" | sed -e 's,.*/,,'` usage="\ -Usage: $0 [OPTION] CPU-MFR-OPSYS - $0 [OPTION] ALIAS +Usage: $0 [OPTION] CPU-MFR-OPSYS or ALIAS Canonicalize a configuration name. -Operation modes: +Options: -h, --help print this help, then exit -t, --time-stamp print date of last modification, then exit -v, --version print version number, then exit @@ -73,9 +67,7 @@ Report bugs and patches to ." version="\ GNU config.sub ($timestamp) -Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, -2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 -Free Software Foundation, Inc. +Copyright 1992-2018 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -102,7 +94,7 @@ while test $# -gt 0 ; do *local*) # First pass through any local machine types. - echo $1 + echo "$1" exit ;; * ) @@ -118,162 +110,596 @@ case $# in exit 1;; esac -# Separate what the user gave into CPU-COMPANY and OS or KERNEL-OS (if any). -# Here we must recognize all the valid KERNEL-OS combinations. -maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'` -case $maybe_os in - nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \ - linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \ - knetbsd*-gnu* | netbsd*-gnu* | \ - kopensolaris*-gnu* | \ - storm-chaos* | os2-emx* | rtmk-nova*) - os=-$maybe_os - basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'` - ;; - android-linux) - os=-linux-android - basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`-unknown - ;; - *) - basic_machine=`echo $1 | sed 's/-[^-]*$//'` - if [ $basic_machine != $1 ] - then os=`echo $1 | sed 's/.*-/-/'` - else os=; fi - ;; -esac +# Split fields of configuration type +IFS="-" read -r field1 field2 field3 field4 <&2 + exit 1 ;; - -wrs) - os=-vxworks - basic_machine=$1 + *-*-*-*) + basic_machine=$field1-$field2 + os=$field3-$field4 ;; - -chorusos*) - os=-chorusos - basic_machine=$1 + *-*-*) + # Ambiguous whether COMPANY is present, or skipped and KERNEL-OS is two + # parts + maybe_os=$field2-$field3 + case $maybe_os in + nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc \ + | linux-newlib* | linux-musl* | linux-uclibc* | uclinux-uclibc* \ + | uclinux-gnu* | kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* \ + | netbsd*-eabi* | kopensolaris*-gnu* | cloudabi*-eabi* \ + | storm-chaos* | os2-emx* | rtmk-nova*) + basic_machine=$field1 + os=$maybe_os + ;; + android-linux) + basic_machine=$field1-unknown + os=linux-android + ;; + *) + basic_machine=$field1-$field2 + os=$field3 + ;; + esac ;; - -chorusrdb) - os=-chorusrdb - basic_machine=$1 + *-*) + # Second component is usually, but not always the OS + case $field2 in + # Prevent following clause from handling this valid os + sun*os*) + basic_machine=$field1 + os=$field2 + ;; + # Manufacturers + dec* | mips* | sequent* | encore* | pc532* | sgi* | sony* \ + | att* | 7300* | 3300* | delta* | motorola* | sun[234]* \ + | unicom* | ibm* | next | hp | isi* | apollo | altos* \ + | convergent* | ncr* | news | 32* | 3600* | 3100* | hitachi* \ + | c[123]* | convex* | sun | crds | omron* | dg | ultra | tti* \ + | harris | dolphin | highlevel | gould | cbm | ns | masscomp \ + | apple | axis | knuth | cray | microblaze* \ + | sim | cisco | oki | wec | wrs | winbond) + basic_machine=$field1-$field2 + os= + ;; + *) + basic_machine=$field1 + os=$field2 + ;; + esac ;; - -hiux*) - os=-hiuxwe2 + *) + # Convert single-component short-hands not valid as part of + # multi-component configurations. + case $field1 in + 386bsd) + basic_machine=i386-pc + os=bsd + ;; + a29khif) + basic_machine=a29k-amd + os=udi + ;; + adobe68k) + basic_machine=m68010-adobe + os=scout + ;; + alliant) + basic_machine=fx80-alliant + os= + ;; + altos | altos3068) + basic_machine=m68k-altos + os= + ;; + am29k) + basic_machine=a29k-none + os=bsd + ;; + amdahl) + basic_machine=580-amdahl + os=sysv + ;; + amigaos | amigados) + basic_machine=m68k-unknown + os=amigaos + ;; + amigaunix | amix) + basic_machine=m68k-unknown + os=sysv4 + ;; + apollo68) + basic_machine=m68k-apollo + os=sysv + ;; + apollo68bsd) + basic_machine=m68k-apollo + os=bsd + ;; + aros) + basic_machine=i386-pc + os=aros + ;; + aux) + basic_machine=m68k-apple + os=aux + ;; + balance) + basic_machine=ns32k-sequent + os=dynix + ;; + blackfin) + basic_machine=bfin-unknown + os=linux + ;; + cegcc) + basic_machine=arm-unknown + os=cegcc + ;; + convex-c1) + basic_machine=c1-convex + os=bsd + ;; + convex-c2) + basic_machine=c2-convex + os=bsd + ;; + convex-c32) + basic_machine=c32-convex + os=bsd + ;; + convex-c34) + basic_machine=c34-convex + os=bsd + ;; + convex-c38) + basic_machine=c38-convex + os=bsd + ;; + cray) + basic_machine=j90-cray + os=unicos + ;; + crds | unos) + basic_machine=m68k-crds + os= + ;; + delta88) + basic_machine=m88k-motorola + os=sysv3 + ;; + dicos) + basic_machine=i686-pc + os=dicos + ;; + djgpp) + basic_machine=i586-pc + os=msdosdjgpp + ;; + ebmon29k) + basic_machine=a29k-amd + os=ebmon + ;; + es1800 | OSE68k | ose68k | ose | OSE) + basic_machine=m68k-ericsson + os=ose + ;; + gmicro) + basic_machine=tron-gmicro + os=sysv + ;; + go32) + basic_machine=i386-pc + os=go32 + ;; + h8300hms) + basic_machine=h8300-hitachi + os=hms + ;; + h8300xray) + basic_machine=h8300-hitachi + os=xray + ;; + h8500hms) + basic_machine=h8500-hitachi + os=hms + ;; + harris) + basic_machine=m88k-harris + os=sysv3 + ;; + hp300bsd) + basic_machine=m68k-hp + os=bsd + ;; + hp300hpux) + basic_machine=m68k-hp + os=hpux + ;; + hppaosf) + basic_machine=hppa1.1-hp + os=osf + ;; + hppro) + basic_machine=hppa1.1-hp + os=proelf + ;; + i386mach) + basic_machine=i386-mach + os=mach + ;; + vsta) + basic_machine=i386-pc + os=vsta + ;; + isi68 | isi) + basic_machine=m68k-isi + os=sysv + ;; + m68knommu) + basic_machine=m68k-unknown + os=linux + ;; + magnum | m3230) + basic_machine=mips-mips + os=sysv + ;; + merlin) + basic_machine=ns32k-utek + os=sysv + ;; + mingw64) + basic_machine=x86_64-pc + os=mingw64 + ;; + mingw32) + basic_machine=i686-pc + os=mingw32 + ;; + mingw32ce) + basic_machine=arm-unknown + os=mingw32ce + ;; + monitor) + basic_machine=m68k-rom68k + os=coff + ;; + morphos) + basic_machine=powerpc-unknown + os=morphos + ;; + moxiebox) + basic_machine=moxie-unknown + os=moxiebox + ;; + msdos) + basic_machine=i386-pc + os=msdos + ;; + msys) + basic_machine=i686-pc + os=msys + ;; + mvs) + basic_machine=i370-ibm + os=mvs + ;; + nacl) + basic_machine=le32-unknown + os=nacl + ;; + ncr3000) + basic_machine=i486-ncr + os=sysv4 + ;; + netbsd386) + basic_machine=i386-pc + os=netbsd + ;; + netwinder) + basic_machine=armv4l-rebel + os=linux + ;; + news | news700 | news800 | news900) + basic_machine=m68k-sony + os=newsos + ;; + news1000) + basic_machine=m68030-sony + os=newsos + ;; + necv70) + basic_machine=v70-nec + os=sysv + ;; + nh3000) + basic_machine=m68k-harris + os=cxux + ;; + nh[45]000) + basic_machine=m88k-harris + os=cxux + ;; + nindy960) + basic_machine=i960-intel + os=nindy + ;; + mon960) + basic_machine=i960-intel + os=mon960 + ;; + nonstopux) + basic_machine=mips-compaq + os=nonstopux + ;; + os400) + basic_machine=powerpc-ibm + os=os400 + ;; + OSE68000 | ose68000) + basic_machine=m68000-ericsson + os=ose + ;; + os68k) + basic_machine=m68k-none + os=os68k + ;; + paragon) + basic_machine=i860-intel + os=osf + ;; + parisc) + basic_machine=hppa-unknown + os=linux + ;; + pw32) + basic_machine=i586-unknown + os=pw32 + ;; + rdos | rdos64) + basic_machine=x86_64-pc + os=rdos + ;; + rdos32) + basic_machine=i386-pc + os=rdos + ;; + rom68k) + basic_machine=m68k-rom68k + os=coff + ;; + sa29200) + basic_machine=a29k-amd + os=udi + ;; + sei) + basic_machine=mips-sei + os=seiux + ;; + sps7) + basic_machine=m68k-bull + os=sysv2 + ;; + st2000) + basic_machine=m68k-tandem + os= + ;; + stratus) + basic_machine=i860-stratus + os=sysv4 + ;; + sun2) + basic_machine=m68000-sun + os= + ;; + sun2os3) + basic_machine=m68000-sun + os=sunos3 + ;; + sun2os4) + basic_machine=m68000-sun + os=sunos4 + ;; + sun3) + basic_machine=m68k-sun + os= + ;; + sun3os3) + basic_machine=m68k-sun + os=sunos3 + ;; + sun3os4) + basic_machine=m68k-sun + os=sunos4 + ;; + sun4) + basic_machine=sparc-sun + os= + ;; + sun4os3) + basic_machine=sparc-sun + os=sunos3 + ;; + sun4os4) + basic_machine=sparc-sun + os=sunos4 + ;; + sun4sol2) + basic_machine=sparc-sun + os=solaris2 + ;; + sun386 | sun386i | roadrunner) + basic_machine=i386-sun + os= + ;; + sv1) + basic_machine=sv1-cray + os=unicos + ;; + symmetry) + basic_machine=i386-sequent + os=dynix + ;; + t3e) + basic_machine=alphaev5-cray + os=unicos + ;; + t90) + basic_machine=t90-cray + os=unicos + ;; + toad1) + basic_machine=pdp10-xkl + os=tops20 + ;; + tpf) + basic_machine=s390x-ibm + os=tpf + ;; + udi29k) + basic_machine=a29k-amd + os=udi + ;; + ultra3) + basic_machine=a29k-nyu + os=sym1 + ;; + v810 | necv810) + basic_machine=v810-nec + os=none + ;; + vaxv) + basic_machine=vax-dec + os=sysv + ;; + vms) + basic_machine=vax-dec + os=vms + ;; + vxworks960) + basic_machine=i960-wrs + os=vxworks + ;; + vxworks68) + basic_machine=m68k-wrs + os=vxworks + ;; + vxworks29k) + basic_machine=a29k-wrs + os=vxworks + ;; + xbox) + basic_machine=i686-pc + os=mingw32 + ;; + ymp) + basic_machine=ymp-cray + os=unicos + ;; + *) + basic_machine=$1 + os= + ;; + esac ;; - -sco6) - os=-sco5v6 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` +esac + +# Decode aliases for certain CPU-COMPANY combinations. +case $basic_machine in + # Here we handle the default manufacturer of certain CPU types. It is in + # some cases the only manufacturer, in others, it is the most popular. + craynv) + basic_machine=craynv-cray + os=${os:-unicosmp} ;; - -sco5) - os=-sco3.2v5 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + fx80) + basic_machine=fx80-alliant ;; - -sco4) - os=-sco3.2v4 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + w89k) + basic_machine=hppa1.1-winbond ;; - -sco3.2.[4-9]*) - os=`echo $os | sed -e 's/sco3.2./sco3.2v/'` - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + op50n) + basic_machine=hppa1.1-oki ;; - -sco3.2v[4-9]*) - # Don't forget version if it is 3.2v4 or newer. - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + op60c) + basic_machine=hppa1.1-oki ;; - -sco5v6*) - # Don't forget version if it is 3.2v4 or newer. - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + romp) + basic_machine=romp-ibm ;; - -sco*) - os=-sco3.2v2 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + mmix) + basic_machine=mmix-knuth ;; - -udk*) - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + rs6000) + basic_machine=rs6000-ibm ;; - -isc) - os=-isc2.2 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + vax) + basic_machine=vax-dec ;; - -clix*) - basic_machine=clipper-intergraph + pdp11) + basic_machine=pdp11-dec ;; - -isc*) - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + we32k) + basic_machine=we32k-att ;; - -lynx*178) - os=-lynxos178 + cydra) + basic_machine=cydra-cydrome ;; - -lynx*5) - os=-lynxos5 + i370-ibm* | ibm*) + basic_machine=i370-ibm ;; - -lynx*) - os=-lynxos + orion) + basic_machine=orion-highlevel ;; - -ptx*) - basic_machine=`echo $1 | sed -e 's/86-.*/86-sequent/'` + orion105) + basic_machine=clipper-highlevel ;; - -windowsnt*) - os=`echo $os | sed -e 's/windowsnt/winnt/'` + mac | mpw | mac-mpw) + basic_machine=m68k-apple ;; - -psos*) - os=-psos + pmac | pmac-mpw) + basic_machine=powerpc-apple ;; - -mint | -mint[0-9]*) - basic_machine=m68k-atari - os=-mint + xps | xps100) + basic_machine=xps100-honeywell ;; -esac -# Decode aliases for certain CPU-COMPANY combinations. -case $basic_machine in # Recognize the basic CPU types without company name. # Some are omitted here because they have special meanings below. 1750a | 580 \ | a29k \ | aarch64 | aarch64_be \ + | abacus \ | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \ | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \ | am33_2.0 \ - | arc | arm | arm[bl]e | arme[lb] | armv[2345] | armv[345][lb] | avr | avr32 \ - | be32 | be64 \ + | arc | arceb \ + | arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv6m | armv[78][arm] \ + | avr | avr32 \ + | asmjs \ + | ba \ + | be32 | be64 \ | bfin \ - | c4x | clipper \ + | c4x | c8051 | clipper | csky \ | d10v | d30v | dlx | dsp16xx \ - | epiphany \ - | fido | fr30 | frv \ + | e2k | epiphany \ + | fido | fr30 | frv | ft32 \ | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ | hexagon \ - | i370 | i860 | i960 | ia64 \ + | i370 | i860 | i960 | ia16 | ia64 \ | ip2k | iq2000 \ + | k1om \ | le32 | le64 \ | lm32 \ | m32c | m32r | m32rle | m68000 | m68k | m88k \ - | maxq | mb | microblaze | mcore | mep | metag \ + | m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | nvptx | picochip \ + | maxq | mb | microblaze | microblazeel | mcore | mep | metag \ | mips | mipsbe | mipseb | mipsel | mipsle \ | mips16 \ | mips64 | mips64el \ @@ -287,26 +713,31 @@ case $basic_machine in | mips64vr5900 | mips64vr5900el \ | mipsisa32 | mipsisa32el \ | mipsisa32r2 | mipsisa32r2el \ + | mipsisa32r6 | mipsisa32r6el \ | mipsisa64 | mipsisa64el \ | mipsisa64r2 | mipsisa64r2el \ + | mipsisa64r6 | mipsisa64r6el \ | mipsisa64sb1 | mipsisa64sb1el \ | mipsisa64sr71k | mipsisa64sr71kel \ + | mipsr5900 | mipsr5900el \ | mipstx39 | mipstx39el \ | mn10200 | mn10300 \ | moxie \ | mt \ | msp430 \ | nds32 | nds32le | nds32be \ - | nios | nios2 \ + | nfp \ + | nios | nios2 | nios2eb | nios2el \ | ns16k | ns32k \ - | open8 \ - | or32 \ - | pdp10 | pdp11 | pj | pjl \ + | open8 | or1k | or1knd | or32 \ + | pdp10 | pj | pjl \ | powerpc | powerpc64 | powerpc64le | powerpcle \ + | pru \ | pyramid \ + | riscv | riscv32 | riscv64 \ | rl78 | rx \ | score \ - | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \ + | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[234]eb | sheb | shbe | shle | sh[1234]le | sh[23]ele \ | sh64 | sh64le \ | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \ | sparcv8 | sparcv9 | sparcv9b | sparcv9v \ @@ -314,8 +745,9 @@ case $basic_machine in | tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \ | ubicom32 \ | v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \ - | we32k \ - | x86 | xc16x | xstormy16 | xtensa \ + | visium \ + | wasm32 \ + | x86 | xc16x | xstormy16 | xgate | xtensa \ | z8k | z80) basic_machine=$basic_machine-unknown ;; @@ -328,23 +760,23 @@ case $basic_machine in c6x) basic_machine=tic6x-unknown ;; - m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | picochip) - basic_machine=$basic_machine-unknown - os=-none + leon|leon[3-9]) + basic_machine=sparc-$basic_machine + ;; + m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65) + ;; + m9s12z | m68hcs12z | hcs12z | s12z) + basic_machine=s12z-unknown ;; - m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65 | z8k) + m9s12z-* | m68hcs12z-* | hcs12z-* | s12z-*) + basic_machine=s12z-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; ms1) basic_machine=mt-unknown ;; - strongarm | thumb | xscale) basic_machine=arm-unknown ;; - xgate) - basic_machine=$basic_machine-unknown - os=-none - ;; xscaleeb) basic_machine=armeb-unknown ;; @@ -359,37 +791,40 @@ case $basic_machine in i*86 | x86_64) basic_machine=$basic_machine-pc ;; - # Object if more than one company name word. - *-*-*) - echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2 - exit 1 - ;; # Recognize the basic CPU types with company name. - 580-* \ + 1750a-* | 580-* \ | a29k-* \ | aarch64-* | aarch64_be-* \ + | abacus-* \ | alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \ | alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \ - | alphapca5[67]-* | alpha64pca5[67]-* | arc-* \ - | arm-* | armbe-* | armle-* | armeb-* | armv*-* \ + | alphapca5[67]-* | alpha64pca5[67]-* \ + | am33_2.0-* \ + | arc-* | arceb-* \ + | arm-* | arm[lb]e-* | arme[lb]-* | armv*-* \ | avr-* | avr32-* \ + | asmjs-* \ + | ba-* \ | be32-* | be64-* \ | bfin-* | bs2000-* \ | c[123]* | c30-* | [cjt]90-* | c4x-* \ - | clipper-* | craynv-* | cydra-* \ - | d10v-* | d30v-* | dlx-* \ - | elxsi-* \ - | f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \ + | c8051-* | clipper-* | craynv-* | csky-* | cydra-* \ + | d10v-* | d30v-* | dlx-* | dsp16xx-* \ + | e2k-* | elxsi-* | epiphany-* \ + | f30[01]-* | f700-* | fido-* | fr30-* | frv-* | ft32-* | fx80-* \ | h8300-* | h8500-* \ | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \ | hexagon-* \ - | i*86-* | i860-* | i960-* | ia64-* \ + | i370-* | i*86-* | i860-* | i960-* | ia16-* | ia64-* \ | ip2k-* | iq2000-* \ + | k1om-* \ | le32-* | le64-* \ | lm32-* \ | m32c-* | m32r-* | m32rle-* \ - | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \ - | m88110-* | m88k-* | maxq-* | mcore-* | metag-* | microblaze-* \ + | m5200-* | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* | v70-* | w65-* \ + | m6811-* | m68hc11-* | m6812-* | m68hc12-* | m68hcs12x-* | nvptx-* | picochip-* \ + | m88110-* | m88k-* | maxq-* | mb-* | mcore-* | mep-* | metag-* \ + | microblaze-* | microblazeel-* \ | mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \ | mips16-* \ | mips64-* | mips64el-* \ @@ -403,37 +838,50 @@ case $basic_machine in | mips64vr5900-* | mips64vr5900el-* \ | mipsisa32-* | mipsisa32el-* \ | mipsisa32r2-* | mipsisa32r2el-* \ + | mipsisa32r6-* | mipsisa32r6el-* \ | mipsisa64-* | mipsisa64el-* \ | mipsisa64r2-* | mipsisa64r2el-* \ + | mipsisa64r6-* | mipsisa64r6el-* \ | mipsisa64sb1-* | mipsisa64sb1el-* \ | mipsisa64sr71k-* | mipsisa64sr71kel-* \ + | mipsr5900-* | mipsr5900el-* \ | mipstx39-* | mipstx39el-* \ | mmix-* \ + | mn10200-* | mn10300-* \ + | moxie-* \ | mt-* \ | msp430-* \ | nds32-* | nds32le-* | nds32be-* \ - | nios-* | nios2-* \ + | nfp-* \ + | nios-* | nios2-* | nios2eb-* | nios2el-* \ | none-* | np1-* | ns16k-* | ns32k-* \ | open8-* \ + | or1k*-* \ + | or32-* \ | orion-* \ | pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \ | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \ + | pru-* \ | pyramid-* \ + | riscv-* | riscv32-* | riscv64-* \ | rl78-* | romp-* | rs6000-* | rx-* \ - | sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \ - | shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \ + | score-* \ + | sh-* | sh[1234]-* | sh[24]a-* | sh[24]ae[lb]-* | sh[23]e-* | she[lb]-* | sh[lb]e-* \ + | sh[1234]e[lb]-* | sh[12345][lb]e-* | sh[23]ele-* | sh64-* | sh64le-* \ | sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \ | sparclite-* \ - | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx?-* \ + | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx*-* \ + | spu-* \ | tahoe-* \ | tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \ - | tile*-* \ | tron-* \ | ubicom32-* \ | v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \ | vax-* \ + | visium-* \ + | wasm32-* \ | we32k-* \ - | x86-* | x86_64-* | xc16x-* | xps100-* \ + | x86-* | x86_64-* | xc16x-* | xgate-* | xps100-* \ | xstormy16-* | xtensa*-* \ | ymp-* \ | z8k-* | z80-*) @@ -444,141 +892,45 @@ case $basic_machine in ;; # Recognize the various machine names and aliases which stand # for a CPU type and a company and sometimes even an OS. - 386bsd) - basic_machine=i386-unknown - os=-bsd - ;; 3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc) basic_machine=m68000-att ;; 3b*) basic_machine=we32k-att ;; - a29khif) - basic_machine=a29k-amd - os=-udi - ;; - abacus) - basic_machine=abacus-unknown - ;; - adobe68k) - basic_machine=m68010-adobe - os=-scout - ;; - alliant | fx80) - basic_machine=fx80-alliant - ;; - altos | altos3068) - basic_machine=m68k-altos - ;; - am29k) - basic_machine=a29k-none - os=-bsd - ;; amd64) basic_machine=x86_64-pc ;; amd64-*) - basic_machine=x86_64-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - amdahl) - basic_machine=580-amdahl - os=-sysv + basic_machine=x86_64-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; amiga | amiga-*) basic_machine=m68k-unknown ;; - amigaos | amigados) - basic_machine=m68k-unknown - os=-amigaos - ;; - amigaunix | amix) - basic_machine=m68k-unknown - os=-sysv4 - ;; - apollo68) - basic_machine=m68k-apollo - os=-sysv - ;; - apollo68bsd) - basic_machine=m68k-apollo - os=-bsd - ;; - aros) - basic_machine=i386-pc - os=-aros - ;; - aux) - basic_machine=m68k-apple - os=-aux - ;; - balance) - basic_machine=ns32k-sequent - os=-dynix - ;; - blackfin) - basic_machine=bfin-unknown - os=-linux - ;; blackfin-*) - basic_machine=bfin-`echo $basic_machine | sed 's/^[^-]*-//'` - os=-linux + basic_machine=bfin-`echo "$basic_machine" | sed 's/^[^-]*-//'` + os=linux ;; bluegene*) basic_machine=powerpc-ibm - os=-cnk + os=cnk ;; c54x-*) - basic_machine=tic54x-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=tic54x-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; c55x-*) - basic_machine=tic55x-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=tic55x-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; c6x-*) - basic_machine=tic6x-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=tic6x-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; c90) basic_machine=c90-cray - os=-unicos - ;; - cegcc) - basic_machine=arm-unknown - os=-cegcc - ;; - convex-c1) - basic_machine=c1-convex - os=-bsd - ;; - convex-c2) - basic_machine=c2-convex - os=-bsd - ;; - convex-c32) - basic_machine=c32-convex - os=-bsd - ;; - convex-c34) - basic_machine=c34-convex - os=-bsd - ;; - convex-c38) - basic_machine=c38-convex - os=-bsd - ;; - cray | j90) - basic_machine=j90-cray - os=-unicos - ;; - craynv) - basic_machine=craynv-cray - os=-unicosmp + os=${os:-unicos} ;; cr16 | cr16-*) basic_machine=cr16-unknown - os=-elf - ;; - crds | unos) - basic_machine=m68k-crds + os=${os:-elf} ;; crisv32 | crisv32-* | etraxfs*) basic_machine=crisv32-axis @@ -588,7 +940,7 @@ case $basic_machine in ;; crx) basic_machine=crx-unknown - os=-elf + os=${os:-elf} ;; da30 | da30-*) basic_machine=m68k-da30 @@ -598,50 +950,38 @@ case $basic_machine in ;; decsystem10* | dec10*) basic_machine=pdp10-dec - os=-tops10 + os=tops10 ;; decsystem20* | dec20*) basic_machine=pdp10-dec - os=-tops20 + os=tops20 ;; delta | 3300 | motorola-3300 | motorola-delta \ | 3300-motorola | delta-motorola) basic_machine=m68k-motorola ;; - delta88) - basic_machine=m88k-motorola - os=-sysv3 - ;; - dicos) - basic_machine=i686-pc - os=-dicos - ;; - djgpp) - basic_machine=i586-pc - os=-msdosdjgpp - ;; dpx20 | dpx20-*) basic_machine=rs6000-bull - os=-bosx + os=${os:-bosx} ;; - dpx2* | dpx2*-bull) + dpx2*) basic_machine=m68k-bull - os=-sysv3 + os=sysv3 ;; - ebmon29k) - basic_machine=a29k-amd - os=-ebmon + e500v[12]) + basic_machine=powerpc-unknown + os=$os"spe" ;; - elxsi) - basic_machine=elxsi-elxsi - os=-bsd + e500v[12]-*) + basic_machine=powerpc-`echo "$basic_machine" | sed 's/^[^-]*-//'` + os=$os"spe" ;; encore | umax | mmax) basic_machine=ns32k-encore ;; - es1800 | OSE68k | ose68k | ose | OSE) - basic_machine=m68k-ericsson - os=-ose + elxsi) + basic_machine=elxsi-elxsi + os=${os:-bsd} ;; fx2800) basic_machine=i860-alliant @@ -649,45 +989,13 @@ case $basic_machine in genix) basic_machine=ns32k-ns ;; - gmicro) - basic_machine=tron-gmicro - os=-sysv - ;; - go32) - basic_machine=i386-pc - os=-go32 - ;; h3050r* | hiux*) basic_machine=hppa1.1-hitachi - os=-hiuxwe2 - ;; - h8300hms) - basic_machine=h8300-hitachi - os=-hms - ;; - h8300xray) - basic_machine=h8300-hitachi - os=-xray - ;; - h8500hms) - basic_machine=h8500-hitachi - os=-hms - ;; - harris) - basic_machine=m88k-harris - os=-sysv3 + os=hiuxwe2 ;; hp300-*) basic_machine=m68k-hp ;; - hp300bsd) - basic_machine=m68k-hp - os=-bsd - ;; - hp300hpux) - basic_machine=m68k-hp - os=-hpux - ;; hp3k9[0-9][0-9] | hp9[0-9][0-9]) basic_machine=hppa1.0-hp ;; @@ -717,193 +1025,79 @@ case $basic_machine in hp9k8[0-9][0-9] | hp8[0-9][0-9]) basic_machine=hppa1.0-hp ;; - hppa-next) - os=-nextstep3 - ;; - hppaosf) - basic_machine=hppa1.1-hp - os=-osf - ;; - hppro) - basic_machine=hppa1.1-hp - os=-proelf - ;; - i370-ibm* | ibm*) - basic_machine=i370-ibm - ;; i*86v32) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` - os=-sysv32 + basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` + os=sysv32 ;; i*86v4*) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` - os=-sysv4 + basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` + os=sysv4 ;; i*86v) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` - os=-sysv + basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` + os=sysv ;; i*86sol2) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` - os=-solaris2 - ;; - i386mach) - basic_machine=i386-mach - os=-mach + basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` + os=solaris2 ;; - i386-vsta | vsta) - basic_machine=i386-unknown - os=-vsta + j90 | j90-cray) + basic_machine=j90-cray + os=${os:-unicos} ;; iris | iris4d) basic_machine=mips-sgi case $os in - -irix*) + irix*) ;; *) - os=-irix4 + os=irix4 ;; esac ;; - isi68 | isi) - basic_machine=m68k-isi - os=-sysv - ;; - m68knommu) - basic_machine=m68k-unknown - os=-linux + leon-*|leon[3-9]-*) + basic_machine=sparc-`echo "$basic_machine" | sed 's/-.*//'` ;; m68knommu-*) - basic_machine=m68k-`echo $basic_machine | sed 's/^[^-]*-//'` - os=-linux - ;; - m88k-omron*) - basic_machine=m88k-omron - ;; - magnum | m3230) - basic_machine=mips-mips - os=-sysv + basic_machine=m68k-`echo "$basic_machine" | sed 's/^[^-]*-//'` + os=linux ;; - merlin) - basic_machine=ns32k-utek - os=-sysv - ;; - microblaze) + microblaze*) basic_machine=microblaze-xilinx ;; - mingw64) - basic_machine=x86_64-pc - os=-mingw64 - ;; - mingw32) - basic_machine=i386-pc - os=-mingw32 - ;; - mingw32ce) - basic_machine=arm-unknown - os=-mingw32ce - ;; miniframe) basic_machine=m68000-convergent ;; - *mint | -mint[0-9]* | *MiNT | *MiNT[0-9]*) + *mint | mint[0-9]* | *MiNT | *MiNT[0-9]*) basic_machine=m68k-atari - os=-mint + os=mint ;; mips3*-*) - basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'` + basic_machine=`echo "$basic_machine" | sed -e 's/mips3/mips64/'` ;; mips3*) - basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown - ;; - monitor) - basic_machine=m68k-rom68k - os=-coff - ;; - morphos) - basic_machine=powerpc-unknown - os=-morphos - ;; - msdos) - basic_machine=i386-pc - os=-msdos + basic_machine=`echo "$basic_machine" | sed -e 's/mips3/mips64/'`-unknown ;; ms1-*) - basic_machine=`echo $basic_machine | sed -e 's/ms1-/mt-/'` - ;; - msys) - basic_machine=i386-pc - os=-msys - ;; - mvs) - basic_machine=i370-ibm - os=-mvs - ;; - nacl) - basic_machine=le32-unknown - os=-nacl - ;; - ncr3000) - basic_machine=i486-ncr - os=-sysv4 - ;; - netbsd386) - basic_machine=i386-unknown - os=-netbsd - ;; - netwinder) - basic_machine=armv4l-rebel - os=-linux - ;; - news | news700 | news800 | news900) - basic_machine=m68k-sony - os=-newsos - ;; - news1000) - basic_machine=m68030-sony - os=-newsos + basic_machine=`echo "$basic_machine" | sed -e 's/ms1-/mt-/'` ;; news-3600 | risc-news) basic_machine=mips-sony - os=-newsos - ;; - necv70) - basic_machine=v70-nec - os=-sysv + os=newsos ;; - next | m*-next ) + next | m*-next) basic_machine=m68k-next case $os in - -nextstep* ) + nextstep* ) ;; - -ns2*) - os=-nextstep2 + ns2*) + os=nextstep2 ;; *) - os=-nextstep3 + os=nextstep3 ;; esac ;; - nh3000) - basic_machine=m68k-harris - os=-cxux - ;; - nh[45]000) - basic_machine=m88k-harris - os=-cxux - ;; - nindy960) - basic_machine=i960-intel - os=-nindy - ;; - mon960) - basic_machine=i960-intel - os=-mon960 - ;; - nonstopux) - basic_machine=mips-compaq - os=-nonstopux - ;; np1) basic_machine=np1-gould ;; @@ -916,40 +1110,26 @@ case $basic_machine in nsr-tandem) basic_machine=nsr-tandem ;; + nsv-tandem) + basic_machine=nsv-tandem + ;; + nsx-tandem) + basic_machine=nsx-tandem + ;; op50n-* | op60c-*) basic_machine=hppa1.1-oki - os=-proelf + os=proelf ;; openrisc | openrisc-*) basic_machine=or32-unknown ;; - os400) - basic_machine=powerpc-ibm - os=-os400 - ;; - OSE68000 | ose68000) - basic_machine=m68000-ericsson - os=-ose - ;; - os68k) - basic_machine=m68k-none - os=-os68k - ;; pa-hitachi) basic_machine=hppa1.1-hitachi - os=-hiuxwe2 - ;; - paragon) - basic_machine=i860-intel - os=-osf - ;; - parisc) - basic_machine=hppa-unknown - os=-linux + os=hiuxwe2 ;; parisc-*) - basic_machine=hppa-`echo $basic_machine | sed 's/^[^-]*-//'` - os=-linux + basic_machine=hppa-`echo "$basic_machine" | sed 's/^[^-]*-//'` + os=linux ;; pbd) basic_machine=sparc-tti @@ -964,7 +1144,7 @@ case $basic_machine in basic_machine=i386-pc ;; pc98-*) - basic_machine=i386-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=i386-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; pentium | p5 | k5 | k6 | nexgen | viac3) basic_machine=i586-pc @@ -979,16 +1159,16 @@ case $basic_machine in basic_machine=i786-pc ;; pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*) - basic_machine=i586-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=i586-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; pentiumpro-* | p6-* | 6x86-* | athlon-*) - basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=i686-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*) - basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=i686-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; pentium4-*) - basic_machine=i786-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=i786-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; pn) basic_machine=pn-gould @@ -998,39 +1178,27 @@ case $basic_machine in ppc | ppcbe) basic_machine=powerpc-unknown ;; ppc-* | ppcbe-*) - basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=powerpc-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; - ppcle | powerpclittle | ppc-le | powerpc-little) + ppcle | powerpclittle) basic_machine=powerpcle-unknown ;; ppcle-* | powerpclittle-*) - basic_machine=powerpcle-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=powerpcle-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; ppc64) basic_machine=powerpc64-unknown ;; - ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'` + ppc64-*) basic_machine=powerpc64-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; - ppc64le | powerpc64little | ppc64-le | powerpc64-little) + ppc64le | powerpc64little) basic_machine=powerpc64le-unknown ;; ppc64le-* | powerpc64little-*) - basic_machine=powerpc64le-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=powerpc64le-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; ps2) basic_machine=i386-ibm ;; - pw32) - basic_machine=i586-unknown - os=-pw32 - ;; - rdos) - basic_machine=i386-pc - os=-rdos - ;; - rom68k) - basic_machine=m68k-rom68k - os=-coff - ;; rm[46]00) basic_machine=mips-siemens ;; @@ -1043,10 +1211,6 @@ case $basic_machine in s390x | s390x-*) basic_machine=s390x-ibm ;; - sa29200) - basic_machine=a29k-amd - os=-udi - ;; sb1) basic_machine=mipsisa64sb1-unknown ;; @@ -1055,105 +1219,32 @@ case $basic_machine in ;; sde) basic_machine=mipsisa32-sde - os=-elf - ;; - sei) - basic_machine=mips-sei - os=-seiux + os=${os:-elf} ;; sequent) basic_machine=i386-sequent ;; - sh) - basic_machine=sh-hitachi - os=-hms - ;; sh5el) basic_machine=sh5le-unknown ;; - sh64) - basic_machine=sh64-unknown + sh5el-*) + basic_machine=sh5le-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; - sparclite-wrs | simso-wrs) + simso-wrs) basic_machine=sparclite-wrs - os=-vxworks - ;; - sps7) - basic_machine=m68k-bull - os=-sysv2 + os=vxworks ;; spur) basic_machine=spur-unknown ;; - st2000) - basic_machine=m68k-tandem - ;; - stratus) - basic_machine=i860-stratus - os=-sysv4 - ;; strongarm-* | thumb-*) - basic_machine=arm-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - sun2) - basic_machine=m68000-sun - ;; - sun2os3) - basic_machine=m68000-sun - os=-sunos3 - ;; - sun2os4) - basic_machine=m68000-sun - os=-sunos4 + basic_machine=arm-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; - sun3os3) - basic_machine=m68k-sun - os=-sunos3 - ;; - sun3os4) - basic_machine=m68k-sun - os=-sunos4 - ;; - sun4os3) - basic_machine=sparc-sun - os=-sunos3 - ;; - sun4os4) - basic_machine=sparc-sun - os=-sunos4 - ;; - sun4sol2) - basic_machine=sparc-sun - os=-solaris2 - ;; - sun3 | sun3-*) - basic_machine=m68k-sun - ;; - sun4) - basic_machine=sparc-sun - ;; - sun386 | sun386i | roadrunner) - basic_machine=i386-sun - ;; - sv1) - basic_machine=sv1-cray - os=-unicos - ;; - symmetry) - basic_machine=i386-sequent - os=-dynix - ;; - t3e) - basic_machine=alphaev5-cray - os=-unicos - ;; - t90) - basic_machine=t90-cray - os=-unicos + tile*-*) ;; tile*) basic_machine=$basic_machine-unknown - os=-linux-gnu + os=${os:-linux-gnu} ;; tx39) basic_machine=mipstx39-unknown @@ -1161,146 +1252,32 @@ case $basic_machine in tx39el) basic_machine=mipstx39el-unknown ;; - toad1) - basic_machine=pdp10-xkl - os=-tops20 - ;; tower | tower-32) basic_machine=m68k-ncr ;; - tpf) - basic_machine=s390x-ibm - os=-tpf - ;; - udi29k) - basic_machine=a29k-amd - os=-udi - ;; - ultra3) - basic_machine=a29k-nyu - os=-sym1 - ;; - v810 | necv810) - basic_machine=v810-nec - os=-none - ;; - vaxv) - basic_machine=vax-dec - os=-sysv - ;; - vms) - basic_machine=vax-dec - os=-vms - ;; vpp*|vx|vx-*) basic_machine=f301-fujitsu ;; - vxworks960) - basic_machine=i960-wrs - os=-vxworks - ;; - vxworks68) - basic_machine=m68k-wrs - os=-vxworks - ;; - vxworks29k) - basic_machine=a29k-wrs - os=-vxworks - ;; w65*) basic_machine=w65-wdc - os=-none + os=none ;; w89k-*) basic_machine=hppa1.1-winbond - os=-proelf - ;; - xbox) - basic_machine=i686-pc - os=-mingw32 + os=proelf ;; - xps | xps100) - basic_machine=xps100-honeywell + x64) + basic_machine=x86_64-pc ;; xscale-* | xscalee[bl]-*) - basic_machine=`echo $basic_machine | sed 's/^xscale/arm/'` - ;; - ymp) - basic_machine=ymp-cray - os=-unicos - ;; - z8k-*-coff) - basic_machine=z8k-unknown - os=-sim - ;; - z80-*-coff) - basic_machine=z80-unknown - os=-sim + basic_machine=`echo "$basic_machine" | sed 's/^xscale/arm/'` ;; none) basic_machine=none-none - os=-none ;; -# Here we handle the default manufacturer of certain CPU types. It is in -# some cases the only manufacturer, in others, it is the most popular. - w89k) - basic_machine=hppa1.1-winbond - ;; - op50n) - basic_machine=hppa1.1-oki - ;; - op60c) - basic_machine=hppa1.1-oki - ;; - romp) - basic_machine=romp-ibm - ;; - mmix) - basic_machine=mmix-knuth - ;; - rs6000) - basic_machine=rs6000-ibm - ;; - vax) - basic_machine=vax-dec - ;; - pdp10) - # there are many clones, so DEC is not a safe bet - basic_machine=pdp10-unknown - ;; - pdp11) - basic_machine=pdp11-dec - ;; - we32k) - basic_machine=we32k-att - ;; - sh[1234] | sh[24]a | sh[24]aeb | sh[34]eb | sh[1234]le | sh[23]ele) - basic_machine=sh-unknown - ;; - sparc | sparcv8 | sparcv9 | sparcv9b | sparcv9v) - basic_machine=sparc-sun - ;; - cydra) - basic_machine=cydra-cydrome - ;; - orion) - basic_machine=orion-highlevel - ;; - orion105) - basic_machine=clipper-highlevel - ;; - mac | mpw | mac-mpw) - basic_machine=m68k-apple - ;; - pmac | pmac-mpw) - basic_machine=powerpc-apple - ;; - *-unknown) - # Make sure to match an already-canonicalized machine name. - ;; *) - echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2 + echo Invalid configuration \`"$1"\': machine \`"$basic_machine"\' not recognized 1>&2 exit 1 ;; esac @@ -1308,10 +1285,10 @@ esac # Here we canonicalize certain aliases for manufacturers. case $basic_machine in *-digital*) - basic_machine=`echo $basic_machine | sed 's/digital.*/dec/'` + basic_machine=`echo "$basic_machine" | sed 's/digital.*/dec/'` ;; *-commodore*) - basic_machine=`echo $basic_machine | sed 's/commodore.*/cbm/'` + basic_machine=`echo "$basic_machine" | sed 's/commodore.*/cbm/'` ;; *) ;; @@ -1319,200 +1296,246 @@ esac # Decode manufacturer-specific aliases for certain operating systems. -if [ x"$os" != x"" ] +if [ x$os != x ] then case $os in - # First match some system type aliases - # that might get confused with valid system types. - # -solaris* is a basic system type, with this one exception. - -auroraux) - os=-auroraux + # First match some system type aliases that might get confused + # with valid system types. + # solaris* is a basic system type, with this one exception. + auroraux) + os=auroraux ;; - -solaris1 | -solaris1.*) - os=`echo $os | sed -e 's|solaris1|sunos4|'` + bluegene*) + os=cnk ;; - -solaris) - os=-solaris2 + solaris1 | solaris1.*) + os=`echo $os | sed -e 's|solaris1|sunos4|'` ;; - -svr4*) - os=-sysv4 + solaris) + os=solaris2 ;; - -unixware*) - os=-sysv4.2uw + unixware*) + os=sysv4.2uw ;; - -gnu/linux*) + gnu/linux*) os=`echo $os | sed -e 's|gnu/linux|linux-gnu|'` ;; - # First accept the basic system types. + # es1800 is here to avoid being matched by es* (a different OS) + es1800*) + os=ose + ;; + # Some version numbers need modification + chorusos*) + os=chorusos + ;; + isc) + os=isc2.2 + ;; + sco6) + os=sco5v6 + ;; + sco5) + os=sco3.2v5 + ;; + sco4) + os=sco3.2v4 + ;; + sco3.2.[4-9]*) + os=`echo $os | sed -e 's/sco3.2./sco3.2v/'` + ;; + sco3.2v[4-9]* | sco5v6*) + # Don't forget version if it is 3.2v4 or newer. + ;; + scout) + # Don't match below + ;; + sco*) + os=sco3.2v2 + ;; + psos*) + os=psos + ;; + # Now accept the basic system types. # The portable systems comes first. - # Each alternative MUST END IN A *, to match a version number. - # -sysv* is not here because it comes later, after sysvr4. - -gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \ - | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\ - | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \ - | -sym* | -kopensolaris* \ - | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \ - | -aos* | -aros* \ - | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \ - | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \ - | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \ - | -bitrig* | -openbsd* | -solidbsd* \ - | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \ - | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \ - | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \ - | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \ - | -chorusos* | -chorusrdb* | -cegcc* \ - | -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \ - | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \ - | -linux-newlib* | -linux-musl* | -linux-uclibc* \ - | -uxpv* | -beos* | -mpeix* | -udk* \ - | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \ - | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \ - | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \ - | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \ - | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \ - | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \ - | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es*) + # Each alternative MUST end in a * to match a version number. + # sysv* is not here because it comes later, after sysvr4. + gnu* | bsd* | mach* | minix* | genix* | ultrix* | irix* \ + | *vms* | esix* | aix* | cnk* | sunos | sunos[34]*\ + | hpux* | unos* | osf* | luna* | dgux* | auroraux* | solaris* \ + | sym* | kopensolaris* | plan9* \ + | amigaos* | amigados* | msdos* | newsos* | unicos* | aof* \ + | aos* | aros* | cloudabi* | sortix* \ + | nindy* | vxsim* | vxworks* | ebmon* | hms* | mvs* \ + | clix* | riscos* | uniplus* | iris* | isc* | rtu* | xenix* \ + | knetbsd* | mirbsd* | netbsd* \ + | bitrig* | openbsd* | solidbsd* | libertybsd* \ + | ekkobsd* | kfreebsd* | freebsd* | riscix* | lynxos* \ + | bosx* | nextstep* | cxux* | aout* | elf* | oabi* \ + | ptx* | coff* | ecoff* | winnt* | domain* | vsta* \ + | udi* | eabi* | lites* | ieee* | go32* | aux* | hcos* \ + | chorusrdb* | cegcc* | glidix* \ + | cygwin* | msys* | pe* | moss* | proelf* | rtems* \ + | midipix* | mingw32* | mingw64* | linux-gnu* | linux-android* \ + | linux-newlib* | linux-musl* | linux-uclibc* \ + | uxpv* | beos* | mpeix* | udk* | moxiebox* \ + | interix* | uwin* | mks* | rhapsody* | darwin* \ + | openstep* | oskit* | conix* | pw32* | nonstopux* \ + | storm-chaos* | tops10* | tenex* | tops20* | its* \ + | os2* | vos* | palmos* | uclinux* | nucleus* \ + | morphos* | superux* | rtmk* | windiss* \ + | powermax* | dnix* | nx6 | nx7 | sei* | dragonfly* \ + | skyos* | haiku* | rdos* | toppers* | drops* | es* \ + | onefs* | tirtos* | phoenix* | fuchsia* | redox* | bme* \ + | midnightbsd*) # Remember, each alternative MUST END IN *, to match a version number. ;; - -qnx*) + qnx*) case $basic_machine in x86-* | i*86-*) ;; *) - os=-nto$os + os=nto-$os ;; esac ;; - -nto-qnx*) + hiux*) + os=hiuxwe2 ;; - -nto*) - os=`echo $os | sed -e 's|nto|nto-qnx|'` + nto-qnx*) ;; - -sim | -es1800* | -hms* | -xray | -os68k* | -none* | -v88r* \ - | -windows* | -osx | -abug | -netware* | -os9* | -beos* | -haiku* \ - | -macos* | -mpw* | -magic* | -mmixware* | -mon960* | -lnews*) + nto*) + os=`echo $os | sed -e 's|nto|nto-qnx|'` ;; - -mac*) - os=`echo $os | sed -e 's|mac|macos|'` + sim | xray | os68k* | v88r* \ + | windows* | osx | abug | netware* | os9* \ + | macos* | mpw* | magic* | mmixware* | mon960* | lnews*) ;; - -linux-dietlibc) - os=-linux-dietlibc + linux-dietlibc) + os=linux-dietlibc ;; - -linux*) + linux*) os=`echo $os | sed -e 's|linux|linux-gnu|'` ;; - -sunos5*) - os=`echo $os | sed -e 's|sunos5|solaris2|'` + lynx*178) + os=lynxos178 ;; - -sunos6*) - os=`echo $os | sed -e 's|sunos6|solaris3|'` + lynx*5) + os=lynxos5 ;; - -opened*) - os=-openedition + lynx*) + os=lynxos ;; - -os400*) - os=-os400 + mac*) + os=`echo "$os" | sed -e 's|mac|macos|'` ;; - -wince*) - os=-wince + opened*) + os=openedition ;; - -osfrose*) - os=-osfrose + os400*) + os=os400 ;; - -osf*) - os=-osf + sunos5*) + os=`echo "$os" | sed -e 's|sunos5|solaris2|'` ;; - -utek*) - os=-bsd + sunos6*) + os=`echo "$os" | sed -e 's|sunos6|solaris3|'` ;; - -dynix*) - os=-bsd + wince*) + os=wince ;; - -acis*) - os=-aos + utek*) + os=bsd ;; - -atheos*) - os=-atheos + dynix*) + os=bsd ;; - -syllable*) - os=-syllable + acis*) + os=aos ;; - -386bsd) - os=-bsd + atheos*) + os=atheos ;; - -ctix* | -uts*) - os=-sysv + syllable*) + os=syllable ;; - -nova*) - os=-rtmk-nova + 386bsd) + os=bsd + ;; + ctix* | uts*) + os=sysv + ;; + nova*) + os=rtmk-nova ;; - -ns2 ) - os=-nextstep2 + ns2) + os=nextstep2 ;; - -nsk*) - os=-nsk + nsk*) + os=nsk ;; # Preserve the version number of sinix5. - -sinix5.*) + sinix5.*) os=`echo $os | sed -e 's|sinix|sysv|'` ;; - -sinix*) - os=-sysv4 + sinix*) + os=sysv4 ;; - -tpf*) - os=-tpf + tpf*) + os=tpf ;; - -triton*) - os=-sysv3 + triton*) + os=sysv3 ;; - -oss*) - os=-sysv3 + oss*) + os=sysv3 ;; - -svr4) - os=-sysv4 + svr4*) + os=sysv4 ;; - -svr3) - os=-sysv3 + svr3) + os=sysv3 ;; - -sysvr4) - os=-sysv4 + sysvr4) + os=sysv4 ;; - # This must come after -sysvr4. - -sysv*) + # This must come after sysvr4. + sysv*) ;; - -ose*) - os=-ose + ose*) + os=ose ;; - -es1800*) - os=-ose + *mint | mint[0-9]* | *MiNT | MiNT[0-9]*) + os=mint ;; - -xenix) - os=-xenix + zvmoe) + os=zvmoe ;; - -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*) - os=-mint + dicos*) + os=dicos ;; - -aros*) - os=-aros - ;; - -kaos*) - os=-kaos + pikeos*) + # Until real need of OS specific support for + # particular features comes up, bare metal + # configurations are quite functional. + case $basic_machine in + arm*) + os=eabi + ;; + *) + os=elf + ;; + esac ;; - -zvmoe) - os=-zvmoe + nacl*) ;; - -dicos*) - os=-dicos + ios) ;; - -nacl*) + none) ;; - -none) + *-eabi) ;; *) - # Get rid of the `-' at the beginning of $os. - os=`echo $os | sed 's/[^-]*-//'` - echo Invalid configuration \`$1\': system \`$os\' not recognized 1>&2 + echo Invalid configuration \`"$1"\': system \`"$os"\' not recognized 1>&2 exit 1 ;; esac @@ -1530,173 +1553,179 @@ else case $basic_machine in score-*) - os=-elf + os=elf ;; spu-*) - os=-elf + os=elf ;; *-acorn) - os=-riscix1.2 + os=riscix1.2 ;; arm*-rebel) - os=-linux + os=linux ;; arm*-semi) - os=-aout + os=aout ;; c4x-* | tic4x-*) - os=-coff + os=coff + ;; + c8051-*) + os=elf + ;; + clipper-intergraph) + os=clix ;; hexagon-*) - os=-elf + os=elf ;; tic54x-*) - os=-coff + os=coff ;; tic55x-*) - os=-coff + os=coff ;; tic6x-*) - os=-coff + os=coff ;; # This must come before the *-dec entry. pdp10-*) - os=-tops20 + os=tops20 ;; pdp11-*) - os=-none + os=none ;; *-dec | vax-*) - os=-ultrix4.2 + os=ultrix4.2 ;; m68*-apollo) - os=-domain + os=domain ;; i386-sun) - os=-sunos4.0.2 + os=sunos4.0.2 ;; m68000-sun) - os=-sunos3 + os=sunos3 ;; m68*-cisco) - os=-aout + os=aout ;; mep-*) - os=-elf + os=elf ;; mips*-cisco) - os=-elf + os=elf ;; mips*-*) - os=-elf + os=elf ;; or32-*) - os=-coff + os=coff ;; *-tti) # must be before sparc entry or we get the wrong os. - os=-sysv3 + os=sysv3 ;; sparc-* | *-sun) - os=-sunos4.1.1 + os=sunos4.1.1 ;; - *-be) - os=-beos + pru-*) + os=elf ;; - *-haiku) - os=-haiku + *-be) + os=beos ;; *-ibm) - os=-aix + os=aix ;; *-knuth) - os=-mmixware + os=mmixware ;; *-wec) - os=-proelf + os=proelf ;; *-winbond) - os=-proelf + os=proelf ;; *-oki) - os=-proelf + os=proelf ;; *-hp) - os=-hpux + os=hpux ;; *-hitachi) - os=-hiux + os=hiux ;; i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent) - os=-sysv + os=sysv ;; *-cbm) - os=-amigaos + os=amigaos ;; *-dg) - os=-dgux + os=dgux ;; *-dolphin) - os=-sysv3 + os=sysv3 ;; m68k-ccur) - os=-rtu + os=rtu ;; m88k-omron*) - os=-luna + os=luna ;; - *-next ) - os=-nextstep + *-next) + os=nextstep ;; *-sequent) - os=-ptx + os=ptx ;; *-crds) - os=-unos + os=unos ;; *-ns) - os=-genix + os=genix ;; i370-*) - os=-mvs - ;; - *-next) - os=-nextstep3 + os=mvs ;; *-gould) - os=-sysv + os=sysv ;; *-highlevel) - os=-bsd + os=bsd ;; *-encore) - os=-bsd + os=bsd ;; *-sgi) - os=-irix + os=irix ;; *-siemens) - os=-sysv4 + os=sysv4 ;; *-masscomp) - os=-rtu + os=rtu ;; f30[01]-fujitsu | f700-fujitsu) - os=-uxpv + os=uxpv ;; *-rom68k) - os=-coff + os=coff ;; *-*bug) - os=-coff + os=coff ;; *-apple) - os=-macos + os=macos ;; *-atari*) - os=-mint + os=mint + ;; + *-wrs) + os=vxworks ;; *) - os=-none + os=none ;; esac fi @@ -1707,79 +1736,82 @@ vendor=unknown case $basic_machine in *-unknown) case $os in - -riscix*) + riscix*) vendor=acorn ;; - -sunos*) + sunos*) vendor=sun ;; - -cnk*|-aix*) + cnk*|-aix*) vendor=ibm ;; - -beos*) + beos*) vendor=be ;; - -hpux*) + hpux*) vendor=hp ;; - -mpeix*) + mpeix*) vendor=hp ;; - -hiux*) + hiux*) vendor=hitachi ;; - -unos*) + unos*) vendor=crds ;; - -dgux*) + dgux*) vendor=dg ;; - -luna*) + luna*) vendor=omron ;; - -genix*) + genix*) vendor=ns ;; - -mvs* | -opened*) + clix*) + vendor=intergraph + ;; + mvs* | opened*) vendor=ibm ;; - -os400*) + os400*) vendor=ibm ;; - -ptx*) + ptx*) vendor=sequent ;; - -tpf*) + tpf*) vendor=ibm ;; - -vxsim* | -vxworks* | -windiss*) + vxsim* | vxworks* | windiss*) vendor=wrs ;; - -aux*) + aux*) vendor=apple ;; - -hms*) + hms*) vendor=hitachi ;; - -mpw* | -macos*) + mpw* | macos*) vendor=apple ;; - -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*) + *mint | mint[0-9]* | *MiNT | MiNT[0-9]*) vendor=atari ;; - -vos*) + vos*) vendor=stratus ;; esac - basic_machine=`echo $basic_machine | sed "s/unknown/$vendor/"` + basic_machine=`echo "$basic_machine" | sed "s/unknown/$vendor/"` ;; esac -echo $basic_machine$os +echo "$basic_machine-$os" exit # Local variables: -# eval: (add-hook 'write-file-hooks 'time-stamp) +# eval: (add-hook 'before-save-hook 'time-stamp) # time-stamp-start: "timestamp='" # time-stamp-format: "%:y-%02m-%02d" # time-stamp-end: "'" diff --git a/configure.ac b/configure.ac index e6b11be2df1..c3007b4b6b4 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,5 @@ -AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./version)$VERSION_SUFFIX"])) +AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./.version)$VERSION_SUFFIX"])) +AC_CONFIG_MACRO_DIRS([m4]) AC_CONFIG_SRCDIR(README.md) AC_CONFIG_AUX_DIR(config) @@ -42,32 +43,33 @@ esac AC_MSG_RESULT($system) AC_SUBST(system) -AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier (`cpu-os')]) +AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')]) # State should be stored in /nix/var, unless the user overrides it explicitly. test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var -# Solaris-specific stuff. -AC_STRUCT_DIRENT_D_TYPE -if test "$sys_name" = sunos; then - # Solaris requires -lsocket -lnsl for network functions - LIBS="-lsocket -lnsl $LIBS" -fi - - CFLAGS= CXXFLAGS= AC_PROG_CC AC_PROG_CXX -AX_CXX_COMPILE_STDCXX_11 +AC_PROG_CPP +AC_CHECK_TOOL([AR], [ar]) # Use 64-bit file system calls so that we can support files > 2 GiB. AC_SYS_LARGEFILE +# Solaris-specific stuff. +AC_STRUCT_DIRENT_D_TYPE +if test "$sys_name" = sunos; then + # Solaris requires -lsocket -lnsl for network functions + LIBS="-lsocket -lnsl $LIBS" +fi + + # Check for pubsetbuf. AC_MSG_CHECKING([for pubsetbuf]) AC_LANG_PUSH(C++) @@ -114,46 +116,16 @@ if test -z "$$1"; then fi ]) -NEED_PROG(curl, curl) NEED_PROG(bash, bash) -NEED_PROG(patch, patch) AC_PATH_PROG(xmllint, xmllint, false) AC_PATH_PROG(xsltproc, xsltproc, false) AC_PATH_PROG(flex, flex, false) AC_PATH_PROG(bison, bison, false) -NEED_PROG(perl, perl) -NEED_PROG(sed, sed) -NEED_PROG(tar, tar) -NEED_PROG(bzip2, bzip2) -NEED_PROG(gzip, gzip) -NEED_PROG(xz, xz) AC_PATH_PROG(dot, dot) -AC_PATH_PROG(pv, pv, pv) - - -# Test that Perl has the open/fork feature (Perl 5.8.0 and beyond). -AC_MSG_CHECKING([whether Perl is recent enough]) -if ! $perl -e 'open(FOO, "-|", "true"); while () { print; }; close FOO or die;'; then - AC_MSG_RESULT(no) - AC_MSG_ERROR([Your Perl version is too old. Nix requires Perl 5.8.0 or newer.]) -fi -AC_MSG_RESULT(yes) - - -# Figure out where to install Perl modules. -AC_MSG_CHECKING([for the Perl installation prefix]) -perlversion=$($perl -e 'use Config; print $Config{version};') -perlarchname=$($perl -e 'use Config; print $Config{archname};') -AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname]) -AC_MSG_RESULT($perllibdir) +AC_PATH_PROG(lsof, lsof, lsof) -NEED_PROG(cat, cat) -NEED_PROG(tr, tr) -AC_ARG_WITH(coreutils-bin, AC_HELP_STRING([--with-coreutils-bin=PATH], - [path of cat, mkdir, etc.]), - coreutils=$withval, coreutils=$(dirname $cat)) -AC_SUBST(coreutils) +AC_SUBST(coreutils, [$(dirname $(type -p cat))]) AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH], @@ -162,24 +134,70 @@ AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH], AC_SUBST(storedir) -# Look for OpenSSL, a required dependency. +# Look for boost, a required dependency. +# Note that AX_BOOST_BASE only exports *CPP* BOOST_CPPFLAGS, no CXX flags, +# and CPPFLAGS are not passed to the C++ compiler automatically. +# Thus we append the returned CPPFLAGS to the CXXFLAGS here. +AX_BOOST_BASE([1.66], [CXXFLAGS="$BOOST_CPPFLAGS $CXXFLAGS"], [AC_MSG_ERROR([Nix requires boost.])]) +# For unknown reasons, setting this directly in the ACTION-IF-FOUND above +# ends up with LDFLAGS being empty, so we set it afterwards. +LDFLAGS="$BOOST_LDFLAGS $LDFLAGS" + +# On some platforms, new-style atomics need a helper library +AC_MSG_CHECKING(whether -latomic is needed) +AC_LINK_IFELSE([AC_LANG_SOURCE([[ +#include +uint64_t v; +int main() { + return (int)__atomic_load_n(&v, __ATOMIC_ACQUIRE); +}]])], GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=no, GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=yes) +AC_MSG_RESULT($GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC) +if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then + LIBS="-latomic $LIBS" +fi + +PKG_PROG_PKG_CONFIG + +AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared], + [Build shared libraries for Nix [default=yes]]), + shared=$enableval, shared=yes) +if test "$shared" = yes; then + AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.]) +else + AC_SUBST(BUILD_SHARED_LIBS, 0, [Whether to build shared libraries.]) + PKG_CONFIG="$PKG_CONFIG --static" +fi + +# Look for OpenSSL, a required dependency. FIXME: this is only (maybe) +# used by S3BinaryCacheStore. PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) # Look for libbz2, a required dependency. AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true], - [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See http://www.bzip.org/.])]) + [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])]) AC_CHECK_HEADERS([bzlib.h], [true], - [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See http://www.bzip.org/.])]) - + [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])]) +# Checks for libarchive +PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"]) # Look for SQLite, a required dependency. PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CXXFLAGS"]) - # Look for libcurl, a required dependency. PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"]) +# Look for editline, a required dependency. +# The the libeditline.pc file was added only in libeditline >= 1.15.2, +# see https://github.com/troglobit/editline/commit/0a8f2ef4203c3a4a4726b9dd1336869cd0da8607, +# but e.g. Ubuntu 16.04 has an older version, so we fall back to searching for +# editline.h when the pkg-config approach fails. +PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"], [ + AC_CHECK_HEADERS([editline.h], [true], + [AC_MSG_ERROR([Nix requires libeditline; it was found neither via pkg-config nor its normal header.])]) + AC_SEARCH_LIBS([readline read_history], [editline], [], + [AC_MSG_ERROR([Nix requires libeditline; it was not found via pkg-config, but via its header, but required functions do not work. Maybe it is too old? >= 1.14 is required.])]) +]) # Look for libsodium, an optional dependency. PKG_CHECK_MODULES([SODIUM], [libsodium], @@ -188,24 +206,59 @@ PKG_CHECK_MODULES([SODIUM], [libsodium], have_sodium=1], [have_sodium=]) AC_SUBST(HAVE_SODIUM, [$have_sodium]) - # Look for liblzma, a required dependency. PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"]) +AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt], + [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])]) + +# Look for zlib, a required dependency. +PKG_CHECK_MODULES([ZLIB], [zlib], [CXXFLAGS="$ZLIB_CFLAGS $CXXFLAGS"]) +AC_CHECK_HEADER([zlib.h],[:],[AC_MSG_ERROR([could not find the zlib.h header])]) +LDFLAGS="-lz $LDFLAGS" + +# Look for libbrotli{enc,dec}. +PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"]) + + +# Look for libseccomp, required for Linux sandboxing. +if test "$sys_name" = linux; then + AC_ARG_ENABLE([seccomp-sandboxing], + AC_HELP_STRING([--disable-seccomp-sandboxing], + [Don't build support for seccomp sandboxing (only recommended if your arch doesn't support libseccomp yet!)] + )) + if test "x$enable_seccomp_sandboxing" != "xno"; then + PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp], + [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"]) + have_seccomp=1 + AC_DEFINE([HAVE_SECCOMP], [1], [Whether seccomp is available and should be used for sandboxing.]) + else + have_seccomp= + fi +else + have_seccomp= +fi +AC_SUBST(HAVE_SECCOMP, [$have_seccomp]) # Look for aws-cpp-sdk-s3. AC_LANG_PUSH(C++) AC_CHECK_HEADERS([aws/s3/S3Client.h], - [AC_DEFINE([ENABLE_S3], [1], [Whether to enable S3 support via aws-cpp-sdk-s3.]) + [AC_DEFINE([ENABLE_S3], [1], [Whether to enable S3 support via aws-sdk-cpp.]) enable_s3=1], [enable_s3=]) AC_SUBST(ENABLE_S3, [$enable_s3]) AC_LANG_POP(C++) +if test -n "$enable_s3"; then + declare -a aws_version_tokens=($(printf '#include \nAWS_SDK_VERSION_STRING' | $CPP $CPPFLAGS - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' ')) + AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_version_tokens@<:@0@:>@}, [Major version of aws-sdk-cpp.]) + AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_version_tokens@<:@1@:>@}, [Minor version of aws-sdk-cpp.]) +fi + # Whether to use the Boehm garbage collector. AC_ARG_ENABLE(gc, AC_HELP_STRING([--enable-gc], - [enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=no]]), - gc=$enableval, gc=no) + [enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=yes]]), + gc=$enableval, gc=yes) if test "$gc" = yes; then PKG_CHECK_MODULES([BDW_GC], [bdw-gc]) CXXFLAGS="$BDW_GC_CFLAGS $CXXFLAGS" @@ -213,44 +266,8 @@ if test "$gc" = yes; then fi -# Check for the required Perl dependencies (DBI, DBD::SQLite). -perlFlags="-I$perllibdir" - -AC_ARG_WITH(dbi, AC_HELP_STRING([--with-dbi=PATH], - [prefix of the Perl DBI library]), - perlFlags="$perlFlags -I$withval") - -AC_ARG_WITH(dbd-sqlite, AC_HELP_STRING([--with-dbd-sqlite=PATH], - [prefix of the Perl DBD::SQLite library]), - perlFlags="$perlFlags -I$withval") - -AC_MSG_CHECKING([whether DBD::SQLite works]) -if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then - AC_MSG_RESULT(no) - AC_MSG_FAILURE([The Perl modules DBI and/or DBD::SQLite are missing.]) -fi -AC_MSG_RESULT(yes) - -AC_SUBST(perlFlags) - - -# Whether to build the Perl bindings -AC_MSG_CHECKING([whether to build the Perl bindings]) -AC_ARG_ENABLE(perl-bindings, AC_HELP_STRING([--enable-perl-bindings], - [whether to build the Perl bindings (recommended) [default=yes]]), - perlbindings=$enableval, perlbindings=yes) -if test "$enable_shared" = no; then - # Perl bindings require shared libraries. - perlbindings=no -fi -AC_SUBST(perlbindings) -AC_MSG_RESULT($perlbindings) - - -AC_ARG_ENABLE(init-state, AC_HELP_STRING([--disable-init-state], - [do not initialise DB etc. in `make install']), - init_state=$enableval, init_state=yes) -#AM_CONDITIONAL(INIT_STATE, test "$init_state" = "yes") +# Look for gtest. +PKG_CHECK_MODULES([GTEST], [gtest_main]) # documentation generation switch @@ -265,7 +282,7 @@ AC_CHECK_FUNCS([setresuid setreuid lchown]) # Nice to have, but not essential. -AC_CHECK_FUNCS([strsignal posix_fallocate nanosleep sysconf]) +AC_CHECK_FUNCS([strsignal posix_fallocate sysconf]) # This is needed if bzip2 is a static library, and the Nix libraries @@ -275,11 +292,6 @@ if test "$(uname)" = "Darwin"; then fi -# Figure out the extension of dynamic libraries. -eval dynlib_suffix=$shrext_cmds -AC_SUBST(dynlib_suffix) - - # Do we have GNU tar? AC_MSG_CHECKING([if you have a recent GNU tar]) if $tar --version 2> /dev/null | grep -q GNU && tar cvf /dev/null --warning=no-timestamp ./config.log > /dev/null; then @@ -291,6 +303,11 @@ fi AC_SUBST(tarFlags) +AC_ARG_WITH(sandbox-shell, AC_HELP_STRING([--with-sandbox-shell=PATH], + [path of a statically-linked shell to use as /bin/sh in sandboxes]), + sandbox_shell=$withval) +AC_SUBST(sandbox_shell) + # Expand all variables in config.status. test "$prefix" = NONE && prefix=$ac_default_prefix test "$exec_prefix" = NONE && exec_prefix='${prefix}' diff --git a/contrib/stack-collapse.py b/contrib/stack-collapse.py new file mode 100755 index 00000000000..f5602c95c48 --- /dev/null +++ b/contrib/stack-collapse.py @@ -0,0 +1,38 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i python3 -p python3 --pure + +# To be used with `--trace-function-calls` and `flamegraph.pl`. +# +# For example: +# +# nix-instantiate --trace-function-calls '' -A hello 2> nix-function-calls.trace +# ./contrib/stack-collapse.py nix-function-calls.trace > nix-function-calls.folded +# nix-shell -p flamegraph --run "flamegraph.pl nix-function-calls.folded > nix-function-calls.svg" + +import sys +from pprint import pprint +import fileinput + +stack = [] +timestack = [] + +for line in fileinput.input(): + components = line.strip().split(" ", 2) + if components[0] != "function-trace": + continue + + direction = components[1] + components = components[2].rsplit(" ", 2) + + loc = components[0] + _at = components[1] + time = int(components[2]) + + if direction == "entered": + stack.append(loc) + timestack.append(time) + elif direction == "exited": + dur = time - timestack.pop() + vst = ";".join(stack) + print(f"{vst} {dur}") + stack.pop() diff --git a/corepkgs/buildenv.nix b/corepkgs/buildenv.nix deleted file mode 100644 index 5e7b40eaa0c..00000000000 --- a/corepkgs/buildenv.nix +++ /dev/null @@ -1,44 +0,0 @@ -with import ; - -{ derivations, manifest }: - -derivation { - name = "user-environment"; - system = builtins.currentSystem; - builder = nixLibexecDir + "/nix/buildenv"; - - inherit manifest; - - # !!! grmbl, need structured data for passing this in a clean way. - derivations = - map (d: - [ (d.meta.active or "true") - (d.meta.priority or 5) - (builtins.length d.outputs) - ] ++ map (output: builtins.getAttr output d) d.outputs) - derivations; - - # Building user environments remotely just causes huge amounts of - # network traffic, so don't do that. - preferLocalBuild = true; - - # Also don't bother substituting. - allowSubstitutes = false; - - __sandboxProfile = '' - (allow sysctl-read) - (allow file-read* - (literal "/usr/lib/libSystem.dylib") - (literal "/usr/lib/libSystem.B.dylib") - (literal "/usr/lib/libobjc.A.dylib") - (literal "/usr/lib/libobjc.dylib") - (literal "/usr/lib/libauto.dylib") - (literal "/usr/lib/libc++abi.dylib") - (literal "/usr/lib/libc++.1.dylib") - (literal "/usr/lib/libDiagnosticMessagesClient.dylib") - (subpath "/usr/lib/system") - (subpath "/dev")) - ''; - - inherit chrootDeps; -} diff --git a/corepkgs/config.nix.in b/corepkgs/config.nix.in index f0f4890a32f..cb994594420 100644 --- a/corepkgs/config.nix.in +++ b/corepkgs/config.nix.in @@ -1,26 +1,13 @@ +# FIXME: remove this file? let fromEnv = var: def: let val = builtins.getEnv var; in if val != "" then val else def; in rec { - shell = "@bash@"; - coreutils = "@coreutils@"; - bzip2 = "@bzip2@"; - gzip = "@gzip@"; - xz = "@xz@"; - tar = "@tar@"; - tarFlags = "@tarFlags@"; - tr = "@tr@"; nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@"; nixPrefix = "@prefix@"; nixLibexecDir = fromEnv "NIX_LIBEXEC_DIR" "@libexecdir@"; - - # If Nix is installed in the Nix store, then automatically add it as - # a dependency to the core packages. This ensures that they work - # properly in a chroot. - chrootDeps = - if dirOf nixPrefix == builtins.storeDir then - [ (builtins.storePath nixPrefix) ] - else - [ ]; + nixLocalstateDir = "@localstatedir@"; + nixSysconfDir = "@sysconfdir@"; + nixStoreDir = fromEnv "NIX_STORE_DIR" "@storedir@"; } diff --git a/corepkgs/fetchurl.nix b/corepkgs/fetchurl.nix index 042705b1abb..a84777f5744 100644 --- a/corepkgs/fetchurl.nix +++ b/corepkgs/fetchurl.nix @@ -1,27 +1,29 @@ -{ system ? builtins.currentSystem +{ system ? "" # obsolete , url -, outputHash ? "" -, outputHashAlgo ? "" -, md5 ? "", sha1 ? "", sha256 ? "" +, hash ? "" # an SRI ash + +# Legacy hash specification +, md5 ? "", sha1 ? "", sha256 ? "", sha512 ? "" +, outputHash ? + if hash != "" then hash else if sha512 != "" then sha512 else if sha1 != "" then sha1 else if md5 != "" then md5 else sha256 +, outputHashAlgo ? + if hash != "" then "" else if sha512 != "" then "sha512" else if sha1 != "" then "sha1" else if md5 != "" then "md5" else "sha256" + , executable ? false , unpack ? false , name ? baseNameOf (toString url) }: -assert (outputHash != "" && outputHashAlgo != "") - || md5 != "" || sha1 != "" || sha256 != ""; - derivation { builder = "builtin:fetchurl"; # New-style output content requirements. - outputHashAlgo = if outputHashAlgo != "" then outputHashAlgo else - if sha256 != "" then "sha256" else if sha1 != "" then "sha1" else "md5"; - outputHash = if outputHash != "" then outputHash else - if sha256 != "" then sha256 else if sha1 != "" then sha1 else md5; + inherit outputHashAlgo outputHash; outputHashMode = if unpack || executable then "recursive" else "flat"; - inherit name system url executable unpack; + inherit name url executable unpack; + + system = "builtin"; # No need to double the amount of network traffic preferLocalBuild = true; diff --git a/corepkgs/local.mk b/corepkgs/local.mk index 362c8eb612e..2c72d3a3199 100644 --- a/corepkgs/local.mk +++ b/corepkgs/local.mk @@ -1,4 +1,7 @@ -corepkgs_FILES = buildenv.nix unpack-channel.nix derivation.nix fetchurl.nix imported-drv-to-derivation.nix +corepkgs_FILES = \ + unpack-channel.nix \ + derivation.nix \ + fetchurl.nix $(foreach file,config.nix $(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs))) diff --git a/corepkgs/unpack-channel.nix b/corepkgs/unpack-channel.nix index 9445532ded0..10515bc8b91 100644 --- a/corepkgs/unpack-channel.nix +++ b/corepkgs/unpack-channel.nix @@ -1,41 +1,12 @@ -with import ; - -let - - builder = builtins.toFile "unpack-channel.sh" - '' - mkdir $out - cd $out - xzpat="\.xz\$" - gzpat="\.gz\$" - if [[ "$src" =~ $xzpat ]]; then - ${xz} -d < $src | ${tar} xf - ${tarFlags} - elif [[ "$src" =~ $gzpat ]]; then - ${gzip} -d < $src | ${tar} xf - ${tarFlags} - else - ${bzip2} -d < $src | ${tar} xf - ${tarFlags} - fi - mv * $out/$channelName - if [ -n "$binaryCacheURL" ]; then - mkdir $out/binary-caches - echo -n "$binaryCacheURL" > $out/binary-caches/$channelName - fi - ''; - -in - -{ name, channelName, src, binaryCacheURL ? "" }: +{ name, channelName, src }: derivation { - system = builtins.currentSystem; - builder = shell; - args = [ "-e" builder ]; - inherit name channelName src binaryCacheURL; + builder = "builtin:unpack-channel"; + + system = "builtin"; - PATH = "${nixBinDir}:${coreutils}"; + inherit name channelName src; # No point in doing this remotely. preferLocalBuild = true; - - inherit chrootDeps; } diff --git a/doc/manual/advanced-topics/advanced-topics.xml b/doc/manual/advanced-topics/advanced-topics.xml index 338aa6f3a23..871b7eb1d37 100644 --- a/doc/manual/advanced-topics/advanced-topics.xml +++ b/doc/manual/advanced-topics/advanced-topics.xml @@ -1,10 +1,14 @@ Advanced Topics + + + diff --git a/doc/manual/advanced-topics/cores-vs-jobs.xml b/doc/manual/advanced-topics/cores-vs-jobs.xml new file mode 100644 index 00000000000..4d58ac7c5e4 --- /dev/null +++ b/doc/manual/advanced-topics/cores-vs-jobs.xml @@ -0,0 +1,121 @@ + + +Tuning Cores and Jobs + +Nix has two relevant settings with regards to how your CPU cores +will be utilized: and +. This chapter will talk about what +they are, how they interact, and their configuration trade-offs. + + + + + + Dictates how many separate derivations will be built at the same + time. If you set this to zero, the local machine will do no + builds. Nix will still substitute from binary caches, and build + remotely if remote builders are configured. + + + + + + Suggests how many cores each derivation should use. Similar to + make -j. + + + + +The setting determines the value of +NIX_BUILD_CORES. NIX_BUILD_CORES is equal +to , unless +equals 0, in which case NIX_BUILD_CORES +will be the total number of cores in the system. + +The maximum number of consumed cores is a simple multiplication, + * NIX_BUILD_CORES. + +The balance on how to set these two independent variables depends +upon each builder's workload and hardware. Here are a few example +scenarios on a machine with 24 cores: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Balancing 24 Build Cores
NIX_BUILD_CORESMaximum ProcessesResult
1242424 + One derivation will be built at a time, each one can use 24 + cores. Undersold if a job can’t use 24 cores. +
46624 + Four derivations will be built at once, each given access to + six cores. +
126672 + 12 derivations will be built at once, each given access to six + cores. This configuration is over-sold. If all 12 derivations + being built simultaneously try to use all six cores, the + machine's performance will be degraded due to extensive context + switching between the 12 builds. +
241124 + 24 derivations can build at the same time, each using a single + core. Never oversold, but derivations which require many cores + will be very slow to compile. +
24024576 + 24 derivations can build at the same time, each using all the + available cores of the machine. Very likely to be oversold, + and very likely to suffer context switches. +
+ +It is up to the derivations' build script to respect +host's requested cores-per-build by following the value of the +NIX_BUILD_CORES environment variable. + + diff --git a/doc/manual/advanced-topics/diff-hook.xml b/doc/manual/advanced-topics/diff-hook.xml new file mode 100644 index 00000000000..fb4bf819f94 --- /dev/null +++ b/doc/manual/advanced-topics/diff-hook.xml @@ -0,0 +1,205 @@ + + +Verifying Build Reproducibility with <option linkend="conf-diff-hook">diff-hook</option> + +Check build reproducibility by running builds multiple times +and comparing their results. + +Specify a program with Nix's to +compare build results when two builds produce different results. Note: +this hook is only executed if the results are not the same, this hook +is not used for determining if the results are the same. + +For purposes of demonstration, we'll use the following Nix file, +deterministic.nix for testing: + + +let + inherit (import <nixpkgs> {}) runCommand; +in { + stable = runCommand "stable" {} '' + touch $out + ''; + + unstable = runCommand "unstable" {} '' + echo $RANDOM > $out + ''; +} + + +Additionally, nix.conf contains: + + +diff-hook = /etc/nix/my-diff-hook +run-diff-hook = true + + +where /etc/nix/my-diff-hook is an executable +file containing: + + +#!/bin/sh +exec >&2 +echo "For derivation $3:" +/run/current-system/sw/bin/diff -r "$1" "$2" + + + + +The diff hook is executed by the same user and group who ran the +build. However, the diff hook does not have write access to the store +path just built. + +
+ + Spot-Checking Build Determinism + + + + Verify a path which already exists in the Nix store by passing + to the build command. + + + If the build passes and is deterministic, Nix will exit with a + status code of 0: + + +$ nix-build ./deterministic.nix -A stable +these derivations will be built: + /nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv +building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'... +/nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable + +$ nix-build ./deterministic.nix -A stable --check +checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'... +/nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable + + + If the build is not deterministic, Nix will exit with a status + code of 1: + + +$ nix-build ./deterministic.nix -A unstable +these derivations will be built: + /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv +building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... +/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable + +$ nix-build ./deterministic.nix -A unstable --check +checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... +error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs + + +In the Nix daemon's log, we will now see: + +For derivation /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv: +1c1 +< 8108 +--- +> 30204 + + + + Using with + will cause Nix to keep the second build's output in a special, + .check path: + + +$ nix-build ./deterministic.nix -A unstable --check --keep-failed +checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... +note: keeping build directory '/tmp/nix-build-unstable.drv-0' +error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs from '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check' + + + In particular, notice the + /nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check + output. Nix has copied the build results to that directory where you + can examine it. + + + <literal>.check</literal> paths are not registered store paths + + Check paths are not protected against garbage collection, + and this path will be deleted on the next garbage collection. + + The path is guaranteed to be alive for the duration of + 's execution, but may be deleted + any time after. + + If the comparison is performed as part of automated tooling, + please use the diff-hook or author your tooling to handle the case + where the build was not deterministic and also a check path does + not exist. + + + + is only usable if the derivation has + been built on the system already. If the derivation has not been + built Nix will fail with the error: + +error: some outputs of '/nix/store/hzi1h60z2qf0nb85iwnpvrai3j2w7rr6-unstable.drv' are not valid, so checking is not possible + + + Run the build without , and then try with + again. + +
+ +
+ + Automatic and Optionally Enforced Determinism Verification + + + + Automatically verify every build at build time by executing the + build multiple times. + + + + Setting and + in your + nix.conf permits the automated verification + of every build Nix performs. + + + + The following configuration will run each build three times, and + will require the build to be deterministic: + + +enforce-determinism = true +repeat = 2 + + + + + Setting to false as in + the following configuration will run the build multiple times, + execute the build hook, but will allow the build to succeed even + if it does not build reproducibly: + + +enforce-determinism = false +repeat = 1 + + + + + An example output of this configuration: + +$ nix-build ./test.nix -A unstable +these derivations will be built: + /nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv +building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)... +building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)... +output '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable' of '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' differs from '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable.check' from previous round +/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable + + +
+ diff --git a/doc/manual/advanced-topics/distributed-builds.xml b/doc/manual/advanced-topics/distributed-builds.xml index f8583700393..9ac4a92cd5b 100644 --- a/doc/manual/advanced-topics/distributed-builds.xml +++ b/doc/manual/advanced-topics/distributed-builds.xml @@ -4,74 +4,110 @@ version="5.0" xml:id='chap-distributed-builds'> -Distributed Builds - -Nix supports distributed builds, where a local Nix installation can -forward Nix builds to other machines over the network. This allows -multiple builds to be performed in parallel (thus improving -performance) and allows Nix to perform multi-platform builds in a -semi-transparent way. For instance, if you perform a build for a -x86_64-darwin on an i686-linux -machine, Nix can automatically forward the build to a -x86_64-darwin machine, if available. - -You can enable distributed builds by setting the environment -variable NIX_BUILD_HOOK to point to a program that Nix -will call whenever it wants to build a derivation. The build hook -(typically a shell or Perl script) can decline the build, in which Nix -will perform it in the usual way if possible, or it can accept it, in -which case it is responsible for somehow getting the inputs of the -build to another machine, doing the build there, and getting the -results back. The details of the build hook protocol are described in -the documentation of the NIX_BUILD_HOOK -variable. - -Remote machine configuration: -<filename>remote-systems.conf</filename> - -nix@mcflurry.labs.cs.uu.nl x86_64-darwin /home/nix/.ssh/id_quarterpounder_auto 2 -nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 1 kvm -nix@itchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 2 -nix@poochie.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 2 kvm perf - - - -Nix ships with a build hook that should be suitable for most -purposes. It uses ssh and -nix-copy-closure to copy the build inputs and -outputs and perform the remote build. To use it, you should set -NIX_BUILD_HOOK to -prefix/libexec/nix/build-remote.pl. -You should also define a list of available build machines and point -the environment variable NIX_REMOTE_SYSTEMS to -it. NIX_REMOTE_SYSTEMS must be an absolute path. An -example configuration is shown in . Each line in the file specifies a machine, with the following -bits of information: +Remote Builds + +Nix supports remote builds, where a local Nix installation can +forward Nix builds to other machines. This allows multiple builds to +be performed in parallel and allows Nix to perform multi-platform +builds in a semi-transparent way. For instance, if you perform a +build for a x86_64-darwin on an +i686-linux machine, Nix can automatically forward +the build to a x86_64-darwin machine, if +available. + +To forward a build to a remote machine, it’s required that the +remote machine is accessible via SSH and that it has Nix +installed. You can test whether connecting to the remote Nix instance +works, e.g. + + +$ nix ping-store --store ssh://mac + + +will try to connect to the machine named mac. It is +possible to specify an SSH identity file as part of the remote store +URI, e.g. + + +$ nix ping-store --store ssh://mac?ssh-key=/home/alice/my-key + + +Since builds should be non-interactive, the key should not have a +passphrase. Alternatively, you can load identities ahead of time into +ssh-agent or gpg-agent. + +If you get the error + + +bash: nix-store: command not found +error: cannot connect to 'mac' + + +then you need to ensure that the PATH of +non-interactive login shells contains Nix. + +If you are building via the Nix daemon, it is the Nix +daemon user account (that is, root) that should +have SSH access to the remote machine. If you can’t or don’t want to +configure root to be able to access to remote +machine, you can use a private Nix store instead by passing +e.g. --store ~/my-nix. + +The list of remote machines can be specified on the command line +or in the Nix configuration file. The former is convenient for +testing. For example, the following command allows you to build a +derivation for x86_64-darwin on a Linux machine: + + +$ uname +Linux + +$ nix build \ + '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \ + --builders 'ssh://mac x86_64-darwin' +[1/0/1 built, 0.0 MiB DL] building foo on ssh://mac + +$ cat ./result +Darwin + + +It is possible to specify multiple builders separated by a semicolon +or a newline, e.g. + + + --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd' + + + +Each machine specification consists of the following elements, +separated by spaces. Only the first element is required. +To leave a field at its default, set it to -. - The name of the remote machine, with optionally the - user under which the remote build should be performed. This is - actually passed as an argument to ssh, so it can - be an alias defined in your + The URI of the remote store in the format + ssh://[username@]hostname, + e.g. ssh://nix@mac or + ssh://mac. For backward compatibility, + ssh:// may be omitted. The hostname may be an + alias defined in your ~/.ssh/config. A comma-separated list of Nix platform type identifiers, such as x86_64-darwin. It is possible for a machine to support multiple platform types, e.g., - i686-linux,x86_64-linux. + i686-linux,x86_64-linux. If omitted, this + defaults to the local platform type. - The SSH private key to be used to log in to the - remote machine. Since builds should be non-interactive, this key - should not have a passphrase! + The SSH identity file to be used to log in to the + remote machine. If omitted, SSH will use its regular + identities. - The maximum number of builds that - build-remote.pl will execute in parallel on the - machine. Typically this should be equal to the number of CPU cores. - For instance, the machine itchy in the example - will execute up to 8 builds in parallel. + The maximum number of builds that Nix will execute + in parallel on the machine. Typically this should be equal to the + number of CPU cores. For instance, the machine + itchy in the example will execute up to 8 builds + in parallel. The “speed factor”, indicating the relative speed of the machine. If there are multiple machines of the right type, Nix @@ -79,38 +115,76 @@ bits of information: A comma-separated list of supported features. If a derivation has the - requiredSystemFeatures attribute, then - build-remote.pl will only perform the - derivation on a machine that has the specified features. For - instance, the attribute + requiredSystemFeatures attribute, then Nix will + only perform the derivation on a machine that has the specified + features. For instance, the attribute requiredSystemFeatures = [ "kvm" ]; will cause the build to be performed on a machine that has the - kvm feature (i.e., scratchy in - the example above). + kvm feature. A comma-separated list of mandatory features. A machine will only be used to build a derivation if all of the machine’s mandatory features appear in the - derivation’s requiredSystemFeatures attribute. - Thus, in the example, the machine poochie will - only do derivations that have - requiredSystemFeatures set to ["kvm" - "perf"] or ["perf"]. + derivation’s requiredSystemFeatures + attribute.. -You should also set up the environment variable -NIX_CURRENT_LOAD to point at a directory (e.g., -/var/run/nix/current-load) that -build-remote.pl uses to remember how many builds -it is currently executing remotely. It doesn't look at the actual -load on the remote machine, so if you have multiple instances of Nix -running, they should use the same NIX_CURRENT_LOAD -file. Maybe in the future build-remote.pl will -look at the actual remote load. +For example, the machine specification + + +nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 1 kvm +nix@itchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 2 +nix@poochie.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 1 2 kvm benchmark + + +specifies several machines that can perform +i686-linux builds. However, +poochie will only do builds that have the attribute + + +requiredSystemFeatures = [ "benchmark" ]; + + +or + + +requiredSystemFeatures = [ "benchmark" "kvm" ]; + + +itchy cannot do builds that require +kvm, but scratchy does support +such builds. For regular builds, itchy will be +preferred over scratchy because it has a higher +speed factor. + +Remote builders can also be configured in +nix.conf, e.g. + + +builders = ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd + + +Finally, remote builders can be configured in a separate configuration +file included in via the syntax +@file. For example, + + +builders = @/etc/nix/machines + + +causes the list of machines in /etc/nix/machines +to be included. (This is the default.) + +If you want the builders to use caches, you likely want to set +the option builders-use-substitutes +in your local nix.conf. + +To build only on remote builders and disable building on the local machine, +you can use the option . diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml new file mode 100644 index 00000000000..acfe9e3cca1 --- /dev/null +++ b/doc/manual/advanced-topics/post-build-hook.xml @@ -0,0 +1,160 @@ + + +Using the <option linkend="conf-post-build-hook">post-build-hook</option> +Uploading to an S3-compatible binary cache after each build + + +
+ Implementation Caveats + Here we use the post-build hook to upload to a binary cache. + This is a simple and working example, but it is not suitable for all + use cases. + + The post build hook program runs after each executed build, + and blocks the build loop. The build loop exits if the hook program + fails. + + Concretely, this implementation will make Nix slow or unusable + when the internet is slow or unreliable. + + A more advanced implementation might pass the store paths to a + user-supplied daemon or queue for processing the store paths outside + of the build loop. +
+ +
+ Prerequisites + + + This tutorial assumes you have configured an S3-compatible binary cache + according to the instructions at + , and + that the root user's default AWS profile can + upload to the bucket. + +
+ +
+ Set up a Signing Key + Use nix-store --generate-binary-cache-key to + create our public and private signing keys. We will sign paths + with the private key, and distribute the public key for verifying + the authenticity of the paths. + + +# nix-store --generate-binary-cache-key example-nix-cache-1 /etc/nix/key.private /etc/nix/key.public +# cat /etc/nix/key.public +example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= + + +Then, add the public key and the cache URL to your +nix.conf's +and like: + + +substituters = https://cache.nixos.org/ s3://example-nix-cache +trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= + + +We will restart the Nix daemon in a later step. +
+ +
+ Implementing the build hook + Write the following script to + /etc/nix/upload-to-cache.sh: + + + +#!/bin/sh + +set -eu +set -f # disable globbing +export IFS=' ' + +echo "Signing paths" $OUT_PATHS +nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS +echo "Uploading paths" $OUT_PATHS +exec nix copy --to 's3://example-nix-cache' $OUT_PATHS + + + + Should <literal>$OUT_PATHS</literal> be quoted? + + The $OUT_PATHS variable is a space-separated + list of Nix store paths. In this case, we expect and want the + shell to perform word splitting to make each output path its + own argument to nix sign-paths. Nix guarantees + the paths will not contain any spaces, however a store path + might contain glob characters. The set -f + disables globbing in the shell. + + + + Then make sure the hook program is executable by the root user: + +# chmod +x /etc/nix/upload-to-cache.sh + +
+ +
+ Updating Nix Configuration + + Edit /etc/nix/nix.conf to run our hook, + by adding the following configuration snippet at the end: + + +post-build-hook = /etc/nix/upload-to-cache.sh + + +Then, restart the nix-daemon. +
+ +
+ Testing + + Build any derivation, for example: + + +$ nix-build -E '(import <nixpkgs> {}).writeText "example" (builtins.toString builtins.currentTime)' +these derivations will be built: + /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv +building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'... +running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'... +post-build-hook: Signing paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +post-build-hook: Uploading paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +/nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example + + + Then delete the path from the store, and try substituting it from the binary cache: + +$ rm ./result +$ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example + + +Now, copy the path back from the cache: + +$ nix-store --realise /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'... +warning: you did not specify '--add-root'; the result might be removed by the garbage collector +/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example + +
+
+ Conclusion + + We now have a Nix installation configured to automatically sign and + upload every local build to a remote binary cache. + + + + Before deploying this to production, be sure to consider the + implementation caveats in . + +
+ diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 6c0af39ecda..1fa74a14334 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -1,7 +1,9 @@ + + xml:id="sec-conf-file" + version="5"> nix.conf @@ -17,46 +19,407 @@ Description -A number of persistent settings of Nix are stored in the file -sysconfdir/nix/nix.conf or -$NIX_CONF_DIR/nix.conf if NIX_CONF_DIR is set. -This file is a list of name = -value pairs, one per line. -Comments start with a # character. Here is an example -configuration file: +By default Nix reads settings from the following places: + +The system-wide configuration file +sysconfdir/nix/nix.conf +(i.e. /etc/nix/nix.conf on most systems), or +$NIX_CONF_DIR/nix.conf if +NIX_CONF_DIR is set. Values loaded in this file are not forwarded to the Nix daemon. The +client assumes that the daemon has already loaded them. + + +User-specific configuration files: + + + If NIX_USER_CONF_FILES is set, then each path separated by + : will be loaded in reverse order. + + + + Otherwise it will look for nix/nix.conf files in + XDG_CONFIG_DIRS and XDG_CONFIG_HOME. + + The default location is $HOME/.config/nix.conf if + those environment variables are unset. + + +The configuration files consist of +name = +value pairs, one per line. Other +files can be included with a line like include +path, where +path is interpreted relative to the current +conf file and a missing file is an error unless +!include is used instead. +Comments start with a # character. Here is an +example configuration file: -gc-keep-outputs = true # Nice for developers -gc-keep-derivations = true # Idem -env-keep-derivations = false +keep-outputs = true # Nice for developers +keep-derivations = true # Idem -You can override settings using the -flag, e.g. --option gc-keep-outputs false. +You can override settings on the command line using the + flag, e.g. --option keep-outputs +false. The following settings are currently available: - gc-keep-outputs + allowed-uris - If true, the garbage collector - will keep the outputs of non-garbage derivations. If - false (default), outputs will be deleted unless - they are GC roots themselves (or reachable from other roots). + - In general, outputs must be registered as roots separately. - However, even if the output of a derivation is registered as a - root, the collector will still delete store paths that are used - only at build time (e.g., the C compiler, or source tarballs - downloaded from the network). To prevent it from doing so, set - this option to true. + A list of URI prefixes to which access is allowed in + restricted evaluation mode. For example, when set to + https://github.com/NixOS, builtin functions + such as fetchGit are allowed to access + https://github.com/NixOS/patchelf.git. + + + + + + + allow-import-from-derivation + + By default, Nix allows you to import from a derivation, + allowing building at evaluation time. With this option set to false, Nix will throw an error + when evaluating an expression that uses this feature, allowing users to ensure their evaluation + will not require any builds to take place. + + + + + allow-new-privileges + + (Linux-specific.) By default, builders on Linux + cannot acquire new privileges by calling setuid/setgid programs or + programs that have file capabilities. For example, programs such + as sudo or ping will + fail. (Note that in sandbox builds, no such programs are available + unless you bind-mount them into the sandbox via the + option.) You can allow the + use of such programs by enabling this option. This is impure and + usually undesirable, but may be useful in certain scenarios + (e.g. to spin up containers or set up userspace network interfaces + in tests). + + + + + allowed-users + + + + A list of names of users (separated by whitespace) that + are allowed to connect to the Nix daemon. As with the + option, you can specify groups by + prefixing them with @. Also, you can allow + all users by specifying *. The default is + *. + + Note that trusted users are always allowed to connect. + + + + + + + auto-optimise-store + + If set to true, Nix + automatically detects files in the store that have identical + contents, and replaces them with hard links to a single copy. + This saves disk space. If set to false (the + default), you can still run nix-store + --optimise to get rid of duplicate + files. + + + + + builders + + A list of machines on which to perform builds. See for details. + + + + + builders-use-substitutes + + If set to true, Nix will instruct + remote build machines to use their own binary substitutes if available. In + practical terms, this means that remote hosts will fetch as many build + dependencies as possible from their own substitutes (e.g, from + cache.nixos.org), instead of waiting for this host to + upload them all. This can drastically reduce build times if the network + connection between this computer and the remote build host is slow. Defaults + to false. + + + + build-users-group + + This options specifies the Unix group containing + the Nix build user accounts. In multi-user Nix installations, + builds should not be performed by the Nix account since that would + allow users to arbitrarily modify the Nix store and database by + supplying specially crafted builders; and they cannot be performed + by the calling user since that would allow him/her to influence + the build result. + + Therefore, if this option is non-empty and specifies a valid + group, builds will be performed under the user accounts that are a + member of the group specified here (as listed in + /etc/group). Those user accounts should not + be used for any other purpose! + + Nix will never run two builds under the same user account at + the same time. This is to prevent an obvious security hole: a + malicious user writing a Nix expression that modifies the build + result of a legitimate Nix expression being built by another user. + Therefore it is good to have as many Nix build user accounts as + you can spare. (Remember: uids are cheap.) + + The build users should have permission to create files in + the Nix store, but not delete them. Therefore, + /nix/store should be owned by the Nix + account, its group should be the group specified here, and its + mode should be 1775. + + If the build users group is empty, builds will be performed + under the uid of the Nix process (that is, the uid of the caller + if NIX_REMOTE is empty, the uid under which the Nix + daemon runs if NIX_REMOTE is + daemon). Obviously, this should not be used in + multi-user settings with untrusted users. + + + + + + + compress-build-log + + If set to true (the default), + build logs written to /nix/var/log/nix/drvs + will be compressed on the fly using bzip2. Otherwise, they will + not be compressed. + + + + connect-timeout + + + + The timeout (in seconds) for establishing connections in + the binary cache substituter. It corresponds to + curl’s + option. + + + + + + + cores + + Sets the value of the + NIX_BUILD_CORES environment variable in the + invocation of builders. Builders can use this variable at their + discretion to control the maximum amount of parallelism. For + instance, in Nixpkgs, if the derivation attribute + enableParallelBuilding is set to + true, the builder passes the + flag to GNU Make. + It can be overridden using the command line switch and + defaults to 1. The value 0 + means that the builder should use all available CPU cores in the + system. + + See also . + + + diff-hook + + + Absolute path to an executable capable of diffing build results. + The hook executes if is + true, and the output of a build is known to not be the same. + This program is not executed to determine if two results are the + same. + + + + The diff hook is executed by the same user and group who ran the + build. However, the diff hook does not have write access to the + store path just built. + + + The diff hook program receives three parameters: + + + + + A path to the previous build's results + + + + + + A path to the current build's results + + + + + + The path to the build's derivation + + + + + + The path to the build's scratch directory. This directory + will exist only if the build was run with + . + + + + + + The stderr and stdout output from the diff hook will not be + displayed to the user. Instead, it will print to the nix-daemon's + log. + + + When using the Nix daemon, diff-hook must + be set in the nix.conf configuration file, and + cannot be passed at the command line. + + + + + + enforce-determinism + + See . + + + + extra-sandbox-paths + + A list of additional paths appended to + . Useful if you want to extend + its default value. + + + + + extra-platforms + + Platforms other than the native one which + this machine is capable of building for. This can be useful for + supporting additional architectures on compatible machines: + i686-linux can be built on x86_64-linux machines (and the default + for this setting reflects this); armv7 is backwards-compatible with + armv6 and armv5tel; some aarch64 machines can also natively run + 32-bit ARM code; and qemu-user may be used to support non-native + platforms (though this may be slow and buggy). Most values for this + are not enabled by default because build systems will often + misdetect the target platform and generate incompatible code, so you + may wish to cross-check the results of using this option against + proper natively-built versions of your + derivations. + + + + + extra-substituters + + Additional binary caches appended to those + specified in . When used by + unprivileged users, untrusted substituters (i.e. those not listed + in ) are silently + ignored. + + + + fallback + + If set to true, Nix will fall + back to building from source if a binary substitute fails. This + is equivalent to the flag. The + default is false. + + + + fsync-metadata + + If set to true, changes to the + Nix store metadata (in /nix/var/nix/db) are + synchronously flushed to disk. This improves robustness in case + of system crashes, but reduces performance. The default is + true. + hashed-mirrors - gc-keep-derivations + A list of web servers used by + builtins.fetchurl to obtain files by + hash. The default is + http://tarballs.nixos.org/. Given a hash type + ht and a base-16 hash + h, Nix will try to download the file + from + hashed-mirror/ht/h. + This allows files to be downloaded even if they have disappeared + from their original URI. For example, given the default mirror + http://tarballs.nixos.org/, when building the derivation + + +builtins.fetchurl { + url = "https://example.org/foo-1.2.3.tar.xz"; + sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"; +} + + + Nix will attempt to download this file from + http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae + first. If it is not available there, if will try the original URI. + + + + + http-connections + + The maximum number of parallel TCP connections + used to fetch files from binary caches and by other downloads. It + defaults to 25. 0 means no limit. + + + + + keep-build-log + + If set to true (the default), + Nix will write the build log of a derivation (i.e. the standard + output and error of its builder) to the directory + /nix/var/log/nix/drvs. The build log can be + retrieved using the command nix-store -l + path. + + + + + keep-derivations If true (default), the garbage collector will keep the derivations from which non-garbage store @@ -68,15 +431,13 @@ flag, e.g. --option gc-keep-outputs false. traceability (e.g., it allows you to ask with what dependencies or options a store path was built), so by default this option is on. Turn it off to save a bit of disk space (or a lot if - gc-keep-outputs is also turned on). - + keep-outputs is also turned on). - - env-keep-derivations + keep-env-derivations If false (default), derivations - are not stored in Nix user environments. That is, the derivation + are not stored in Nix user environments. That is, the derivations of any build-time-only dependencies may be garbage-collected. If true, when you add a Nix derivation to @@ -85,51 +446,72 @@ flag, e.g. --option gc-keep-outputs false. garbage-collected until the user environment generation is deleted (nix-env --delete-generations). To prevent build-time-only dependencies from being collected, you should also - turn on gc-keep-outputs. + turn on keep-outputs. The difference between this option and - gc-keep-derivations is that this one is + keep-derivations is that this one is “sticky”: it applies to any user environment created while this - option was enabled, while gc-keep-derivations + option was enabled, while keep-derivations only applies at the moment the garbage collector is run. + keep-outputs - build-max-jobs - - This option defines the maximum number of jobs - that Nix will try to build in parallel. The default is - 1. You should generally set it to the number - of CPUs in your system (e.g., 2 on an Athlon 64 - X2). It can be overridden using the () - command line switch. + If true, the garbage collector + will keep the outputs of non-garbage derivations. If + false (default), outputs will be deleted unless + they are GC roots themselves (or reachable from other roots). + In general, outputs must be registered as roots separately. + However, even if the output of a derivation is registered as a + root, the collector will still delete store paths that are used + only at build time (e.g., the C compiler, or source tarballs + downloaded from the network). To prevent it from doing so, set + this option to true. + max-build-log-size - build-cores + - Sets the value of the - NIX_BUILD_CORES environment variable in the - invocation of builders. Builders can use this variable at their - discretion to control the maximum amount of parallelism. For - instance, in Nixpkgs, if the derivation attribute - enableParallelBuilding is set to - true, the builder passes the - flag to GNU Make. - It can be overridden using the command line switch and - defaults to 1. The value 0 - means that the builder should use all available CPU cores in the - system. + This option defines the maximum number of bytes that a + builder can write to its stdout/stderr. If the builder exceeds + this limit, it’s killed. A value of 0 (the + default) means that there is no limit. + + + max-free - build-max-silent-time + When a garbage collection is triggered by the + min-free option, it stops as soon as + max-free bytes are available. The default is + infinity (i.e. delete all garbage). + + + + max-jobs + + This option defines the maximum number of jobs + that Nix will try to build in parallel. The default is + 1. The special value auto + causes Nix to use the number of CPUs in your system. 0 + is useful when using remote builders to prevent any local builds (except for + preferLocalBuild derivation attribute which executes locally + regardless). It can be + overridden using the () + command line switch. + + See also . + + + + max-silent-time @@ -149,83 +531,253 @@ flag, e.g. --option gc-keep-outputs false. + min-free + + + When free disk space in /nix/store + drops below min-free during a build, Nix + performs a garbage-collection until max-free + bytes are available or there is no more garbage. A value of + 0 (the default) disables this feature. + + + - build-timeout + narinfo-cache-negative-ttl - This option defines the maximum number of seconds that a - builder can run. This is useful (for instance in an automated - build system) to catch builds that are stuck in an infinite loop - but keep writing to their standard output or standard error. It - can be overridden using the command line - switch. + The TTL in seconds for negative lookups. If a store path is + queried from a substituter but was not found, there will be a + negative lookup cached in the local disk cache database for the + specified duration. - The value 0 means that there is no - timeout. This is also the default. + + + + + narinfo-cache-positive-ttl + + + + The TTL in seconds for positive lookups. If a store path is + queried from a substituter, the result of the query will be cached + in the local disk cache database including some of the NAR + metadata. The default TTL is a month, setting a shorter TTL for + positive lookups can be useful for binary caches that have + frequent garbage collection, in which case having a more frequent + cache invalidation would prevent trying to pull the path again and + failing with a hash mismatch if the build isn't reproducible. + + netrc-file + + If set to an absolute path to a netrc + file, Nix will use the HTTP authentication credentials in this file when + trying to download from a remote host through HTTP or HTTPS. Defaults to + $NIX_CONF_DIR/netrc. + + The netrc file consists of a list of + accounts in the following format: + + +machine my-machine +login my-username +password my-password + - build-max-log-size + For the exact syntax, see the + curl documentation. + This must be an absolute path, and ~ + is not resolved. For example, ~/.netrc won't + resolve to your home directory's .netrc. + + + + + + + plugin-files + + A list of plugin files to be loaded by Nix. Each of these + files will be dlopened by Nix, allowing them to affect + execution through static initialization. In particular, these + plugins may construct static instances of RegisterPrimOp to + add new primops or constants to the expression language, + RegisterStoreImplementation to add new store implementations, + RegisterCommand to add new subcommands to the + nix command, and RegisterSetting to add new + nix config settings. See the constructors for those types for + more details. + + + Since these files are loaded into the same address space as + Nix itself, they must be DSOs compatible with the instance of + Nix running at the time (i.e. compiled against the same + headers, not linked to any incompatible libraries). They + should not be linked to any Nix libs directly, as those will + be available already at load time. + + + If an entry in the list is a directory, all files in the + directory are loaded as plugins (non-recursively). + + - This option defines the maximum number of bytes that a - builder can write to its stdout/stderr. If the builder exceeds - this limit, it’s killed. A value of 0 (the - default) means that there is no limit. + + + pre-build-hook + + + + + If set, the path to a program that can set extra + derivation-specific settings for this system. This is used for settings + that can't be captured by the derivation model itself and are too variable + between different versions of the same system to be hard-coded into nix. + + + The hook is passed the derivation path and, if sandboxes are enabled, + the sandbox directory. It can then modify the sandbox and send a series of + commands to modify various settings to stdout. The currently recognized + commands are: + + + + extra-sandbox-paths + + + Pass a list of files and directories to be included in the + sandbox for this build. One entry per line, terminated by an empty + line. Entries have the same format as + sandbox-paths. + + + + + + + post-build-hook + + Optional. The path to a program to execute after each build. - build-users-group + This option is only settable in the global + nix.conf, or on the command line by trusted + users. - This options specifies the Unix group containing - the Nix build user accounts. In multi-user Nix installations, - builds should not be performed by the Nix account since that would - allow users to arbitrarily modify the Nix store and database by - supplying specially crafted builders; and they cannot be performed - by the calling user since that would allow him/her to influence - the build result. + When using the nix-daemon, the daemon executes the hook as + root. If the nix-daemon is not involved, the + hook runs as the user executing the nix-build. - Therefore, if this option is non-empty and specifies a valid - group, builds will be performed under the user accounts that are a - member of the group specified here (as listed in - /etc/group). Those user accounts should not - be used for any other purpose! + + The hook executes after an evaluation-time build. + The hook does not execute on substituted paths. + The hook's output always goes to the user's terminal. + If the hook fails, the build succeeds but no further builds execute. + The hook executes synchronously, and blocks other builds from progressing while it runs. + - Nix will never run two builds under the same user account at - the same time. This is to prevent an obvious security hole: a - malicious user writing a Nix expression that modifies the build - result of a legitimate Nix expression being built by another user. - Therefore it is good to have as many Nix build user accounts as - you can spare. (Remember: uids are cheap.) + The program executes with no arguments. The program's environment + contains the following environment variables: + + + + DRV_PATH + + The derivation for the built paths. + Example: + /nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv + + + + + + OUT_PATHS + + Output paths of the built derivation, separated by a space character. + Example: + /nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev + /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc + /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info + /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man + /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23. + + + + + + See for an example + implementation. + + + + + repeat + + How many times to repeat builds to check whether + they are deterministic. The default value is 0. If the value is + non-zero, every build is repeated the specified number of + times. If the contents of any of the runs differs from the + previous ones and is + true, the build is rejected and the resulting store paths are not + registered as “valid” in Nix’s database. + + + require-sigs + + If set to true (the default), + any non-content-addressed path added or copied to the Nix store + (e.g. when substituting from a binary cache) must have a valid + signature, that is, be signed using one of the keys listed in + or + . Set to false + to disable signature checking. + + - The build users should have permission to create files in - the Nix store, but not delete them. Therefore, - /nix/store should be owned by the Nix - account, its group should be the group specified here, and its - mode should be 1775. - If the build users group is empty, builds will be performed - under the uid of the Nix process (that is, the uid of the caller - if NIX_REMOTE is empty, the uid under which the Nix - daemon runs if NIX_REMOTE is - daemon). Obviously, this should not be used in - multi-user settings with untrusted users. + restrict-eval + + + + If set to true, the Nix evaluator will + not allow access to any files outside of the Nix search path (as + set via the NIX_PATH environment variable or the + option), or to URIs outside of + . The default is + false. + run-diff-hook + + + If true, enable the execution of . + + + + When using the Nix daemon, run-diff-hook must + be set in the nix.conf configuration file, + and cannot be passed at the command line. + + + - build-use-sandbox + sandbox If set to true, builds will be performed in a sandboxed environment, i.e., @@ -234,7 +786,7 @@ flag, e.g. --option gc-keep-outputs false. directory, private versions of /proc, /dev, /dev/shm and /dev/pts (on Linux), and the paths configured with the - build-sandbox-paths + sandbox-paths option. This is useful to prevent undeclared dependencies on files in directories such as /usr/bin. In addition, on Linux, builds run in private PID, mount, network, IPC @@ -242,7 +794,7 @@ flag, e.g. --option gc-keep-outputs false. system (except that fixed-output derivations do not run in private network namespace to ensure they can access the network). - Currently, sandboxing only work on Linux and Mac OS X. The use + Currently, sandboxing only work on Linux and macOS. The use of a sandbox requires that Nix is run as root (so you should use the “build users” feature to perform the actual builds under different users @@ -253,15 +805,28 @@ flag, e.g. --option gc-keep-outputs false. __noChroot attribute set to true do not run in sandboxes. - The default is false. + The default is true on Linux and + false on all other platforms. + sandbox-dev-shm-size + + This option determines the maximum size of the + tmpfs filesystem mounted on + /dev/shm in Linux sandboxes. For the format, + see the description of the option of + tmpfs in + mount8. The + default is 50%. + + + - - build-sandbox-paths + + sandbox-paths A list of paths bind-mounted into Nix sandbox environments. You can use the syntax @@ -283,154 +848,52 @@ flag, e.g. --option gc-keep-outputs false. - - build-extra-sandbox-paths - - A list of additional paths appended to - . Useful if you want to extend - its default value. - - - - - build-use-substitutes - - If set to true (default), Nix - will use binary substitutes if available. This option can be - disabled to force building from source. - - - - - build-fallback + secret-key-files - If set to true, Nix will fall - back to building from source if a binary substitute fails. This - is equivalent to the flag. The - default is false. + A whitespace-separated list of files containing + secret (private) keys. These are used to sign locally-built + paths. They can be generated using nix-store + --generate-binary-cache-key. The corresponding public + key can be distributed to other users, who can add it to + in their + nix.conf. - build-keep-log + show-trace - If set to true (the default), - Nix will write the build log of a derivation (i.e. the standard - output and error of its builder) to the directory - /nix/var/log/nix/drvs. The build log can be - retrieved using the command nix-store -l - path. + Causes Nix to print out a stack trace in case of Nix + expression evaluation errors. - build-compress-log + substitute - If set to true (the default), - build logs written to /nix/var/log/nix/drvs - will be compressed on the fly using bzip2. Otherwise, they will - not be compressed. + If set to true (default), Nix + will use binary substitutes if available. This option can be + disabled to force building from source. - - use-binary-caches - - If set to true (the default), - Nix will check the binary caches specified by - and related options to obtain - binary substitutes. - + stalled-download-timeout + + The timeout (in seconds) for receiving data from servers + during download. Nix cancels idle downloads after this timeout's + duration. + + substituters - binary-caches - - A list of URLs of binary caches, separated by + A list of URLs of substituters, separated by whitespace. The default is https://cache.nixos.org. - - binary-caches-files - - A list of names of files that will be read to - obtain additional binary cache URLs. The default is - /nix/var/nix/profiles/per-user/username/channels/binary-caches/*. - Note that when you’re using the Nix daemon, - username is always equal to - root, so Nix will only use the binary caches - provided by the channels installed by root. Do not set this - option to read files created by untrusted users! - - - - - trusted-binary-caches - - A list of URLs of binary caches, separated by - whitespace. These are not used by default, but can be enabled by - users of the Nix daemon by specifying --option - binary-caches urls on the - command line. Unprivileged users are only allowed to pass a - subset of the URLs listed in binary-caches and - trusted-binary-caches. - - - - - extra-binary-caches - - Additional binary caches appended to those - specified in and - . When used by unprivileged - users, untrusted binary caches (i.e. those not listed in - ) are silently - ignored. - - - - - signed-binary-caches - - If set to *, Nix will only - download binaries if they are signed using one of the keys listed - in . - - - - - binary-cache-public-keys - - A whitespace-separated list of public keys - corresponding to the secret keys trusted to sign binary - caches. For example: - cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=. - - - - - binary-caches-parallel-connections - - The maximum number of parallel TCP connections - used to fetch files from binary caches and by other downloads. It - defaults to 25. 0 means no limit. - - - - - verify-https-binary-caches - - Whether HTTPS binary caches are required to have a - certificate that can be verified. Defaults to - true. - - - - - system + system This option specifies the canonical Nix system name of the current installation, such as @@ -451,57 +914,126 @@ flag, e.g. --option gc-keep-outputs false. - fsync-metadata + system-features - If set to true, changes to the - Nix store metadata (in /nix/var/nix/db) are - synchronously flushed to disk. This improves robustness in case - of system crashes, but reduces performance. The default is - true. + A set of system “features” supported by this + machine, e.g. kvm. Derivations can express a + dependency on such features through the derivation attribute + requiredSystemFeatures. For example, the + attribute - + +requiredSystemFeatures = [ "kvm" ]; + + ensures that the derivation can only be built on a machine with + the kvm feature. - auto-optimise-store + This setting by default includes kvm if + /dev/kvm is accessible, and the + pseudo-features nixos-test, + benchmark and big-parallel + that are used in Nixpkgs to route builds to specific + machines. - If set to true, Nix - automatically detects files in the store that have identical - contents, and replaces them with hard links to a single copy. - This saves disk space. If set to false (the - default), you can still run nix-store - --optimise to get rid of duplicate - files. + + tarball-ttl - connect-timeout + + Default: 3600 seconds. + + The number of seconds a downloaded tarball is considered + fresh. If the cached tarball is stale, Nix will check whether + it is still up to date using the ETag header. Nix will download + a new version if the ETag header is unsupported, or the + cached ETag doesn't match. + + + Setting the TTL to 0 forces Nix to always + check if the tarball is up to date. + + Nix caches tarballs in + $XDG_CACHE_HOME/nix/tarballs. + + Files fetched via NIX_PATH, + fetchGit, fetchMercurial, + fetchTarball, and fetchurl + respect this TTL. + + + + + timeout - The timeout (in seconds) for establishing connections in - the binary cache substituter. It corresponds to - curl’s - option. + This option defines the maximum number of seconds that a + builder can run. This is useful (for instance in an automated + build system) to catch builds that are stuck in an infinite loop + but keep writing to their standard output or standard error. It + can be overridden using the command line + switch. + + The value 0 means that there is no + timeout. This is also the default. - - log-servers + trace-function-calls - A list of URL prefixes (such as - http://hydra.nixos.org/log) from which - nix-store -l will try to fetch build logs if - they’re not available locally. + Default: false. + + If set to true, the Nix evaluator will + trace every function call. Nix will print a log message at the + "vomit" level for every function entrance and function exit. + + +function-trace entered undefined position at 1565795816999559622 +function-trace exited undefined position at 1565795816999581277 +function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150 +function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684 + + + The undefined position means the function + call is a builtin. + + Use the contrib/stack-collapse.py script + distributed with the Nix source code to convert the trace logs + in to a format suitable for flamegraph.pl. + trusted-public-keys + + A whitespace-separated list of public keys. When + paths are copied from another Nix store (such as a binary cache), + they must be signed with one of these keys. For example: + cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= + hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=. + + + + trusted-substituters + + A list of URLs of substituters, separated by + whitespace. These are not used by default, but can be enabled by + users of the Nix daemon by specifying --option + substituters urls on the + command line. Unprivileged users are only allowed to pass a + subset of the URLs listed in substituters and + trusted-substituters. + + trusted-users @@ -516,114 +1048,189 @@ flag, e.g. --option gc-keep-outputs false. wheel group. The default is root. - The users listed here have the ability to - compromise the security of a multi-user Nix store. For instance, - they could install Trojan horses subsequently executed by other - users. So you should consider carefully whether to add users to - this list. + Adding a user to + is essentially equivalent to giving that user root access to the + system. For example, the user can set + and thereby obtain read access to + directories that are otherwise inacessible to + them. + + + + + Deprecated Settings - allowed-users + - + - A list of names of users (separated by whitespace) that - are allowed to connect to the Nix daemon. As with the - option, you can specify groups by - prefixing them with @. Also, you can allow - all users by specifying *. The default is - *. + + binary-caches - Note that trusted users are always allowed to connect. + Deprecated: + binary-caches is now an alias to + . + - + + binary-cache-public-keys + Deprecated: + binary-cache-public-keys is now an alias to + . + + build-compress-log - restrict-eval + Deprecated: + build-compress-log is now an alias to + . + - + + build-cores - If set to true, the Nix evaluator will - not allow access to any files outside of the Nix search path (as - set via the NIX_PATH environment variable or the - option). The default is - false. + Deprecated: + build-cores is now an alias to + . + - + + build-extra-chroot-dirs + Deprecated: + build-extra-chroot-dirs is now an alias to + . + + build-extra-sandbox-paths - pre-build-hook + Deprecated: + build-extra-sandbox-paths is now an alias to + . + - + + build-fallback + Deprecated: + build-fallback is now an alias to + . + - If set, the path to a program that can set extra - derivation-specific settings for this system. This is used for settings - that can't be captured by the derivation model itself and are too variable - between different versions of the same system to be hard-coded into nix. - + + build-max-jobs - The hook is passed the derivation path and, if sandboxes are enabled, - the sandbox directory. It can then modify the sandbox and send a series of - commands to modify various settings to stdout. The currently recognized - commands are: + Deprecated: + build-max-jobs is now an alias to + . + - - - extra-sandbox-paths + + build-max-log-size - + Deprecated: + build-max-log-size is now an alias to + . + - Pass a list of files and directories to be included in the - sandbox for this build. One entry per line, terminated by an empty - line. Entries have the same format as - build-sandbox-paths. + + build-max-silent-time - + Deprecated: + build-max-silent-time is now an alias to + . + - - - + + build-repeat + Deprecated: + build-repeat is now an alias to + . + + build-timeout - build-repeat + Deprecated: + build-timeout is now an alias to + . + - How many times to repeat builds to check whether - they are deterministic. The default value is 0. If the value is - non-zero, every build is repeated the specified number of - times. If the contents of any of the runs differs from the - previous ones, the build is rejected and the resulting store paths - are not registered as “valid” in Nix’s database. + + build-use-chroot + Deprecated: + build-use-chroot is now an alias to + . + + build-use-sandbox - sandbox-dev-shm-size + Deprecated: + build-use-sandbox is now an alias to + . + - This option determines the maximum size of the - tmpfs filesystem mounted on - /dev/shm in Linux sandboxes. For the format, - see the description of the option of - tmpfs in - mount8. The - default is 50%. + + build-use-substitutes + Deprecated: + build-use-substitutes is now an alias to + . + + gc-keep-derivations - + Deprecated: + gc-keep-derivations is now an alias to + . + + + + gc-keep-outputs + + Deprecated: + gc-keep-outputs is now an alias to + . + + + + env-keep-derivations + + Deprecated: + env-keep-derivations is now an alias to + . + + + + extra-binary-caches + + Deprecated: + extra-binary-caches is now an alias to + . + + + trusted-binary-caches + + Deprecated: + trusted-binary-caches is now an alias to + . + + + diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml index c757cb17bd1..8466cc46366 100644 --- a/doc/manual/command-ref/env-common.xml +++ b/doc/manual/command-ref/env-common.xml @@ -14,7 +14,8 @@ IN_NIX_SHELL Indicator that tells if the current environment was set up by - nix-shell. + nix-shell. Since Nix 2.0 the values are + "pure" and "impure" @@ -32,7 +33,7 @@ will cause Nix to look for paths relative to /home/eelco/Dev and - /etc/nixos, in that order. It is also + /etc/nixos, in this order. It is also possible to match paths against a prefix. For example, the value @@ -52,10 +53,15 @@ nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos NIX_PATH to -nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz +nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz tells Nix to download the latest revision in the Nixpkgs/NixOS - 14.12 channel. + 15.09 channel. + + A following shorthand can be used to refer to the official channels: + + nixpkgs=channel:nixos-15.09 + The search path can be extended using the option, which takes precedence over @@ -116,7 +122,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix NIX_LOG_DIR Overrides the location of the Nix log directory - (default prefix/log/nix). + (default prefix/var/log/nix). @@ -131,12 +137,19 @@ $ mount -o bind /mnt/otherdisk/nix /nix NIX_CONF_DIR - Overrides the location of the Nix configuration + Overrides the location of the system Nix configuration directory (default prefix/etc/nix). +NIX_USER_CONF_FILES + + Overrides the location of the user Nix configuration files + to load from (defaults to the XDG spec locations). The variable is treated + as a list separated by the : token. + + TMPDIR @@ -148,145 +161,14 @@ $ mount -o bind /mnt/otherdisk/nix /nix -NIX_BUILD_HOOK - - - - Specifies the location of the build hook, - which is a program (typically some script) that Nix will call - whenever it wants to build a derivation. This is used to implement - distributed builds (see ). - - - - - - - - - NIX_REMOTE This variable should be set to daemon if you want to use the Nix daemon to execute Nix operations. This is necessary in multi-user Nix installations. + If the Nix daemon's Unix socket is at some non-standard path, + this variable should be set to unix://path/to/socket. Otherwise, it should be left unset. diff --git a/doc/manual/command-ref/nix-build.xml b/doc/manual/command-ref/nix-build.xml index d6b2e5e5adb..886d2591043 100644 --- a/doc/manual/command-ref/nix-build.xml +++ b/doc/manual/command-ref/nix-build.xml @@ -29,9 +29,8 @@ attrPath - drvlink - + @@ -91,25 +90,6 @@ also . - drvlink - - Add a symlink named - drvlink to the store derivation - produced by nix-instantiate. The derivation is - a root of the garbage collector until the symlink is deleted or - renamed. If there are multiple derivations, numbers are suffixed - to drvlink to distinguish between - them. - - - - - - Shorthand for - ./derivation. - - - Do not create a symlink to the output path. Note @@ -119,6 +99,10 @@ also . + + Show what store paths would be built or downloaded. + + / outlink diff --git a/doc/manual/command-ref/nix-channel.xml b/doc/manual/command-ref/nix-channel.xml index 9acf44e5298..ebcf56aff89 100644 --- a/doc/manual/command-ref/nix-channel.xml +++ b/doc/manual/command-ref/nix-channel.xml @@ -31,12 +31,14 @@ Description -A Nix channel is mechanism that allows you to automatically stay -up-to-date with a set of pre-built Nix expressions. A Nix channel is -just a URL that points to a place containing both a set of Nix -expressions and a pointer to a binary cache. See also . +A Nix channel is a mechanism that allows you to automatically +stay up-to-date with a set of pre-built Nix expressions. A Nix +channel is just a URL that points to a place containing a set of Nix +expressions. See also . + +To see the list of official NixOS channels, visit . This command has the following operations: @@ -112,13 +114,13 @@ $ nix-env -iA nixpkgs.hello You can revert channel updates using : -$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.nixpkgsVersion' +$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version' "14.04.527.0e935f1" $ nix-channel --rollback switching from generation 483 to 482 -$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.nixpkgsVersion' +$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version' "14.04.526.dbadfad" @@ -165,25 +167,13 @@ following files: nixexprs.tar.xz A tarball containing Nix expressions and files - referenced by them (such as build scripts and patches). At - top-level, the tarball should contain a single directory. That + referenced by them (such as build scripts and patches). At the + top level, the tarball should contain a single directory. That directory must contain a file default.nix that serves as the channel’s “entry point”. - binary-cache-url - - A file containing the URL to a binary cache (such - as https://cache.nixos.org. Nix will automatically - check this cache for pre-built binaries, if the user has - sufficient rights to add binary caches. For instance, in a - multi-user Nix setup, the binary caches provided by the channels - of the root user are used automatically, but caches corresponding - to the channels of non-root users are ignored. - - - diff --git a/doc/manual/command-ref/nix-collect-garbage.xml b/doc/manual/command-ref/nix-collect-garbage.xml index 35a78c5b201..43e06879691 100644 --- a/doc/manual/command-ref/nix-collect-garbage.xml +++ b/doc/manual/command-ref/nix-collect-garbage.xml @@ -22,12 +22,6 @@ period - - - - - - bytes diff --git a/doc/manual/command-ref/nix-copy-closure.xml b/doc/manual/command-ref/nix-copy-closure.xml index 97e261ae993..e6dcf180ad6 100644 --- a/doc/manual/command-ref/nix-copy-closure.xml +++ b/doc/manual/command-ref/nix-copy-closure.xml @@ -27,8 +27,10 @@ --> - - + + + + user@machine @@ -93,15 +95,6 @@ those paths. If this bothers you, use - - Also copy the outputs of store derivations diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index 85f10e0760b..2b95b68191e 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -146,8 +146,7 @@ also . - - + / path Specifies the Nix expression (designated below as the active Nix expression) used by the @@ -166,8 +165,7 @@ also . - - + / path Specifies the profile to be used by those operations that operate on a profile (designated below as the @@ -223,31 +221,53 @@ also . ~/.nix-defexpr - A directory that contains the default Nix + The source for the default Nix expressions used by the , , and operations to obtain derivations. The + --available operations to obtain derivations. The option may be used to override this default. - The Nix expressions in this directory are combined into a - single set, with each file as an attribute that has the name of - the file. Thus, if ~/.nix-defexpr contains - two files, foo and bar, + If ~/.nix-defexpr is a file, + it is loaded as a Nix expression. If the expression + is a set, it is used as the default Nix expression. + If the expression is a function, an empty set is passed + as argument and the return value is used as + the default Nix expression. + + If ~/.nix-defexpr is a directory + containing a default.nix file, that file + is loaded as in the above paragraph. + + If ~/.nix-defexpr is a directory without + a default.nix file, then its contents + (both files and subdirectories) are loaded as Nix expressions. + The expressions are combined into a single set, each expression + under an attribute with the same name as the original file + or subdirectory. + + + For example, if ~/.nix-defexpr contains + two files, foo.nix and bar.nix, then the default Nix expression will essentially be { - foo = import ~/.nix-defexpr/foo; - bar = import ~/.nix-defexpr/bar; + foo = import ~/.nix-defexpr/foo.nix; + bar = import ~/.nix-defexpr/bar.nix; } + The file manifest.nix is always ignored. + Subdirectories without a default.nix file + are traversed recursively in search of more Nix expressions, + but the names of these intermediate directories are not + added to the attribute paths of the default Nix expression. + The command nix-channel places symlinks to the downloaded Nix expressions from each subscribed channel in this directory. - @@ -458,7 +478,7 @@ $ nix-env -f ~/foo.nix -i '.*' from another profile: -$ nix-env -i --from-profile /nix/var/nix/profiles/foo -i gcc +$ nix-env -i --from-profile /nix/var/nix/profiles/foo gcc @@ -506,13 +526,10 @@ these paths will be fetched (0.04 MiB download, 0.19 MiB unpacked): 14.12 channel: -$ nix-env -f https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz -iA firefox +$ nix-env -f https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz -iA firefox -(The GitHub repository nixpkgs-channels is updated -automatically from the main nixpkgs repository -after certain tests have succeeded and binaries have been built and -uploaded to the binary cache at cache.nixos.org.) + @@ -639,7 +656,7 @@ upgrading `mozilla-1.2' to `mozilla-1.4' gcc-3.3.1 are split into two parts: the package name (gcc), and the version (3.3.1). The version part starts after the first -dash not following by a letter. x is considered an +dash not followed by a letter. x is considered an upgrade of y if their package names match, and the version of y is higher that that of x. @@ -1046,7 +1063,8 @@ user environment elements, etc. --> the derivation, which can be used to unambiguously select it using the option available in commands that install derivations like - nix-env --install. + nix-env --install. This option only works + together with @@ -1136,7 +1154,7 @@ user environment elements, etc. --> Print all of the meta-attributes of the derivation. This option is only available with - . + or . @@ -1348,11 +1366,15 @@ $ nix-env --list-generations This operation deletes the specified generations of the current profile. The generations can be a list of generation numbers, the special value old to delete all non-current -generations, or a value such as 30d to delete all +generations, a value such as 30d to delete all generations older than the specified number of days (except for the -generation that was active at that point in time). -Periodically deleting old generations is important to make garbage -collection effective. +generation that was active at that point in time), or a value such as ++5 to keep the last 5 generations +ignoring any newer than current, e.g., if 30 is the current +generation +5 will delete generation 25 +and all older generations. +Periodically deleting old generations is important to make garbage collection +effective. @@ -1361,6 +1383,8 @@ collection effective. $ nix-env --delete-generations 3 4 8 +$ nix-env --delete-generations +5 + $ nix-env --delete-generations 30d $ nix-env -p other_profile --delete-generations old @@ -1460,7 +1484,7 @@ error: no generation older than the current (91) exists Environment variables - + NIX_PROFILE Location of the Nix profile. Defaults to the @@ -1474,6 +1498,6 @@ error: no generation older than the current (91) exists - + diff --git a/doc/manual/command-ref/nix-hash.xml b/doc/manual/command-ref/nix-hash.xml index b4b509773d3..80263e18e33 100644 --- a/doc/manual/command-ref/nix-hash.xml +++ b/doc/manual/command-ref/nix-hash.xml @@ -44,7 +44,9 @@ cryptographic hash of the contents of each path and prints it on standard output. By default, it computes an MD5 hash, but other hash algorithms are -available as well. The hash is printed in hexadecimal. +available as well. The hash is printed in hexadecimal. To generate +the same hash as nix-prefetch-url you have to +specify multiple arguments, see below for an example. The hash is computed over a serialisation of each path: a dump of the file system tree rooted at the path. This @@ -122,6 +124,15 @@ cryptographic hash as nix-store --dump Examples +Computing the same hash as nix-prefetch-url: + +$ nix-prefetch-url file://<(echo test) +1lkgqb6fclns49861dwk9rzb6xnfkxbpws74mxnx01z9qyv1pjpj +$ nix-hash --type sha256 --flat --base32 <(echo test) +1lkgqb6fclns49861dwk9rzb6xnfkxbpws74mxnx01z9qyv1pjpj + + + Computing hashes: diff --git a/doc/manual/command-ref/nix-instantiate.xml b/doc/manual/command-ref/nix-instantiate.xml index 1e556c7ed7c..53f06aed124 100644 --- a/doc/manual/command-ref/nix-instantiate.xml +++ b/doc/manual/command-ref/nix-instantiate.xml @@ -24,6 +24,7 @@ + @@ -38,12 +39,13 @@ path - + files - + + nix-instantiate files @@ -115,26 +117,6 @@ input. - - - When used with and - , print the resulting expression as an - XML representation of the abstract syntax tree rather than as an - ATerm. The schema is the same as that used by the toXML - built-in. - - - - - - When used with and - , print the resulting expression as an - JSON representation of the abstract syntax tree rather than as an - ATerm. - - - When used with , @@ -149,12 +131,32 @@ input. + + + When used with , print the resulting + value as an JSON representation of the abstract syntax tree rather + than as an ATerm. + + + + + + When used with , print the resulting + value as an XML representation of the abstract syntax tree rather than as + an ATerm. The schema is the same as that used by the toXML built-in. + + + + When used with , perform evaluation in read/write mode so nix language features that require it will still work (at the cost of needing to do - instantiation of every evaluated derivation). + instantiation of every evaluated derivation). If this option is + not enabled, there may be uninstantiated store paths in the final + output. diff --git a/doc/manual/command-ref/nix-prefetch-url.xml b/doc/manual/command-ref/nix-prefetch-url.xml index 016d8863a94..621ded72ec2 100644 --- a/doc/manual/command-ref/nix-prefetch-url.xml +++ b/doc/manual/command-ref/nix-prefetch-url.xml @@ -19,14 +19,16 @@ nix-prefetch-url + hashAlgo + + name url hash - Description The command nix-prefetch-url downloads the @@ -51,7 +53,7 @@ avoided. If hash is specified, then a download is not performed if the Nix store already contains a file with the same hash and base name. Otherwise, the file is downloaded, and an -error if signaled if the actual hash of the file does not match the +error is signaled if the actual hash of the file does not match the specified hash. This command prints the hash on standard output. Additionally, @@ -91,7 +93,7 @@ downloaded file in the Nix store is also printed. - + name Override the name of the file in the Nix store. By default, this is diff --git a/doc/manual/command-ref/nix-shell.xml b/doc/manual/command-ref/nix-shell.xml index c64c93ec3ac..2fef323c53d 100644 --- a/doc/manual/command-ref/nix-shell.xml +++ b/doc/manual/command-ref/nix-shell.xml @@ -32,14 +32,20 @@ cmd regexp + name - - + + - packages - + + + packages + expressions + + + path @@ -144,7 +150,7 @@ also . - / + / packages… Set up an environment in which the specified packages are present. The command line arguments are interpreted @@ -165,6 +171,13 @@ also . + name + + When a shell is started, + keep the listed environment variables. + + + The following common options are supported: @@ -181,8 +194,8 @@ also . NIX_BUILD_SHELL - - Shell used to start the interactive environment. + + Shell used to start the interactive environment. Defaults to the bash found in PATH. @@ -214,8 +227,9 @@ $ nix-shell '<nixpkgs>' -A pan --pure \ --command 'export NIX_DEBUG=1; export NIX_CORES=8; return' -Nix expressions can also be given on the command line. For instance, -the following starts a shell containing the packages +Nix expressions can also be given on the command line using the +-E and -p flags. +For instance, the following starts a shell containing the packages sqlite and libX11: @@ -230,13 +244,21 @@ $ nix-shell -p sqlite xorg.libX11 … -L/nix/store/j1zg5v…-sqlite-3.8.0.2/lib -L/nix/store/0gmcz9…-libX11-1.6.1/lib … +Note that -p accepts multiple full nix expressions that +are valid in the buildInputs = [ ... ] shown above, +not only package names. So the following is also legal: + + +$ nix-shell -p sqlite 'git.override { withManual = false; }' + + The -p flag looks up Nixpkgs in the Nix search path. You can override it by passing or setting NIX_PATH. For example, the following gives you a shell containing the Pan package from a specific revision of Nixpkgs: -$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz +$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz [nix-shell:~]$ pan --version Pan 0.139 @@ -309,13 +331,28 @@ while (my $token = $p->get_tag("a")) { -Finally, the following Haskell script uses a specific branch of -Nixpkgs/NixOS (the 14.12 stable branch): +Sometimes you need to pass a simple Nix expression to customize +a package like Terraform: + + + +You must use double quotes (") when +passing a simple Nix expression in a nix-shell shebang. + + +Finally, using the merging of multiple nix-shell shebangs the +following Haskell script uses a specific branch of Nixpkgs/NixOS (the +18.03 stable branch): -#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/0672315759b3e15e2121365f067c1c8c56bb4722.tar.gz +#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/0672315759b3e15e2121365f067c1c8c56bb4722.tar.gz diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index 0f6172defb3..d71f9d8e47d 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -204,7 +204,7 @@ printed.) with , if an output path is not identical to the corresponding output from the previous build, the new output path is left in - /nix/store/name-check. + /nix/store/name.check. See also the configuration option, which repeats a derivation a number of times and prevents @@ -215,6 +215,48 @@ printed.) +Special exit codes: + + + + 100 + Generic build failure, the builder process + returned with a non-zero exit code. + + + 101 + Build timeout, the build was aborted because it + did not complete within the specified timeout. + + + + 102 + Hash mismatch, the build output was rejected + because it does not match the specified outputHash. + + + + 104 + Not deterministic, the build succeeded in check + mode but the resulting output is not binary reproducable. + + + + + +With the flag it's possible for +multiple failures to occur, in this case the 1xx status codes are or combined +using binary or. +1100100 + ^^^^ + |||`- timeout + ||`-- output hash mismatch + |`--- build failure + `---- not deterministic + + @@ -234,7 +276,7 @@ linkend="sec-nix-build">nix-build does. To test whether a previously-built derivation is deterministic: -$ nix-build -r '<nixpkgs>' -A hello --check -K +$ nix-build '<nixpkgs>' -A hello --check -K @@ -275,7 +317,7 @@ as a means of providing Nix store access to a restricted ssh user. Allow the connected client to request the realization of derivations. In effect, this can be used to make the host act - as a build slave. + as a remote builder. @@ -318,7 +360,6 @@ EOF - bytes @@ -365,14 +406,6 @@ the Nix store not reachable via file system references from a set of - - - This operation performs an actual garbage - collection. All dead paths are removed from the - store. This is the default. - - - By default, all unreachable paths are deleted. The following @@ -397,15 +430,15 @@ options control what gets deleted and in what order: The behaviour of the collector is also influenced by the gc-keep-outputs +linkend="conf-keep-outputs">keep-outputs and gc-keep-derivations +linkend="conf-keep-derivations">keep-derivations variables in the Nix configuration file. -With , the collector prints the total -number of freed bytes when it finishes (or when it is interrupted). -With , it prints the number of bytes that -would be freed. +By default, the collector prints the total number of freed bytes +when it finishes (or when it is interrupted). With +, it prints the number of bytes that would +be freed. @@ -501,10 +534,11 @@ error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4' - + name + name @@ -642,6 +676,7 @@ query is applied to the target of the symlink. + Prints the deriver of the store paths @@ -677,7 +712,20 @@ query is applied to the target of the symlink. + + + Prints the references graph of the store paths + paths in the GraphML file format. + This can be used to visualise dependency graphs. To obtain a + build-time dependency graph, apply this to a store derivation. To + obtain a runtime dependency graph, apply it to an output + path. + + + name + name Prints the value of the attribute name (i.e., environment variable) of @@ -868,6 +916,60 @@ $ nix-store --add ./foo.c + + +Operation <option>--add-fixed</option> + +Synopsis + + + nix-store + + + algorithm + paths + + + + +Description + +The operation adds the specified paths to +the Nix store. Unlike paths are registered using the +specified hashing algorithm, resulting in the same output path as a fixed-output +derivation. This can be used for sources that are not available from a public +url or broke since the download expression was written. + + +This operation has the following options: + + + + + + + Use recursive instead of flat hashing mode, used when adding directories + to the store. + + + + + + + + + + +Example + + +$ nix-store --add-fixed sha256 ./hello-2.10.tar.gz +/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz + + + + + @@ -1037,7 +1139,7 @@ the information that Nix considers important. For instance, timestamps are elided because all files in the Nix store have their timestamp set to 0 anyway. Likewise, all permissions are left out except for the execute bit, because all files in the Nix store have -644 or 755 permission. +444 or 555 permission. Also, a NAR archive is canonical, meaning that “equal” paths always produce the same NAR archive. For instance, @@ -1236,12 +1338,7 @@ the store path is used. /nix/var/log/nix/drvs. However, there is no guarantee that a build log is available for any particular store path. For instance, if the path was downloaded as a pre-built binary through -a substitute, then the log is unavailable. If the log is not available -locally, then nix-store will try to download the -log from the servers specified in the Nix option -. For example, if it’s set to -http://hydra.nixos.org/log, then Nix will check -http://hydra.nixos.org/log/base-name. +a substitute, then the log is unavailable. @@ -1272,6 +1369,7 @@ ktorrent-2.2.1/NEWS nix-store + paths @@ -1282,6 +1380,13 @@ Nix database to standard output. It can be loaded into an empty Nix store using . This is useful for making backups and when migrating to different database schemas. +By default, will dump the entire Nix +database. When one or more store paths is passed, only the subset of +the Nix database for those store paths is dumped. As with +, the user is responsible for passing all the +store paths for a closure. See for an +example. + diff --git a/doc/manual/command-ref/opt-common-syn.xml b/doc/manual/command-ref/opt-common-syn.xml index 5b793639395..2660e3bb1cc 100644 --- a/doc/manual/command-ref/opt-common-syn.xml +++ b/doc/manual/command-ref/opt-common-syn.xml @@ -1,11 +1,26 @@ - + - - - - + + + + + + + + + + + + format + + + + + + + @@ -25,13 +40,20 @@ number - - - - + + + + + + + + + + + + - path diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml index 2a076877a1b..a68eef1d0e7 100644 --- a/doc/manual/command-ref/opt-common.xml +++ b/doc/manual/command-ref/opt-common.xml @@ -22,8 +22,7 @@ - - + / @@ -76,8 +75,55 @@ - - + + + + + Decreases the level of verbosity of diagnostic messages + printed on standard error. This is the inverse option to + / . + + + This option may be specified repeatedly. See the previous + verbosity levels list. + + + + + + + format + + + + This option can be used to change the output of the log format, with + format being one of: + + + + raw + This is the raw format, as outputted by nix-build. + + + internal-json + Outputs the logs in a structured manner. NOTE: the json schema is not guarantees to be stable between releases. + + + bar + Only display a progress bar during the builds. + + + bar-with-logs + Display the raw logs, with the progress bar at the bottom. + + + + + + + + + / By default, output written by builders to standard output and standard error is echoed to the Nix command's standard @@ -89,16 +135,25 @@ - - + / +number - Sets the maximum number of build jobs that Nix will - perform in parallel to the specified number. The default is - specified by the build-max-jobs + + + Sets the maximum number of build jobs that Nix will + perform in parallel to the specified number. Specify + auto to use the number of CPUs in the system. + The default is specified by the max-jobs configuration setting, which itself defaults to 1. A higher value is useful on SMP systems or to - exploit I/O latency. + exploit I/O latency. + + Setting it to 0 disallows building on the local + machine, which is useful when you want builds to happen only on remote + builders. + + @@ -113,7 +168,7 @@ true, the builder passes the flag to GNU Make. It defaults to the value of the build-cores + linkend='conf-cores'>cores configuration setting, if set, or 1 otherwise. The value 0 means that the builder should use all available CPU cores in the system. @@ -126,7 +181,7 @@ Sets the maximum number of seconds that a builder can go without producing any data on standard output or standard error. The default is specified by the build-max-silent-time + linkend='conf-max-silent-time'>max-silent-time configuration setting. 0 means no time-out. @@ -136,14 +191,13 @@ Sets the maximum number of seconds that a builder can run. The default is specified by the build-timeout + linkend='conf-timeout'>timeout configuration setting. 0 means no timeout. - - + / Keep going in case of failed builds, to the greatest extent possible. That is, if building an input of some @@ -155,8 +209,7 @@ - - + / Specifies that in case of a build failure, the temporary directory (usually in /tmp) in which @@ -221,9 +274,10 @@ name value This option is accepted by - nix-env, nix-instantiate and - nix-build. When evaluating Nix expressions, the - expression evaluator will automatically try to call functions that + nix-env, nix-instantiate, + nix-shell and nix-build. + When evaluating Nix expressions, the expression evaluator will + automatically try to call functions that it encounters. It can automatically call functions for which every argument has a default value (e.g., { argName ? @@ -300,14 +354,14 @@ Nix expressions to be parsed and evaluated, rather than as a list of file names of Nix expressions. (nix-instantiate, nix-build - and nix-shell only.) - - - - - - Causes Nix to print out a stack trace in case of Nix - expression evaluation errors. + and nix-shell only.) + + For nix-shell, this option is commonly used + to give you a shell in which you can build the packages returned + by the expression. If you want to get a shell which contain the + built packages ready for use, give your + expression to the nix-shell -p convenience flag + instead. diff --git a/doc/manual/expressions/advanced-attributes.xml b/doc/manual/expressions/advanced-attributes.xml index 83ad6eefc8b..5759ff50e73 100644 --- a/doc/manual/expressions/advanced-attributes.xml +++ b/doc/manual/expressions/advanced-attributes.xml @@ -11,7 +11,7 @@ attributes. - allowedReferences + allowedReferences The optional attribute allowedReferences specifies a list of legal @@ -32,7 +32,7 @@ allowedReferences = []; - allowedRequisites + allowedRequisites This attribute is similar to allowedReferences, but it specifies the legal @@ -50,8 +50,42 @@ allowedRequisites = [ foobar ]; + disallowedReferences + + The optional attribute + disallowedReferences specifies a list of illegal + references (dependencies) of the output of the builder. For + example, + + +disallowedReferences = [ foo ]; + + + enforces that the output of a derivation cannot have a direct runtime + dependencies on the derivation foo. + + + + + disallowedRequisites + + This attribute is similar to + disallowedReferences, but it specifies illegal + requisites for the whole closure, so all the dependencies + recursively. For example, + + +disallowedRequisites = [ foobar ]; + + + enforces that the output of a derivation cannot have any + runtime dependency on foobar or any other derivation + depending recursively on foobar. + + - exportReferencesGraph + + exportReferencesGraph This attribute allows builders access to the references graph of their inputs. The attribute is a list of @@ -90,7 +124,7 @@ derivation { - impureEnvVars + impureEnvVars This attribute allows you to specify a list of environment variables that should be passed from the environment @@ -112,15 +146,21 @@ impureEnvVars = [ "http_proxy" "https_proxy" ... ]; linkend="fixed-output-drvs">fixed-output derivations, where impurities such as these are okay since (the hash of) the output is known in advance. It is ignored for all other - derivations. + derivations. + + impureEnvVars implementation takes + environment variables from the current builder process. When a daemon is + building its environmental variables are used. Without the daemon, the + environmental variables come from the environment of the + nix-build. - outputHash - outputHashAlgo - outputHashMode + outputHash + outputHashAlgo + outputHashMode These attributes declare that the derivation is a so-called fixed-output derivation, which @@ -138,8 +178,8 @@ impureEnvVars = [ "http_proxy" "https_proxy" ... ]; fetchurl { - url = http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz; - md5 = "70c9ccf9fac07f762c24f2df2290784d"; + url = "http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz"; + sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; } @@ -149,8 +189,8 @@ fetchurl { fetchurl { - url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz; - md5 = "70c9ccf9fac07f762c24f2df2290784d"; + url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz"; + sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; } @@ -176,7 +216,7 @@ fetchurl { { stdenv, curl }: # The curl program is used for downloading. -{ url, md5 }: +{ url, sha256 }: stdenv.mkDerivation { name = baseNameOf (toString url); @@ -184,10 +224,10 @@ stdenv.mkDerivation { buildInputs = [ curl ]; # This is a fixed-output derivation; the output must be a regular - # file with MD5 hash md5. + # file with SHA256 hash sha256. outputHashMode = "flat"; - outputHashAlgo = "md5"; - outputHash = md5; + outputHashAlgo = "sha256"; + outputHash = sha256; inherit url; } @@ -197,8 +237,8 @@ stdenv.mkDerivation { The outputHashAlgo attribute specifies the hash algorithm used to compute the hash. It can currently be - "md5", "sha1" or - "sha256". + "sha1", "sha256" or + "sha512". The outputHashMode attribute determines how the hash is computed. It must be one of the following two @@ -211,7 +251,7 @@ stdenv.mkDerivation { The output must be a non-executable regular file. If it isn’t, the build fails. The hash is simply computed over the contents of that file (so it’s equal to what - Unix commands like md5sum or + Unix commands like sha256sum or sha1sum produce). This is the default. @@ -242,7 +282,7 @@ stdenv.mkDerivation { - passAsFile + passAsFile A list of names of attributes that should be passed via files rather than environment variables. For example, @@ -269,12 +309,10 @@ big = "a very long string"; - preferLocalBuild + preferLocalBuild If this attribute is set to - true, it has two effects. First, the - derivation will always be built, not substituted, even if a - substitute is available. Second, if true and distributed building is enabled, then, if possible, the derivaton will be built locally instead of forwarded to a remote machine. This is @@ -284,6 +322,30 @@ big = "a very long string"; + + allowSubstitutes + + + If this attribute is set to + false, then Nix will always build this + derivation; it will not try to substitute its outputs. This is + useful for very trivial derivations (such as + writeText in Nixpkgs) that are cheaper to + build than to substitute from a binary cache. + + You need to have a builder configured which satisfies + the derivation’s system attribute, since the + derivation cannot be substituted. Thus it is usually a good idea + to align system with + builtins.currentSystem when setting + allowSubstitutes to false. + For most trivial derivations this should be the case. + + + + + + diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 063bc04be48..a18c5801a62 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -21,7 +21,9 @@ available as builtins.derivation. - abort s + + abort s + builtins.abort s Abort Nix expression evaluation, print error message s. @@ -29,8 +31,10 @@ available as builtins.derivation. - builtins.add - e1 e2 + + builtins.add + e1 e2 + Return the sum of the numbers e1 and @@ -39,8 +43,9 @@ available as builtins.derivation. - builtins.all - pred list + + builtins.all + pred list Return true if the function pred returns true @@ -50,8 +55,9 @@ available as builtins.derivation. - builtins.any - pred list + + builtins.any + pred list Return true if the function pred returns true @@ -61,19 +67,21 @@ available as builtins.derivation. - builtins.attrNames - set + + builtins.attrNames + set Return the names of the attributes in the set - set in a sorted list. For instance, + set in an alphabetically sorted list. For instance, builtins.attrNames { y = 1; x = "foo"; } evaluates to [ "x" "y" ]. - builtins.attrValues - set + + builtins.attrValues + set Return the values of the attributes in the set set in the order corresponding to the @@ -82,7 +90,8 @@ available as builtins.derivation. - baseNameOf s + + baseNameOf s Return the base name of the string s, that is, everything following @@ -92,7 +101,41 @@ available as builtins.derivation. - builtins + + builtins.bitAnd + e1 e2 + + Return the bitwise AND of the integers + e1 and + e2. + + + + + + builtins.bitOr + e1 e2 + + Return the bitwise OR of the integers + e1 and + e2. + + + + + + builtins.bitXor + e1 e2 + + Return the bitwise XOR of the integers + e1 and + e2. + + + + + + builtins The set builtins contains all the built-in functions and values. You can use @@ -109,8 +152,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else "" - builtins.compareVersions - s1 s2 + + builtins.compareVersions + s1 s2 Compare two strings representing versions and return -1 if version @@ -126,17 +170,27 @@ if builtins ? getEnv then builtins.getEnv "PATH" else "" - builtins.concatLists - lists + + builtins.concatLists + lists Concatenate a list of lists into a single list. + + builtins.concatStringsSep + separator list - builtins.currentSystem + Concatenate a list of strings with a separator + between each element, e.g. concatStringsSep "/" + ["usr" "local" "bin"] == "usr/local/bin" + + + + + builtins.currentSystem The built-in value currentSystem evaluates to the Nix platform identifier for the Nix installation @@ -169,8 +223,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else "" --> - builtins.deepSeq - e1 e2 + + builtins.deepSeq + e1 e2 This is like seq e1 @@ -182,8 +237,11 @@ if builtins ? getEnv then builtins.getEnv "PATH" else "" - derivation - attrs + + derivation + attrs + builtins.derivation + attrs derivation is described in . @@ -191,7 +249,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else "" - dirOf s + + dirOf s + builtins.dirOf s Return the directory part of the string s, that is, everything before the final @@ -201,8 +261,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else "" - builtins.div - e1 e2 + + builtins.div + e1 e2 Return the quotient of the numbers e1 and @@ -210,38 +271,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else "" - builtins.match - regex str - - Returns a list if - regex matches - str precisely, otherwise returns null. - Each item in the list is a regex group. - - -builtins.match "ab" "abc" - - -Evaluates to null. - - -builtins.match "abc" "abc" - - -Evaluates to [ ]. - - -builtins.match "a(b)(c)" "abc" - - -Evaluates to [ "b" "c" ]. - - - - - - builtins.elem - x xs + + builtins.elem + x xs Return true if a value equal to x occurs in the list @@ -251,19 +283,21 @@ Evaluates to [ "b" "c" ]. - builtins.elemAt - xs n + + builtins.elemAt + xs n Return element n from the list xs. Elements are counted - starting from 0. A fatal error occurs in the index is out of + starting from 0. A fatal error occurs if the index is out of bounds. - builtins.fetchurl - url + + builtins.fetchurl + url Download the specified URL and return the path of the downloaded file. This function is not available if [ "b" "c" ]. - fetchTarball - url + + fetchTarball + url + builtins.fetchTarball + url Download the specified URL, unpack it and return the path of the unpacked tree. The file must be a tape archive @@ -287,7 +324,34 @@ Evaluates to [ "b" "c" ]. particular version of Nixpkgs, e.g. -with import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz) {}; +with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {}; + +stdenv.mkDerivation { … } + + + + The fetched tarball is cached for a certain amount of time + (1 hour by default) in ~/.cache/nix/tarballs/. + You can change the cache timeout either on the command line with + or + in the Nix configuration file with this option: + number of seconds to cache. + + + Note that when obtaining the hash with nix-prefetch-url + the option --unpack is required. + + + This function can also verify the contents against a hash. + In that case, the function takes a set instead of a URL. The set + requires the attribute url and the attribute + sha256, e.g. + + +with import (fetchTarball { + url = "https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz"; + sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2"; +}) {}; stdenv.mkDerivation { … } @@ -300,6 +364,161 @@ stdenv.mkDerivation { … } + + + builtins.fetchGit + args + + + + + Fetch a path from git. args can be + a URL, in which case the HEAD of the repo at that URL is + fetched. Otherwise, it can be an attribute with the following + attributes (all except url optional): + + + + + url + + + The URL of the repo. + + + + + name + + + The name of the directory the repo should be exported to + in the store. Defaults to the basename of the URL. + + + + + rev + + + The git revision to fetch. Defaults to the tip of + ref. + + + + + ref + + + The git ref to look for the requested revision under. + This is often a branch or tag name. Defaults to + HEAD. + + + + By default, the ref value is prefixed + with refs/heads/. As of Nix 2.3.0 + Nix will not prefix refs/heads/ if + ref starts with refs/. + + + + + submodules + + + A Boolean parameter that specifies whether submodules + should be checked out. Defaults to + false. + + + + + + + Fetching a private repository over SSH + builtins.fetchGit { + url = "git@github.com:my-secret/repository.git"; + ref = "master"; + rev = "adab8b916a45068c044658c4158d81878f9ed1c3"; +} + + + + Fetching an arbitrary ref + builtins.fetchGit { + url = "https://github.com/NixOS/nix.git"; + ref = "refs/heads/0.5-release"; +} + + + + Fetching a repository's specific commit on an arbitrary branch + + If the revision you're looking for is in the default branch + of the git repository you don't strictly need to specify + the branch name in the ref attribute. + + + However, if the revision you're looking for is in a future + branch for the non-default branch you will need to specify + the the ref attribute as well. + + builtins.fetchGit { + url = "https://github.com/nixos/nix.git"; + rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452"; + ref = "1.11-maintenance"; +} + + + It is nice to always specify the branch which a revision + belongs to. Without the branch being specified, the + fetcher might fail if the default branch changes. + Additionally, it can be confusing to try a commit from a + non-default branch and see the fetch fail. If the branch + is specified the fault is much more obvious. + + + + + + Fetching a repository's specific commit on the default branch + + If the revision you're looking for is in the default branch + of the git repository you may omit the + ref attribute. + + builtins.fetchGit { + url = "https://github.com/nixos/nix.git"; + rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452"; +} + + + + Fetching a tag + builtins.fetchGit { + url = "https://github.com/nixos/nix.git"; + ref = "refs/tags/1.9"; +} + + + + Fetching the latest version of a remote branch + + builtins.fetchGit can behave impurely + fetch the latest version of a remote branch. + + Nix will refetch the branch in accordance to + . + This behavior is disabled in + Pure evaluation mode. + builtins.fetchGit { + url = "ssh://git@github.com/nixos/nix.git"; + ref = "master"; +} + + + + builtins.filter f xs @@ -312,8 +531,9 @@ stdenv.mkDerivation { … } - builtins.filterSource - e1 e2 + + builtins.filterSource + e1 e2 @@ -357,14 +577,17 @@ stdenv.mkDerivation { "unknown" (for other kinds of files such as device nodes or fifos — but note that those cannot be copied to the Nix store, so if the predicate returns - true for them, the copy will fail). + true for them, the copy will fail). If you + exclude a directory, the entire corresponding subtree of + e2 will be excluded. - builtins.foldl’ + + builtins.foldl’ op nul list Reduce a list by applying a binary operator, from @@ -377,7 +600,8 @@ stdenv.mkDerivation { - builtins.functionArgs + + builtins.functionArgs f @@ -395,7 +619,8 @@ stdenv.mkDerivation { - builtins.fromJSON e + + builtins.fromJSON e Convert a JSON string to a Nix value. For example, @@ -405,18 +630,18 @@ builtins.fromJSON ''{"x": [1, 2, 3], "y": null}'' returns the value { x = [ 1 2 3 ]; y = null; - }. Floating point numbers are not - supported. + }. - builtins.genList - generator length + + builtins.genList + generator length Generate list of size length, with each element - i> equal to the value returned by + i equal to the value returned by generator i. For example, @@ -429,8 +654,9 @@ builtins.genList (x: x * x) 5 - builtins.getAttr - s set + + builtins.getAttr + s set getAttr returns the attribute named s from @@ -442,8 +668,9 @@ builtins.genList (x: x * x) 5 - builtins.getEnv - s + + builtins.getEnv + s getEnv returns the value of the environment variable s, or an empty @@ -460,8 +687,9 @@ builtins.genList (x: x * x) 5 - builtins.hasAttr - s set + + builtins.hasAttr + s set hasAttr returns true if set has an @@ -474,20 +702,35 @@ builtins.genList (x: x * x) 5 - builtins.hashString - type s + + builtins.hashString + type s Return a base-16 representation of the cryptographic hash of string s. The hash algorithm specified by type must - be one of "md5", "sha1" or - "sha256". + be one of "md5", "sha1", + "sha256" or "sha512". + + + + + + builtins.hashFile + type p + + Return a base-16 representation of the + cryptographic hash of the file at path p. The + hash algorithm specified by type must + be one of "md5", "sha1", + "sha256" or "sha512". - builtins.head - list + + builtins.head + list Return the first element of a list; abort evaluation if the argument isn’t a list or is an empty list. You @@ -497,8 +740,11 @@ builtins.genList (x: x * x) 5 - import - path + + import + path + builtins.import + path Load, parse and return the Nix expression in the file path. If path @@ -510,6 +756,11 @@ builtins.genList (x: x * x) 5 separate file, and use it from Nix expressions in other files. + Unlike some languages, import is a regular + function in Nix. Paths using the angle bracket syntax (e.g., + import <foo>) are normal path + values (see ). + A Nix expression loaded by import must not contain any free variables (identifiers that are not defined in the Nix expression itself and are not @@ -552,8 +803,9 @@ x: x + 456 - builtins.intersectAttrs - e1 e2 + + builtins.intersectAttrs + e1 e2 Return a set consisting of the attributes in the set e2 that also exist in the set @@ -562,8 +814,9 @@ x: x + 456 - builtins.isAttrs - e + + builtins.isAttrs + e Return true if e evaluates to a set, and @@ -572,8 +825,9 @@ x: x + 456 - builtins.isList - e + + builtins.isList + e Return true if e evaluates to a list, and @@ -582,7 +836,7 @@ x: x + 456 - builtins.isFunction + builtins.isFunction e Return true if @@ -592,8 +846,9 @@ x: x + 456 - builtins.isString - e + + builtins.isString + e Return true if e evaluates to a string, and @@ -602,8 +857,9 @@ x: x + 456 - builtins.isInt - e + + builtins.isInt + e Return true if e evaluates to an int, and @@ -612,19 +868,42 @@ x: x + 456 - builtins.isBool - e + + builtins.isFloat + e Return true if - e evaluates to a bool, and + e evaluates to a float, and false otherwise. - isNull + + builtins.isBool + e + + Return true if + e evaluates to a bool, and + false otherwise. + + + + builtins.isPath e + Return true if + e evaluates to a path, and + false otherwise. + + + + + isNull + e + builtins.isNull + e + Return true if e evaluates to null, and false otherwise. @@ -637,8 +916,9 @@ x: x + 456 - builtins.length - e + + builtins.length + e Return the length of the list e. @@ -646,8 +926,9 @@ x: x + 456 - builtins.lessThan - e1 e2 + + builtins.lessThan + e1 e2 Return true if the number e1 is less than the number @@ -659,8 +940,9 @@ x: x + 456 - builtins.listToAttrs - e + + builtins.listToAttrs + e Construct a set from a list specifying the names and values of each attribute. Each element of the list should be @@ -686,8 +968,11 @@ builtins.listToAttrs - map - f list + + map + f list + builtins.map + f list Apply the function f to each element in the list list. For @@ -702,8 +987,46 @@ map (x: "foo" + x) [ "bar" "bla" "abc" ] - builtins.mul - e1 e2 + + builtins.match + regex str + + Returns a list if the extended + POSIX regular expression regex + matches str precisely, otherwise returns + null. Each item in the list is a regex group. + + +builtins.match "ab" "abc" + + +Evaluates to null. + + +builtins.match "abc" "abc" + + +Evaluates to [ ]. + + +builtins.match "a(b)(c)" "abc" + + +Evaluates to [ "b" "c" ]. + + +builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" " FOO " + + +Evaluates to [ "foo" ]. + + + + + + builtins.mul + e1 e2 Return the product of the numbers e1 and @@ -712,8 +1035,9 @@ map (x: "foo" + x) [ "bar" "bla" "abc" ] - builtins.parseDrvName - s + + builtins.parseDrvName + s Split the string s into a package name and version. The package name is everything up to @@ -726,33 +1050,100 @@ map (x: "foo" + x) [ "bar" "bla" "abc" ] + + + builtins.path + args + - builtins.pathExists - path + + + An enrichment of the built-in path type, based on the attributes + present in args. All are optional + except path: + - Return true if the path - path exists, and - false otherwise. One application of this - function is to conditionally include a Nix expression containing - user configuration: + + + path + + The underlying path. + + + + name + + + The name of the path when added to the store. This can + used to reference paths that have nix-illegal characters + in their names, like @. + + + + + filter + + + A function of the type expected by + builtins.filterSource, + with the same semantics. + + + + + recursive + + + When false, when + path is added to the store it is with a + flat hash, rather than a hash of the NAR serialization of + the file. Thus, path must refer to a + regular file, not a directory. This allows similar + behavior to fetchurl. Defaults to + true. + + + + + sha256 + + + When provided, this is the expected hash of the file at + the path. Evaluation will fail if the hash is incorrect, + and providing a hash allows + builtins.path to be used even when the + pure-eval nix config option is on. + + + + + + - -let - fileName = builtins.getEnv "CONFIG_FILE"; - config = - if fileName != "" && builtins.pathExists (builtins.toPath fileName) - then import (builtins.toPath fileName) - else { someSetting = false; }; # default configuration -in config.someSetting + + builtins.pathExists + path - (Note that CONFIG_FILE must be an absolute path for - this to work.) + Return true if the path + path exists at evaluation time, and + false otherwise. + + builtins.placeholder + output - builtins.readDir - path + Return a placeholder string for the specified + output that will be substituted by the + corresponding output path at build time. Typical outputs would be + "out", "bin" or + "dev". + + + + builtins.readDir + path Return the contents of the directory path as a set mapping directory entries @@ -773,8 +1164,9 @@ in config.someSetting - builtins.readFile - path + + builtins.readFile + path Return the contents of the file path as a string. @@ -782,8 +1174,11 @@ in config.someSetting - removeAttrs - set list + + removeAttrs + set list + builtins.removeAttrs + set list Remove the attributes listed in list from @@ -798,8 +1193,9 @@ removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ] - builtins.replaceStrings - from to s + + builtins.replaceStrings + from to s Given string s, replace every occurrence of the strings in from @@ -815,8 +1211,9 @@ builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar" - builtins.seq - e1 e2 + + builtins.seq + e1 e2 Evaluate e1, then evaluate and return e2. This ensures @@ -826,8 +1223,9 @@ builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar" - builtins.sort - comparator list + + builtins.sort + comparator list Return list in sorted order. It repeatedly calls the function @@ -849,8 +1247,60 @@ builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ] - builtins.stringLength - e + + builtins.split + regex str + + Returns a list composed of non matched strings interleaved + with the lists of the extended + POSIX regular expression regex matches + of str. Each item in the lists of matched + sequences is a regex group. + + +builtins.split "(a)b" "abc" + + +Evaluates to [ "" [ "a" ] "c" ]. + + +builtins.split "([ac])" "abc" + + +Evaluates to [ "" [ "a" ] "b" [ "c" ] "" ]. + + +builtins.split "(a)|(c)" "abc" + + +Evaluates to [ "" [ "a" null ] "b" [ null "c" ] "" ]. + + +builtins.split "([[:upper:]]+)" " FOO " + + +Evaluates to [ " " [ "FOO" ] " " ]. + + + + + + + builtins.splitVersion + s + + Split a string representing a version into its + components, by the same version splitting logic underlying the + version comparison in + nix-env -u. + + + + + + builtins.stringLength + e Return the length of the string e. If e is @@ -859,8 +1309,9 @@ builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ] - builtins.sub - e1 e2 + + builtins.sub + e1 e2 Return the difference between the numbers e1 and @@ -869,9 +1320,10 @@ builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ] - builtins.substring - start len - s + + builtins.substring + start len + s Return the substring of s from character position @@ -894,8 +1346,9 @@ builtins.substring 0 3 "nixos" - builtins.tail - list + + builtins.tail + list Return the second to last elements of a list; abort evaluation if the argument isn’t a list or is an empty @@ -904,8 +1357,11 @@ builtins.substring 0 3 "nixos" - throw - s + + throw + s + builtins.throw + s Throw an error message s. This usually aborts Nix expression @@ -918,9 +1374,10 @@ builtins.substring 0 3 "nixos" - builtins.toFile - name s + + builtins.toFile + name + s Store the string s in a file in the Nix store and return its path. The file has suffix @@ -949,8 +1406,8 @@ stdenv.mkDerivation { "; src = fetchurl { - url = http://nix.cs.uu.nl/dist/tarballs/hello-2.1.1.tar.gz; - md5 = "70c9ccf9fac07f762c24f2df2290784d"; + url = "http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz"; + sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; }; inherit perl; } @@ -988,12 +1445,16 @@ in foo This is not allowed because it would cause a cyclic dependency in the computation of the cryptographic hashes for - foo and bar. + foo and bar. + It is also not possible to reference the result of a derivation. + If you are using Nixpkgs, the writeTextFile function is able to + do that. - builtins.toJSON e + + builtins.toJSON e Return a string containing a JSON representation of e. Strings, integers, floats, booleans, @@ -1006,32 +1467,40 @@ in foo - builtins.toPath s + + builtins.toPath s - Convert the string value - s into a path value. The string - s must represent an absolute path - (i.e., must start with /). The path need not - exist. The resulting path is canonicalised, e.g., - builtins.toPath "//foo/xyzzy/../bar/" returns - /foo/bar. + DEPRECATED. Use /. + "/path" + to convert a string into an absolute path. For relative paths, + use ./. + "/path". + - toString e + + toString e + builtins.toString e Convert the expression e to a string. - e can be a string (in which case - toString is a no-op), a path (e.g., - toString /foo/bar yields - "/foo/bar" or a set containing { __toString = self: ...; }. + e can be: + + A string (in which case the string is returned unmodified). + A path (e.g., toString /foo/bar yields "/foo/bar". + A set containing { __toString = self: ...; }. + An integer. + A list, in which case the string representations of its elements are joined with spaces. + A Boolean (false yields "", true yields "1"). + null, which yields the empty string. + + - builtins.toXML e + + builtins.toXML e Return a string containing an XML representation of e. The main application for @@ -1087,7 +1556,7 @@ stdenv.mkDerivation (rec { builder = builtins.toFile "builder.sh" " source $stdenv/setup mkdir $out - echo $servlets | xsltproc ${stylesheet} - > $out/server-conf.xml]]> $out/server-conf.xml]]> - builtins.trace - e1 e2 + + builtins.trace + e1 e2 Evaluate e1 and print its abstract syntax representation on standard error. Then return @@ -1156,16 +1626,38 @@ stdenv.mkDerivation (rec { + + builtins.tryEval + e - builtins.typeOf - e + Try to shallowly evaluate e. + Return a set containing the attributes success + (true if e evaluated + successfully, false if an error was thrown) and + value, equalling e + if successful and false otherwise. Note that this + doesn't evaluate e deeply, so + let e = { x = throw ""; }; in (builtins.tryEval e).success + will be true. Using builtins.deepSeq + one can get the expected result: let e = { x = throw ""; + }; in (builtins.tryEval (builtins.deepSeq e e)).success will be + false. + + + + + + + builtins.typeOf + e Return a string representing the type of the value e, namely "int", "bool", "string", "path", "null", - "set", "list" or - "lambda". + "set", "list", + "lambda" or + "float". diff --git a/doc/manual/expressions/debug-build.xml b/doc/manual/expressions/debug-build.xml deleted file mode 100644 index 0c1f4e6719b..00000000000 --- a/doc/manual/expressions/debug-build.xml +++ /dev/null @@ -1,34 +0,0 @@ -
- -Debugging Build Failures - -At the beginning of each phase of the build (such as unpacking, -building or installing), the set of all shell variables is written to -the file env-vars at the top-level build -directory. This is useful for debugging: it allows you to recreate -the environment in which a build was performed. For instance, if a -build fails, then assuming you used the flag, you -can go to the output directory and switch to the -environment of the builder: - - -$ nix-build -K ./foo.nix -... fails, keeping build directory `/tmp/nix-1234-0' - -$ cd /tmp/nix-1234-0 - -$ source env-vars - -(edit some files...) - -$ make - -(execution continues with the same GCC, make, etc.) - - - -
diff --git a/doc/manual/expressions/derivations.xml b/doc/manual/expressions/derivations.xml index 5efe2213e37..6f6297565ca 100644 --- a/doc/manual/expressions/derivations.xml +++ b/doc/manual/expressions/derivations.xml @@ -100,7 +100,7 @@ outputs = [ "lib" "headers" "doc" ]; buildInputs = [ pkg.lib pkg.headers ]; - The first element of output determines the + The first element of outputs determines the default output. Thus, you could also write buildInputs = [ pkg pkg.headers ]; diff --git a/doc/manual/expressions/expression-syntax.xml b/doc/manual/expressions/expression-syntax.xml index 6f1a3a10c4e..a3de207135e 100644 --- a/doc/manual/expressions/expression-syntax.xml +++ b/doc/manual/expressions/expression-syntax.xml @@ -15,8 +15,8 @@ stdenv.mkDerivation { name = "hello-2.1.1"; builder = ./builder.sh; src = fetchurl { - url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz; - md5 = "70c9ccf9fac07f762c24f2df2290784d"; + url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz"; + sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; }; inherit perl; } @@ -108,7 +108,7 @@ the single Nix expression in that directory The builder has to know what the sources of the package are. Here, the attribute src is bound to the result of a call to the fetchurl function. - Given a URL and an MD5 hash of the expected contents of the file + Given a URL and a SHA-256 hash of the expected contents of the file at that URL, this function builds a derivation that downloads the file and checks its hash. So the sources are a dependency that like all other dependencies is built before Hello itself is @@ -145,4 +145,4 @@ perl = perl; - \ No newline at end of file + diff --git a/doc/manual/expressions/language-constructs.xml b/doc/manual/expressions/language-constructs.xml index fe69dba837a..0d0cbbe1553 100644 --- a/doc/manual/expressions/language-constructs.xml +++ b/doc/manual/expressions/language-constructs.xml @@ -41,9 +41,9 @@ encountered).. -Let-expressions +Let-expressions -A let-expression allows you define local variables for an +A let-expression allows you to define local variables for an expression. For instance, @@ -61,7 +61,7 @@ evaluates to "foobar". Inheriting attributes -When defining a set it is often convenient to copy variables +When defining a set or in a let-expression it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). This can be shortened using the inherit keyword. For instance, @@ -72,7 +72,15 @@ let x = 123; in y = 456; } -evaluates to { x = 123; y = 456; }. (Note that +is equivalent to + + +let x = 123; in +{ x = x; + y = 456; +} + +and both evaluate to { x = 123; y = 456; }. (Note that this works because x is added to the lexical scope by the let construct.) It is also possible to inherit attributes from another set. For instance, in this fragment @@ -101,6 +109,26 @@ variables from the surrounding scope (fetchurl libXaw (the X Athena Widgets) from the xlibs (X11 client-side libraries) set. + +Summarizing the fragment + + +... +inherit x y z; +inherit (src-set) a b c; +... + +is equivalent to + + +... +x = x; y = y; z = z; +a = src-set.a; b = src-set.b; c = src-set.c; +... + +when used while defining local variables in a let-expression or +while defining a set. + @@ -189,7 +217,25 @@ but can also be written as: ellipsis(...) as you can access attribute names as a, using args.a, which was given as an additional attribute to the function. - + + + + + The args@ expression is bound to the argument passed to the function which + means that attributes with defaults that aren't explicitly specified in the function call + won't cause an evaluation error, but won't exist in args. + + + For instance + +let + function = args@{ a ? 23, ... }: args; +in + function {} + + will evaluate to an empty attribute set. + + @@ -333,7 +379,20 @@ with (import ./definitions.nix); ... makes all attributes defined in the file definitions.nix available as if they were defined -locally in a rec-expression. +locally in a let-expression. + +The bindings introduced by with do not shadow bindings +introduced by other means, e.g. + + +let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... + +establishes the same scope as + + +let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... + + diff --git a/doc/manual/expressions/language-operators.xml b/doc/manual/expressions/language-operators.xml index a3323ced4c5..4f11bf52938 100644 --- a/doc/manual/expressions/language-operators.xml +++ b/doc/manual/expressions/language-operators.xml @@ -15,13 +15,16 @@ weakest binding). + Name Syntax Associativity Description + Precedence + Select e . attrpath [ or def ] @@ -33,14 +36,25 @@ weakest binding). dot-separated list of attribute names.) If the attribute doesn’t exist, return def if provided, otherwise abort evaluation. + 1 + Application e1 e2 left Call function e1 with argument e2. + 2 + Arithmetic Negation + - e + none + Arithmetic negation. + 3 + + + Has Attribute e ? attrpath none @@ -48,23 +62,69 @@ weakest binding). the attribute denoted by attrpath; return true or false. + 4 + List Concatenation e1 ++ e2 right List concatenation. + 5 - e1 + e2 + Multiplication + + e1 * e2, + left - String or path concatenation. + Arithmetic multiplication. + 6 - ! e + Division + + e1 / e2 + left + Arithmetic division. + 6 + + + Addition + + e1 + e2 + + left + Arithmetic addition. + 7 + + + Subtraction + + e1 - e2 + + left + Arithmetic subtraction. + 7 + + + String Concatenation + + string1 + string2 + + left + String concatenation. + 7 + + + Not + ! e + none Boolean negation. + 8 + Update e1 // e2 right @@ -73,41 +133,90 @@ weakest binding). e2 (with the latter taking precedence over the former in case of equally named attributes). + 9 - e1 == - e2 + Less Than + + e1 < e2, + + none + Arithmetic comparison. + 10 + + + Less Than or Equal To + + e1 <= e2 + + none + Arithmetic comparison. + 10 + + + Greater Than + + e1 > e2 + + none + Arithmetic comparison. + 10 + + + Greater Than or Equal To + + e1 >= e2 + + none + Arithmetic comparison. + 10 + + + Equality + + e1 == e2 + none Equality. + 11 - e1 != - e2 + Inequality + + e1 != e2 + none Inequality. + 11 + Logical AND e1 && e2 left Logical AND. + 12 + Logical OR e1 || e2 left Logical OR. + 13 + Logical Implication e1 -> e2 none Logical implication (equivalent to !e1 || e2). + 14 - \ No newline at end of file + diff --git a/doc/manual/expressions/language-values.xml b/doc/manual/expressions/language-values.xml index 67da688a4fc..bb2090c881f 100644 --- a/doc/manual/expressions/language-values.xml +++ b/doc/manual/expressions/language-values.xml @@ -98,13 +98,17 @@ configureFlags = " Since ${ and '' have special meaning in indented strings, you need a way to quote them. - ${ can be escaped by prefixing it with + $ can be escaped by prefixing it with '' (that is, two single quotes), i.e., - ''${. '' can be escaped by + ''$. '' can be escaped by prefixing it with ', i.e., - '''. Finally, linefeed, carriage-return and - tab characters can be written as ''\n, - ''\r, ''\t. + '''. $ removes any special meaning + from the following $. Linefeed, carriage-return and tab + characters can be written as ''\n, + ''\r, ''\t, and ''\ + escapes any other character. + + Indented strings are primarily useful in that they allow multi-line string literals to follow the indentation of the diff --git a/doc/manual/expressions/simple-building-testing.xml b/doc/manual/expressions/simple-building-testing.xml index bd3901a1335..ce0a1636dc9 100644 --- a/doc/manual/expressions/simple-building-testing.xml +++ b/doc/manual/expressions/simple-building-testing.xml @@ -43,7 +43,7 @@ use nix-build’s switch to give the symlink another name. -Nix has a transactional semantics. Once a build finishes +Nix has transactional semantics. Once a build finishes successfully, Nix makes a note of this in its database: it registers that the path denoted by out is now valid. If you try to build the derivation again, Nix @@ -73,14 +73,4 @@ waiting for lock on `/nix/store/0h5b7hp8d4hqfrw8igvx97x1xawrjnac-hello-2.1.1x'make). -If you have a system with multiple CPUs, you may want to have -Nix build different derivations in parallel (insofar as possible). -Just pass the option , where -N is the maximum number of jobs to be run -in parallel, or set. Typically this should be the number of -CPUs. - - - diff --git a/doc/manual/glossary/glossary.xml b/doc/manual/glossary/glossary.xml index d74940c90b3..e3162ed8d46 100644 --- a/doc/manual/glossary/glossary.xml +++ b/doc/manual/glossary/glossary.xml @@ -1,5 +1,6 @@ + xmlns:xlink="http://www.w3.org/1999/xlink" + xml:id="part-glossary"> Glossary @@ -54,7 +55,7 @@ A substitute is a command invocation stored in the Nix database that describes how to build a store object, bypassing - normal the build mechanism (i.e., derivations). Typically, the + the normal build mechanism (i.e., derivations). Typically, the substitute builds the store object by downloading a pre-built version of the store object from some server. @@ -85,29 +86,48 @@ reference - A store path P is said to have a - reference to a store path Q if the store object - at P contains the path Q - somewhere. This implies than an execution involving - P potentially needs Q to be - present. The references of a store path are - the set of store paths to which it has a reference. + + A store path P is said to have a + reference to a store path Q if the store object + at P contains the path Q + somewhere. The references of a store path are + the set of store paths to which it has a reference. + + A derivation can reference other derivations and sources + (but not output paths), whereas an output path only references other + output paths. + + +reachable + + A store path Q is reachable from + another store path P if Q is in the + closure of the + references relation. + + closure The closure of a store path is the set of store paths that are directly or indirectly “reachable” from that store path; that is, it’s the closure of the path under the references relation. For instance, - if the store object at path P contains a - reference to path Q, then Q is - in the closure of P. For correct deployment it - is necessary to deploy whole closures, since otherwise at runtime - files could be missing. The command nix-store - -qR prints out closures of store paths. + linkend="gloss-reference">references relation. For a package, the + closure of its derivation is equivalent to the build-time + dependencies, while the closure of its output path is equivalent to its + runtime dependencies. For correct deployment it is necessary to deploy whole + closures, since otherwise at runtime files could be missing. The command + nix-store -qR prints out closures of store paths. + + As an example, if the store object at path P contains + a reference to path Q, then Q is + in the closure of P. Further, if Q + references R then R is also in + the closure of P. + @@ -147,7 +167,7 @@ linkend="sec-profiles" />. - + diff --git a/doc/manual/hacking.xml b/doc/manual/hacking.xml index 183aed7adff..b671811d3a3 100644 --- a/doc/manual/hacking.xml +++ b/doc/manual/hacking.xml @@ -30,7 +30,7 @@ To build Nix itself in this shell: [nix-shell]$ configurePhase [nix-shell]$ make -To install it in $(pwd)/nix and test it: +To install it in $(pwd)/inst and test it: [nix-shell]$ make install [nix-shell]$ make installcheck diff --git a/doc/manual/images/callouts/1.gif b/doc/manual/images/callouts/1.gif deleted file mode 100644 index 9e7a87f7546..00000000000 Binary files a/doc/manual/images/callouts/1.gif and /dev/null differ diff --git a/doc/manual/images/callouts/10.gif b/doc/manual/images/callouts/10.gif deleted file mode 100644 index e80f7f8e632..00000000000 Binary files a/doc/manual/images/callouts/10.gif and /dev/null differ diff --git a/doc/manual/images/callouts/11.gif b/doc/manual/images/callouts/11.gif deleted file mode 100644 index 67f91a239d6..00000000000 Binary files a/doc/manual/images/callouts/11.gif and /dev/null differ diff --git a/doc/manual/images/callouts/12.gif b/doc/manual/images/callouts/12.gif deleted file mode 100644 index 54c4b42f190..00000000000 Binary files a/doc/manual/images/callouts/12.gif and /dev/null differ diff --git a/doc/manual/images/callouts/13.gif b/doc/manual/images/callouts/13.gif deleted file mode 100644 index dd5d7d9b643..00000000000 Binary files a/doc/manual/images/callouts/13.gif and /dev/null differ diff --git a/doc/manual/images/callouts/14.gif b/doc/manual/images/callouts/14.gif deleted file mode 100644 index 3d7a952a310..00000000000 Binary files a/doc/manual/images/callouts/14.gif and /dev/null differ diff --git a/doc/manual/images/callouts/15.gif b/doc/manual/images/callouts/15.gif deleted file mode 100644 index 1c9183d5bb6..00000000000 Binary files a/doc/manual/images/callouts/15.gif and /dev/null differ diff --git a/doc/manual/images/callouts/2.gif b/doc/manual/images/callouts/2.gif deleted file mode 100644 index 94d42a30f99..00000000000 Binary files a/doc/manual/images/callouts/2.gif and /dev/null differ diff --git a/doc/manual/images/callouts/3.gif b/doc/manual/images/callouts/3.gif deleted file mode 100644 index dd3541a1bc2..00000000000 Binary files a/doc/manual/images/callouts/3.gif and /dev/null differ diff --git a/doc/manual/images/callouts/4.gif b/doc/manual/images/callouts/4.gif deleted file mode 100644 index 4bcbf7e31a1..00000000000 Binary files a/doc/manual/images/callouts/4.gif and /dev/null differ diff --git a/doc/manual/images/callouts/5.gif b/doc/manual/images/callouts/5.gif deleted file mode 100644 index 1c62b4f9209..00000000000 Binary files a/doc/manual/images/callouts/5.gif and /dev/null differ diff --git a/doc/manual/images/callouts/6.gif b/doc/manual/images/callouts/6.gif deleted file mode 100644 index 23bc5555d2a..00000000000 Binary files a/doc/manual/images/callouts/6.gif and /dev/null differ diff --git a/doc/manual/images/callouts/7.gif b/doc/manual/images/callouts/7.gif deleted file mode 100644 index e55ce89585a..00000000000 Binary files a/doc/manual/images/callouts/7.gif and /dev/null differ diff --git a/doc/manual/images/callouts/8.gif b/doc/manual/images/callouts/8.gif deleted file mode 100644 index 49375e09f4c..00000000000 Binary files a/doc/manual/images/callouts/8.gif and /dev/null differ diff --git a/doc/manual/images/callouts/9.gif b/doc/manual/images/callouts/9.gif deleted file mode 100644 index da12a4fe282..00000000000 Binary files a/doc/manual/images/callouts/9.gif and /dev/null differ diff --git a/doc/manual/installation/env-variables.xml b/doc/manual/installation/env-variables.xml index fc39cdd9dfe..e2b8fc867cd 100644 --- a/doc/manual/installation/env-variables.xml +++ b/doc/manual/installation/env-variables.xml @@ -21,4 +21,69 @@ in your ~/.profile (or similar), like this: source prefix/etc/profile.d/nix.sh - \ No newline at end of file +
+ +<envar>NIX_SSL_CERT_FILE</envar> + +If you need to specify a custom certificate bundle to account +for an HTTPS-intercepting man in the middle proxy, you must specify +the path to the certificate bundle in the environment variable +NIX_SSL_CERT_FILE. + + +If you don't specify a NIX_SSL_CERT_FILE +manually, Nix will install and use its own certificate +bundle. + + + Set the environment variable and install Nix + +$ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt +$ sh <(curl https://nixos.org/nix/install) + + + In the shell profile and rc files (for example, + /etc/bashrc, /etc/zshrc), + add the following line: + +export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt + + + + +You must not add the export and then do the install, as +the Nix installer will detect the presense of Nix configuration, and +abort. + +
+<envar>NIX_SSL_CERT_FILE</envar> with macOS and the Nix daemon + +On macOS you must specify the environment variable for the Nix +daemon service, then restart it: + + +$ sudo launchctl setenv NIX_SSL_CERT_FILE /etc/ssl/my-certificate-bundle.crt +$ sudo launchctl kickstart -k system/org.nixos.nix-daemon + +
+ +
+ +Proxy Environment Variables + +The Nix installer has special handling for these proxy-related +environment variables: +http_proxy, https_proxy, +ftp_proxy, no_proxy, +HTTP_PROXY, HTTPS_PROXY, +FTP_PROXY, NO_PROXY. + +If any of these variables are set when running the Nix installer, +then the installer will create an override file at +/etc/systemd/system/nix-daemon.service.d/override.conf +so nix-daemon will use them. + +
+ +
+ diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml index 2a9beec98c9..8d548f0ea0b 100644 --- a/doc/manual/installation/installing-binary.xml +++ b/doc/manual/installation/installing-binary.xml @@ -6,20 +6,51 @@ Installing a Binary Distribution -If you are using Linux or Mac OS X, the easiest way to install -Nix is to run the following command: + + If you are using Linux or macOS versions up to 10.14 (Mojave), the + easiest way to install Nix is to run the following command: + -$ bash <(curl https://nixos.org/nix/install) + $ sh <(curl https://nixos.org/nix/install) + + + + If you're using macOS 10.15 (Catalina) or newer, consult + the macOS installation instructions + before installing. + + + + As of Nix 2.1.0, the Nix installer will always default to creating a + single-user installation, however opting in to the multi-user + installation is highly recommended. + + + +
+ Single User Installation + + + To explicitly select a single-user installation on your system: + + + sh <(curl https://nixos.org/nix/install) --no-daemon + + This will perform a single-user installation of Nix, meaning that /nix is owned by the invoking user. You should run this under your usual user account, not as root. The script will invoke sudo to create /nix if it doesn’t already exist. If you don’t have sudo, you should manually create -/nix first as root, e.g.: +/nix first as root, e.g.: $ mkdir /nix @@ -30,71 +61,409 @@ The install script will modify the first writable file from amongst .bash_profile, .bash_login and .profile to source ~/.nix-profile/etc/profile.d/nix.sh. You can set -the NIX_INSTALLER_NO_MODIFY_PROFILE environment +the NIX_INSTALLER_NO_MODIFY_PROFILE environment variable before executing the install script to disable this behaviour. - - + + Linux running systemd, with SELinux disabled + + macOS + + + + You can instruct the installer to perform a multi-user + installation on your system: + + + sh <(curl https://nixos.org/nix/install) --daemon + + + The multi-user installation of Nix will create build users between + the user IDs 30001 and 30032, and a group with the group ID 30000. -You can also download a binary tarball that contains Nix and all -its dependencies. (This is what the install script at -https://nixos.org/nix/install does automatically.) You -should unpack it somewhere (e.g. in /tmp), and -then run the script named install inside the binary -tarball: + You should run this under your usual user account, + not as root. The script will invoke + sudo as needed. + + + + If you need Nix to use a different group ID or user ID set, you + will have to download the tarball manually and edit the install + script. + + + + The installer will modify /etc/bashrc, and + /etc/zshrc if they exist. The installer will + first back up these files with a + .backup-before-nix extension. The installer + will also create /etc/profile.d/nix.sh. + + + You can uninstall Nix with the following commands: -alice$ cd /tmp -alice$ tar xfj nix-1.8-x86_64-darwin.tar.bz2 -alice$ cd nix-1.8-x86_64-darwin -alice$ ./install +sudo rm -rf /etc/profile/nix.sh /etc/nix /nix ~root/.nix-profile ~root/.nix-defexpr ~root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels + +# If you are on Linux with systemd, you will need to run: +sudo systemctl stop nix-daemon.socket +sudo systemctl stop nix-daemon.service +sudo systemctl disable nix-daemon.socket +sudo systemctl disable nix-daemon.service +sudo systemctl daemon-reload + +# If you are on macOS, you will need to run: +sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist +sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist - + There may also be references to Nix in + /etc/profile, + /etc/bashrc, and + /etc/zshrc which you may remove. + -Nix can be uninstalled using rpm -e nix or -dpkg -r nix on RPM- and Dpkg-based systems, -respectively. After this you should manually remove the Nix store and -other auxiliary data, if desired: +
- -$ rm -rf /nix +
+ macOS Installation - + + Starting with macOS 10.15 (Catalina), the root filesystem is read-only. + This means /nix can no longer live on your system + volume, and that you'll need a workaround to install Nix. + -You can uninstall Nix simply by running: + + The recommended approach, which creates an unencrypted APFS volume + for your Nix store and a "synthetic" empty directory to mount it + over at /nix, is least likely to impair Nix + or your system. + + + + With all separate-volume approaches, it's possible something on + your system (particularly daemons/services and restored apps) may + need access to your Nix store before the volume is mounted. Adding + additional encryption makes this more likely. + + + + If you're using a recent Mac with a + T2 chip, + your drive will still be encrypted at rest (in which case "unencrypted" + is a bit of a misnomer). To use this approach, just install Nix with: + + + $ sh <(curl https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume + + + If you don't like the sound of this, you'll want to weigh the + other approaches and tradeoffs detailed in this section. + + + + Eventual solutions? + + All of the known workarounds have drawbacks, but we hope + better solutions will be available in the future. Some that + we have our eye on are: + + + + + A true firmlink would enable the Nix store to live on the + primary data volume without the build problems caused by + the symlink approach. End users cannot currently + create true firmlinks. + + + + + If the Nix store volume shared FileVault encryption + with the primary data volume (probably by using the same + volume group and role), FileVault encryption could be + easily supported by the installer without requiring + manual setup by each user. + + + + + +
+ Change the Nix store path prefix + + Changing the default prefix for the Nix store is a simple + approach which enables you to leave it on your root volume, + where it can take full advantage of FileVault encryption if + enabled. Unfortunately, this approach also opts your device out + of some benefits that are enabled by using the same prefix + across systems: + + + + + Your system won't be able to take advantage of the binary + cache (unless someone is able to stand up and support + duplicate caching infrastructure), which means you'll + spend more time waiting for builds. + + + + + It's harder to build and deploy packages to Linux systems. + + + + + + + + It would also possible (and often requested) to just apply this + change ecosystem-wide, but it's an intrusive process that has + side effects we want to avoid for now. + + + + +
+ +
+ Use a separate encrypted volume + + If you like, you can also add encryption to the recommended + approach taken by the installer. You can do this by pre-creating + an encrypted volume before you run the installer--or you can + run the installer and encrypt the volume it creates later. + + + + In either case, adding encryption to a second volume isn't quite + as simple as enabling FileVault for your boot volume. Before you + dive in, there are a few things to weigh: + + + + + The additional volume won't be encrypted with your existing + FileVault key, so you'll need another mechanism to decrypt + the volume. + + + + + You can store the password in Keychain to automatically + decrypt the volume on boot--but it'll have to wait on Keychain + and may not mount before your GUI apps restore. If any of + your launchd agents or apps depend on Nix-installed software + (for example, if you use a Nix-installed login shell), the + restore may fail or break. + + + On a case-by-case basis, you may be able to work around this + problem by using wait4path to block + execution until your executable is available. + + + It's also possible to decrypt and mount the volume earlier + with a login hook--but this mechanism appears to be + deprecated and its future is unclear. + + + + + You can hard-code the password in the clear, so that your + store volume can be decrypted before Keychain is available. + + + + + If you are comfortable navigating these tradeoffs, you can encrypt the volume with + something along the lines of: + + + + alice$ diskutil apfs enableFileVault /nix -user disk + + +
+ +
+ + Symlink the Nix store to a custom location + + Another simple approach is using /etc/synthetic.conf + to symlink the Nix store to the data volume. This option also + enables your store to share any configured FileVault encryption. + Unfortunately, builds that resolve the symlink may leak the + canonical path or even fail. + + + Because of these downsides, we can't recommend this approach. + + +
+ +
+ Notes on the recommended approach + + This section goes into a little more detail on the recommended + approach. You don't need to understand it to run the installer, + but it can serve as a helpful reference if you run into trouble. + + + + + In order to compose user-writable locations into the new + read-only system root, Apple introduced a new concept called + firmlinks, which it describes as a + "bi-directional wormhole" between two filesystems. You can + see the current firmlinks in /usr/share/firmlinks. + Unfortunately, firmlinks aren't (currently?) user-configurable. + + + + For special cases like NFS mount points or package manager roots, + synthetic.conf(5) + supports limited user-controlled file-creation (of symlinks, + and synthetic empty directories) at /. + To create a synthetic empty directory for mounting at /nix, + add the following line to /etc/synthetic.conf + (create it if necessary): + + + nix + + + + + This configuration is applied at boot time, but you can use + apfs.util to trigger creation (not deletion) + of new entries without a reboot: + + + alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B + + + + + Create the new APFS volume with diskutil: + + + alice$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix + + + + + Using vifs, add the new mount to + /etc/fstab. If it doesn't already have + other entries, it should look something like: + -$ rm -rf /nix +# +# Warning - this file should only be modified with vifs(8) +# +# Failure to do so is unsupported and may be destructive. +# +LABEL=Nix\040Store /nix apfs rw,nobrowse - + + The nobrowse setting will keep Spotlight from indexing this + volume, and keep it from showing up on your desktop. + + + +
+ +
+ +
+ Installing a pinned Nix version from a URL + + + NixOS.org hosts version-specific installation URLs for all Nix + versions since 1.11.16, at + https://releases.nixos.org/nix/nix-version/install. + + + + These install scripts can be used the same as the main + NixOS.org installation script: + + + sh <(curl https://nixos.org/nix/install) + + + + + In the same directory of the install script are sha256 sums, and + gpg signature files. + +
+ +
+ Installing from a binary tarball + + + You can also download a binary tarball that contains Nix and all + its dependencies. (This is what the install script at + https://nixos.org/nix/install does automatically.) You + should unpack it somewhere (e.g. in /tmp), + and then run the script named install inside + the binary tarball: + + + +alice$ cd /tmp +alice$ tar xfj nix-1.8-x86_64-darwin.tar.bz2 +alice$ cd nix-1.8-x86_64-darwin +alice$ ./install + + + + If you need to edit the multi-user installation script to use + different group ID or a different user ID range, modify the + variables set in the file named + install-multi-user. + +
diff --git a/doc/manual/installation/multi-user.xml b/doc/manual/installation/multi-user.xml index 49c4f723597..69ae1ef2704 100644 --- a/doc/manual/installation/multi-user.xml +++ b/doc/manual/installation/multi-user.xml @@ -52,34 +52,6 @@ This creates 10 build users. There can never be more concurrent builds than the number of build users, so you may want to increase this if you expect to do many builds at the same time. -On Mac OS X, you can create the required group and users by -running the following script: - - -#! /bin/bash -e - -dseditgroup -o create nixbld -q - -gid=$(dscl . -read /Groups/nixbld | awk '($1 == "PrimaryGroupID:") {print $2 }') - -echo "created nixbld group with gid $gid" - -for i in $(seq 1 10); do - user=/Users/nixbld$i - uid="$((30000 + $i))" - dscl . create $user - dscl . create $user RealName "Nix build user $i" - dscl . create $user PrimaryGroupID "$gid" - dscl . create $user UserShell /usr/bin/false - dscl . create $user NFSHomeDirectory /var/empty - dscl . create $user UniqueID "$uid" - dseditgroup -o edit -a nixbld$i -t user nixbld - echo "created nixbld$i user with uid $uid" -done - - - - diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml index cd6d61356ba..fa6da9b1ed0 100644 --- a/doc/manual/installation/prerequisites-source.xml +++ b/doc/manual/installation/prerequisites-source.xml @@ -8,37 +8,56 @@ - GNU Make. + GNU Autoconf + () + and the autoconf-archive macro collection + (). + These are only needed to run the bootstrap script, and are not necessary + if your source distribution came with a pre-built + ./configure script. - A version of GCC or Clang that supports C++11. + GNU Make. + + Bash Shell. The ./configure script + relies on bashisms, so Bash is required. - Perl 5.8 or higher. + A version of GCC or Clang that supports C++17. pkg-config to locate dependencies. If your distribution does not provide it, you can get it from . - + The OpenSSL library to calculate cryptographic hashes. If your distribution does not provide it, you can get it from . + The libbrotlienc and + libbrotlidec libraries to provide implementation + of the Brotli compression algorithm. They are available for download + from the official repository . + The bzip2 compressor program and the libbz2 library. Thus you must have bzip2 installed, including development headers and libraries. If your distribution does not provide these, you can obtain bzip2 from . + xlink:href="https://web.archive.org/web/20180624184756/http://www.bzip.org/" + />. + liblzma, which is provided by + XZ Utils. If your distribution does not provide this, you can + get it from . + + cURL and its library. If your distribution does not + provide it, you can get it from . + The SQLite embedded database library, version 3.6.19 or higher. If your distribution does not provide it, please install it from . - The Perl DBI and DBD::SQLite libraries, which are - available from CPAN if your - distribution does not provide them. - The Boehm garbage collector to reduce the evaluator’s memory @@ -47,6 +66,14 @@ pass the flag to configure. + The boost library of version + 1.66.0 or higher. It can be obtained from the official web site + . + + The editline library of version + 1.14.0 or higher. It can be obtained from the its repository + . + The xmllint and xsltproc programs to build this manual and the man-pages. These are part of the libxml2 and @@ -72,6 +99,15 @@ modify the parser or when you are building from the Git repository. + The libseccomp is used to provide + syscall filtering on Linux. This is an optional dependency and can + be disabled passing a + option to the configure script (Not recommended + unless your system doesn't support + libseccomp). To get the library, visit . + diff --git a/doc/manual/installation/supported-platforms.xml b/doc/manual/installation/supported-platforms.xml index cbe528690cd..3e74be49d1f 100644 --- a/doc/manual/installation/supported-platforms.xml +++ b/doc/manual/installation/supported-platforms.xml @@ -10,9 +10,9 @@ - Linux (i686, x86_64). + Linux (i686, x86_64, aarch64). - Mac OS X (x86_64). + macOS (x86_64). - + + + + + diff --git a/doc/manual/release-notes/rl-0.8.xml b/doc/manual/release-notes/rl-0.8.xml index 784b26c6b7d..825798fa9b0 100644 --- a/doc/manual/release-notes/rl-0.8.xml +++ b/doc/manual/release-notes/rl-0.8.xml @@ -8,7 +8,7 @@ NOTE: the hashing scheme in Nix 0.8 changed (as detailed below). As a result, nix-pull manifests and channels built -for Nix 0.7 and below will now work anymore. However, the Nix +for Nix 0.7 and below will not work anymore. However, the Nix expression language has not changed, so you can still build from source. Also, existing user environments continue to work. Nix 0.8 will automatically upgrade the database schema of previous diff --git a/doc/manual/release-notes/rl-1.11.10.xml b/doc/manual/release-notes/rl-1.11.10.xml new file mode 100644 index 00000000000..415388b3e2d --- /dev/null +++ b/doc/manual/release-notes/rl-1.11.10.xml @@ -0,0 +1,31 @@ +
+ +Release 1.11.10 (2017-06-12) + +This release fixes a security bug in Nix’s “build user” build +isolation mechanism. Previously, Nix builders had the ability to +create setuid binaries owned by a nixbld +user. Such a binary could then be used by an attacker to assume a +nixbld identity and interfere with subsequent +builds running under the same UID. + +To prevent this issue, Nix now disallows builders to create +setuid and setgid binaries. On Linux, this is done using a seccomp BPF +filter. Note that this imposes a small performance penalty (e.g. 1% +when building GNU Hello). Using seccomp, we now also prevent the +creation of extended attributes and POSIX ACLs since these cannot be +represented in the NAR format and (in the case of POSIX ACLs) allow +bypassing regular Nix store permissions. On macOS, the restriction is +implemented using the existing sandbox mechanism, which now uses a +minimal “allow all except the creation of setuid/setgid binaries” +profile when regular sandboxing is disabled. On other platforms, the +“build user” mechanism is now disabled. + +Thanks go to Linus Heckemann for discovering and reporting this +bug. + +
diff --git a/doc/manual/release-notes/rl-1.11.xml b/doc/manual/release-notes/rl-1.11.xml index efb03d61393..fe422dd1f89 100644 --- a/doc/manual/release-notes/rl-1.11.xml +++ b/doc/manual/release-notes/rl-1.11.xml @@ -121,13 +121,6 @@ $ diffoscope /nix/store/11a27shh6n2i…-zlib-1.2.8 /nix/store/11a27shh6n2i…-zl also improves performance. - - The Nix language now supports floating point numbers. They are - based on regular C++ float and compatible with - existing integers and number-related operations. Export and import to and - from JSON and XML works, too. - - All "chroot"-containing strings got renamed to "sandbox". In particular, some Nix options got renamed, but the old names diff --git a/doc/manual/release-notes/rl-1.12.xml b/doc/manual/release-notes/rl-1.12.xml deleted file mode 100644 index d6864b3f55d..00000000000 --- a/doc/manual/release-notes/rl-1.12.xml +++ /dev/null @@ -1,24 +0,0 @@ -
- -Release 1.12 (TBA) - -This release has the following new features: - - - - - It is no longer necessary to set the - NIX_REMOTE environment variable if you need to use - the Nix daemon. Nix will use the daemon automatically if you don’t - have write access to the Nix database. - - - - -This release has contributions from TBD. - -
diff --git a/doc/manual/release-notes/rl-1.2.xml b/doc/manual/release-notes/rl-1.2.xml index dc272c420dd..748fd9e6702 100644 --- a/doc/manual/release-notes/rl-1.2.xml +++ b/doc/manual/release-notes/rl-1.2.xml @@ -40,7 +40,7 @@ $ nix-env -i thunderbird --option binary-caches http://cache.nixos.org Binary caches are created using nix-push. For details on the operation and format of binary caches, see the nix-push manpage. More details are provided in - this + this nix-dev posting. diff --git a/doc/manual/release-notes/rl-1.8.xml b/doc/manual/release-notes/rl-1.8.xml index 48caac2c6b6..c854c5c5f85 100644 --- a/doc/manual/release-notes/rl-1.8.xml +++ b/doc/manual/release-notes/rl-1.8.xml @@ -83,8 +83,8 @@ $ nix-store -l $(which xterm) caches). The configuration option - now defaults to the number of - available CPU cores. + now defaults to the number of available + CPU cores. Build users are now used by default when Nix is invoked as root. This prevents builds from accidentally running as diff --git a/doc/manual/release-notes/rl-2.0.xml b/doc/manual/release-notes/rl-2.0.xml new file mode 100644 index 00000000000..4c683dd3d72 --- /dev/null +++ b/doc/manual/release-notes/rl-2.0.xml @@ -0,0 +1,1012 @@ +
+ +Release 2.0 (2018-02-22) + +The following incompatible changes have been made: + + + + + The manifest-based substituter mechanism + (download-using-manifests) has been removed. It + has been superseded by the binary cache substituter mechanism + since several years. As a result, the following programs have been + removed: + + + nix-pull + nix-generate-patches + bsdiff + bspatch + + + + + + The “copy from other stores” substituter mechanism + (copy-from-other-stores and the + NIX_OTHER_STORES environment variable) has been + removed. It was primarily used by the NixOS installer to copy + available paths from the installation medium. The replacement is + to use a chroot store as a substituter + (e.g. --substituters /mnt), or to build into a + chroot store (e.g. --store /mnt --substituters /). + + + + The command nix-push has been removed as + part of the effort to eliminate Nix's dependency on Perl. You can + use nix copy instead, e.g. nix copy + --to file:///tmp/my-binary-cache paths… + + + + The “nested” log output feature () has been removed. As a result, + nix-log2xml was also removed. + + + + OpenSSL-based signing has been removed. This + feature was never well-supported. A better alternative is provided + by the and + options. + + + + Failed build caching has been removed. This + feature was introduced to support the Hydra continuous build + system, but Hydra no longer uses it. + + + + nix-mode.el has been removed from + Nix. It is now a separate + repository and can be installed through the MELPA package + repository. + + + + +This release has the following new features: + + + + + It introduces a new command named nix, + which is intended to eventually replace all + nix-* commands with a more consistent and + better designed user interface. It currently provides replacements + for some (but not all) of the functionality provided by + nix-store, nix-build, + nix-shell -p, nix-env -qa, + nix-instantiate --eval, + nix-push and + nix-copy-closure. It has the following major + features: + + + + + Unlike the legacy commands, it has a consistent way to + refer to packages and package-like arguments (like store + paths). For example, the following commands all copy the GNU + Hello package to a remote machine: + + nix copy --to ssh://machine nixpkgs.hello + nix copy --to ssh://machine /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10 + nix copy --to ssh://machine '(with import <nixpkgs> {}; hello)' + + By contrast, nix-copy-closure only accepted + store paths as arguments. + + + + It is self-documenting: shows + all available command-line arguments. If + is given after a subcommand, it shows + examples for that subcommand. nix + --help-config shows all configuration + options. + + + + It is much less verbose. By default, it displays a + single-line progress indicator that shows how many packages + are left to be built or downloaded, and (if there are running + builds) the most recent line of builder output. If a build + fails, it shows the last few lines of builder output. The full + build log can be retrieved using nix + log. + + + + It provides + all nix.conf configuration options as + command line flags. For example, instead of --option + http-connections 100 you can write + --http-connections 100. Boolean options can + be written as + --foo or + --no-foo + (e.g. ). + + + + Many subcommands have a flag to + write results to stdout in JSON format. + + + + + Please note that the nix command + is a work in progress and the interface is subject to + change. + + It provides the following high-level (“porcelain”) + subcommands: + + + + + nix build is a replacement for + nix-build. + + + + nix run executes a command in an + environment in which the specified packages are available. It + is (roughly) a replacement for nix-shell + -p. Unlike that command, it does not execute the + command in a shell, and has a flag (-c) + that specifies the unquoted command line to be + executed. + + It is particularly useful in conjunction with chroot + stores, allowing Linux users who do not have permission to + install Nix in /nix/store to still use + binary substitutes that assume + /nix/store. For example, + + nix run --store ~/my-nix nixpkgs.hello -c hello --greeting 'Hi everybody!' + + downloads (or if not substitutes are available, builds) the + GNU Hello package into + ~/my-nix/nix/store, then runs + hello in a mount namespace where + ~/my-nix/nix/store is mounted onto + /nix/store. + + + + nix search replaces nix-env + -qa. It searches the available packages for + occurrences of a search string in the attribute name, package + name or description. Unlike nix-env -qa, it + has a cache to speed up subsequent searches. + + + + nix copy copies paths between + arbitrary Nix stores, generalising + nix-copy-closure and + nix-push. + + + + nix repl replaces the external + program nix-repl. It provides an + interactive environment for evaluating and building Nix + expressions. Note that it uses linenoise-ng + instead of GNU Readline. + + + + nix upgrade-nix upgrades Nix to the + latest stable version. This requires that Nix is installed in + a profile. (Thus it won’t work on NixOS, or if it’s installed + outside of the Nix store.) + + + + nix verify checks whether store paths + are unmodified and/or “trusted” (see below). It replaces + nix-store --verify and nix-store + --verify-path. + + + + nix log shows the build log of a + package or path. If the build log is not available locally, it + will try to obtain it from the configured substituters (such + as cache.nixos.org, which now provides build + logs). + + + + nix edit opens the source code of a + package in your editor. + + + + nix eval replaces + nix-instantiate --eval. + + + + nix + why-depends shows why one store path has another in + its closure. This is primarily useful to finding the causes of + closure bloat. For example, + + nix why-depends nixpkgs.vlc nixpkgs.libdrm.dev + + shows a chain of files and fragments of file contents that + cause the VLC package to have the “dev” output of + libdrm in its closure — an undesirable + situation. + + + + nix path-info shows information about + store paths, replacing nix-store -q. A + useful feature is the option + (). For example, the following command show + the closure sizes of every path in the current NixOS system + closure, sorted by size: + + nix path-info -rS /run/current-system | sort -nk2 + + + + + + nix optimise-store replaces + nix-store --optimise. The main difference + is that it has a progress indicator. + + + + + A number of low-level (“plumbing”) commands are also + available: + + + + + nix ls-store and nix + ls-nar list the contents of a store path or NAR + file. The former is primarily useful in conjunction with + remote stores, e.g. + + nix ls-store --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10 + + lists the contents of path in a binary cache. + + + + nix cat-store and nix + cat-nar allow extracting a file from a store path or + NAR file. + + + + nix dump-path writes the contents of + a store path to stdout in NAR format. This replaces + nix-store --dump. + + + + nix + show-derivation displays a store derivation in JSON + format. This is an alternative to + pp-aterm. + + + + nix + add-to-store replaces nix-store + --add. + + + + nix sign-paths signs store + paths. + + + + nix copy-sigs copies signatures from + one store to another. + + + + nix show-config shows all + configuration options and their current values. + + + + + + + + The store abstraction that Nix has had for a long time to + support store access via the Nix daemon has been extended + significantly. In particular, substituters (which used to be + external programs such as + download-from-binary-cache) are now subclasses + of the abstract Store class. This allows + many Nix commands to operate on such store types. For example, + nix path-info shows information about paths in + your local Nix store, while nix path-info --store + https://cache.nixos.org/ shows information about paths + in the specified binary cache. Similarly, + nix-copy-closure, nix-push + and substitution are all instances of the general notion of + copying paths between different kinds of Nix stores. + + Stores are specified using an URI-like syntax, + e.g. https://cache.nixos.org/ or + ssh://machine. The following store types are supported: + + + + + + LocalStore (stori URI + local or an absolute path) and the misnamed + RemoteStore (daemon) + provide access to a local Nix store, the latter via the Nix + daemon. You can use auto or the empty + string to auto-select a local or daemon store depending on + whether you have write permission to the Nix store. It is no + longer necessary to set the NIX_REMOTE + environment variable to use the Nix daemon. + + As noted above, LocalStore now + supports chroot builds, allowing the “physical” location of + the Nix store + (e.g. /home/alice/nix/store) to differ + from its “logical” location (typically + /nix/store). This allows non-root users + to use Nix while still getting the benefits from prebuilt + binaries from cache.nixos.org. + + + + + + BinaryCacheStore is the abstract + superclass of all binary cache stores. It supports writing + build logs and NAR content listings in JSON format. + + + + + + HttpBinaryCacheStore + (http://, https://) + supports binary caches via HTTP or HTTPS. If the server + supports PUT requests, it supports + uploading store paths via commands such as nix + copy. + + + + + + LocalBinaryCacheStore + (file://) supports binary caches in the + local filesystem. + + + + + + S3BinaryCacheStore + (s3://) supports binary caches stored in + Amazon S3, if enabled at compile time. + + + + + + LegacySSHStore (ssh://) + is used to implement remote builds and + nix-copy-closure. + + + + + + SSHStore + (ssh-ng://) supports arbitrary Nix + operations on a remote machine via the same protocol used by + nix-daemon. + + + + + + + + + + + + Security has been improved in various ways: + + + + + Nix now stores signatures for local store + paths. When paths are copied between stores (e.g., copied from + a binary cache to a local store), signatures are + propagated. + + Locally-built paths are signed automatically using the + secret keys specified by the + store option. Secret/public key pairs can be generated using + nix-store + --generate-binary-cache-key. + + In addition, locally-built store paths are marked as + “ultimately trusted”, but this bit is not propagated when + paths are copied between stores. + + + + Content-addressable store paths no longer require + signatures — they can be imported into a store by unprivileged + users even if they lack signatures. + + + + The command nix verify checks whether + the specified paths are trusted, i.e., have a certain number + of trusted signatures, are ultimately trusted, or are + content-addressed. + + + + Substitutions from binary caches now + require signatures by default. This was already the case on + NixOS. + + + + In Linux sandbox builds, we now + use /build instead of + /tmp as the temporary build + directory. This fixes potential security problems when a build + accidentally stores its TMPDIR in some + security-sensitive place, such as an RPATH. + + + + + + + + + + Pure evaluation mode. With the + --pure-eval flag, Nix enables a variant of the existing + restricted evaluation mode that forbids access to anything that could cause + different evaluations of the same command line arguments to produce a + different result. This includes builtin functions such as + builtins.getEnv, but more importantly, + all filesystem or network access unless a content hash + or commit hash is specified. For example, calls to + builtins.fetchGit are only allowed if a + rev attribute is specified. + + The goal of this feature is to enable true reproducibility + and traceability of builds (including NixOS system configurations) + at the evaluation level. For example, in the future, + nixos-rebuild might build configurations from a + Nix expression in a Git repository in pure mode. That expression + might fetch other repositories such as Nixpkgs via + builtins.fetchGit. The commit hash of the + top-level repository then uniquely identifies a running system, + and, in conjunction with that repository, allows it to be + reproduced or modified. + + + + + There are several new features to support binary + reproducibility (i.e. to help ensure that multiple builds of the + same derivation produce exactly the same output). When + is set to + false, it’s no + longer a fatal error if build rounds produce different + output. Also, a hook named is provided + to allow you to run tools such as diffoscope + when build rounds produce different output. + + + + Configuring remote builds is a lot easier now. Provided you + are not using the Nix daemon, you can now just specify a remote + build machine on the command line, e.g. --option builders + 'ssh://my-mac x86_64-darwin'. The environment variable + NIX_BUILD_HOOK has been removed and is no longer + needed. The environment variable NIX_REMOTE_SYSTEMS + is still supported for compatibility, but it is also possible to + specify builders in nix.conf by setting the + option builders = + @path. + + + + If a fixed-output derivation produces a result with an + incorrect hash, the output path is moved to the location + corresponding to the actual hash and registered as valid. Thus, a + subsequent build of the fixed-output derivation with the correct + hash is unnecessary. + + + + nix-shell now + sets the IN_NIX_SHELL environment variable + during evaluation and in the shell itself. This can be used to + perform different actions depending on whether you’re in a Nix + shell or in a regular build. Nixpkgs provides + lib.inNixShell to check this variable during + evaluation. + + + + NIX_PATH is now lazy, so URIs in the path are + only downloaded if they are needed for evaluation. + + + + You can now use + channel:channel-name as a + short-hand for + https://nixos.org/channels/channel-name/nixexprs.tar.xz. For + example, nix-build channel:nixos-15.09 -A hello + will build the GNU Hello package from the + nixos-15.09 channel. In the future, this may + use Git to fetch updates more efficiently. + + + + When is given, the last + 10 lines of the build log will be shown if a build + fails. + + + + Networking has been improved: + + + + + HTTP/2 is now supported. This makes binary cache lookups + much + more efficient. + + + + We now retry downloads on many HTTP errors, making + binary caches substituters more resilient to temporary + failures. + + + + HTTP credentials can now be configured via the standard + netrc mechanism. + + + + If S3 support is enabled at compile time, + s3:// URIs are supported + in all places where Nix allows URIs. + + + + Brotli compression is now supported. In particular, + cache.nixos.org build logs are now compressed using + Brotli. + + + + + + + + + + nix-env now + ignores packages with bad derivation names (in particular those + starting with a digit or containing a dot). + + + + Many configuration options have been renamed, either because + they were unnecessarily verbose + (e.g. is now just + ) or to reflect generalised behaviour + (e.g. is now + because it allows arbitrary store + URIs). The old names are still supported for compatibility. + + + + The option can now + be set to auto to use the number of CPUs in the + system. + + + + Hashes can now + be specified in base-64 format, in addition to base-16 and the + non-standard base-32. + + + + nix-shell now uses + bashInteractive from Nixpkgs, rather than the + bash command that happens to be in the caller’s + PATH. This is especially important on macOS where + the bash provided by the system is seriously + outdated and cannot execute stdenv’s setup + script. + + + + Nix can now automatically trigger a garbage collection if + free disk space drops below a certain level during a build. This + is configured using the and + options. + + + + nix-store -q --roots and + nix-store --gc --print-roots now show temporary + and in-memory roots. + + + + + Nix can now be extended with plugins. See the documentation of + the option for more details. + + + + + +The Nix language has the following new features: + + + + + It supports floating point numbers. They are based on the + C++ float type and are supported by the + existing numerical operators. Export and import to and from JSON + and XML works, too. + + + + Derivation attributes can now reference the outputs of the + derivation using the placeholder builtin + function. For example, the attribute + + +configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"}"; + + + will cause the configureFlags environment variable + to contain the actual store paths corresponding to the + out and dev outputs. + + + + + + +The following builtin functions are new or extended: + + + + + builtins.fetchGit + allows Git repositories to be fetched at evaluation time. Thus it + differs from the fetchgit function in + Nixpkgs, which fetches at build time and cannot be used to fetch + Nix expressions during evaluation. A typical use case is to import + external NixOS modules from your configuration, e.g. + + imports = [ (builtins.fetchGit https://github.com/edolstra/dwarffs + "/module.nix") ]; + + + + + + Similarly, builtins.fetchMercurial + allows you to fetch Mercurial repositories. + + + + builtins.path generalises + builtins.filterSource and path literals + (e.g. ./foo). It allows specifying a store path + name that differs from the source path name + (e.g. builtins.path { path = ./foo; name = "bar"; + }) and also supports filtering out unwanted + files. + + + + builtins.fetchurl and + builtins.fetchTarball now support + sha256 and name + attributes. + + + + builtins.split + splits a string using a POSIX extended regular expression as the + separator. + + + + builtins.partition + partitions the elements of a list into two lists, depending on a + Boolean predicate. + + + + <nix/fetchurl.nix> now uses the + content-addressable tarball cache at + http://tarballs.nixos.org/, just like + fetchurl in + Nixpkgs. (f2682e6e18a76ecbfb8a12c17e3a0ca15c084197) + + + + In restricted and pure evaluation mode, builtin functions + that download from the network (such as + fetchGit) are permitted to fetch underneath a + list of URI prefixes specified in the option + . + + + + + + +The Nix build environment has the following changes: + + + + + Values such as Booleans, integers, (nested) lists and + attribute sets can now + be passed to builders in a non-lossy way. If the special attribute + __structuredAttrs is set to + true, the other derivation attributes are + serialised in JSON format and made available to the builder via + the file .attrs.json in the builder’s temporary + directory. This obviates the need for + passAsFile since JSON files have no size + restrictions, unlike process environments. + + As + a convenience to Bash builders, Nix writes a script named + .attrs.sh to the builder’s directory that + initialises shell variables corresponding to all attributes that + are representable in Bash. This includes non-nested (associative) + arrays. For example, the attribute hardening.format = + true ends up as the Bash associative array element + ${hardening[format]}. + + + + Builders can now + communicate what build phase they are in by writing messages to + the file descriptor specified in NIX_LOG_FD. The + current phase is shown by the nix progress + indicator. + + + + + In Linux sandbox builds, we now + provide a default /bin/sh (namely + ash from BusyBox). + + + + In structured attribute mode, + exportReferencesGraph exports + extended information about closures in JSON format. In particular, + it includes the sizes and hashes of paths. This is primarily + useful for NixOS image builders. + + + + Builds are now + killed as soon as Nix receives EOF on the builder’s stdout or + stderr. This fixes a bug that allowed builds to hang Nix + indefinitely, regardless of + timeouts. + + + + The configuration + option can now specify optional paths by appending a + ?, e.g. /dev/nvidiactl? will + bind-mount /dev/nvidiactl only if it + exists. + + + + On Linux, builds are now executed in a user + namespace with UID 1000 and GID 100. + + + + + + +A number of significant internal changes were made: + + + + + Nix no longer depends on Perl and all Perl components have + been rewritten in C++ or removed. The Perl bindings that used to + be part of Nix have been moved to a separate package, + nix-perl. + + + + All Store classes are now + thread-safe. RemoteStore supports multiple + concurrent connections to the daemon. This is primarily useful in + multi-threaded programs such as + hydra-queue-runner. + + + + + + +This release has contributions from + +Adrien Devresse, +Alexander Ried, +Alex Cruice, +Alexey Shmalko, +AmineChikhaoui, +Andy Wingo, +Aneesh Agrawal, +Anthony Cowley, +Armijn Hemel, +aszlig, +Ben Gamari, +Benjamin Hipple, +Benjamin Staffin, +Benno Fünfstück, +Bjørn Forsman, +Brian McKenna, +Charles Strahan, +Chase Adams, +Chris Martin, +Christian Theune, +Chris Warburton, +Daiderd Jordan, +Dan Connolly, +Daniel Peebles, +Dan Peebles, +davidak, +David McFarland, +Dmitry Kalinkin, +Domen Kožar, +Eelco Dolstra, +Emery Hemingway, +Eric Litak, +Eric Wolf, +Fabian Schmitthenner, +Frederik Rietdijk, +Gabriel Gonzalez, +Giorgio Gallo, +Graham Christensen, +Guillaume Maudoux, +Harmen, +Iavael, +James Broadhead, +James Earl Douglas, +Janus Troelsen, +Jeremy Shaw, +Joachim Schiele, +Joe Hermaszewski, +Joel Moberg, +Johannes 'fish' Ziemke, +Jörg Thalheim, +Jude Taylor, +kballou, +Keshav Kini, +Kjetil Orbekk, +Langston Barrett, +Linus Heckemann, +Ludovic Courtès, +Manav Rathi, +Marc Scholten, +Markus Hauck, +Matt Audesse, +Matthew Bauer, +Matthias Beyer, +Matthieu Coudron, +N1X, +Nathan Zadoks, +Neil Mayhew, +Nicolas B. Pierron, +Niklas Hambüchen, +Nikolay Amiantov, +Ole Jørgen Brønner, +Orivej Desh, +Peter Simons, +Peter Stuart, +Pyry Jahkola, +regnat, +Renzo Carbonara, +Rhys, +Robert Vollmert, +Scott Olson, +Scott R. Parish, +Sergei Trofimovich, +Shea Levy, +Sheena Artrip, +Spencer Baugh, +Stefan Junker, +Susan Potter, +Thomas Tuegel, +Timothy Allen, +Tristan Hume, +Tuomas Tynkkynen, +tv, +Tyson Whitehead, +Vladimír Čunát, +Will Dietz, +wmertens, +Wout Mertens, +zimbatm and +Zoran Plesivčak. + + +
diff --git a/doc/manual/release-notes/rl-2.1.xml b/doc/manual/release-notes/rl-2.1.xml new file mode 100644 index 00000000000..16c243fc191 --- /dev/null +++ b/doc/manual/release-notes/rl-2.1.xml @@ -0,0 +1,133 @@ +
+ +Release 2.1 (2018-09-02) + +This is primarily a bug fix release. It also reduces memory +consumption in certain situations. In addition, it has the following +new features: + + + + + The Nix installer will no longer default to the Multi-User + installation for macOS. You can still instruct the installer to + run in multi-user mode. + + + + + The Nix installer now supports performing a Multi-User + installation for Linux computers which are running systemd. You + can select a Multi-User installation by passing the + flag to the installer: sh <(curl + https://nixos.org/nix/install) --daemon. + + + The multi-user installer cannot handle systems with SELinux. + If your system has SELinux enabled, you can force the installer to run + in single-user mode. + + + + New builtin functions: + builtins.bitAnd, + builtins.bitOr, + builtins.bitXor, + builtins.fromTOML, + builtins.concatMap, + builtins.mapAttrs. + + + + + The S3 binary cache store now supports uploading NARs larger + than 5 GiB. + + + + The S3 binary cache store now supports uploading to + S3-compatible services with the endpoint + option. + + + + The flag is no longer required + to recover from disappeared NARs in binary caches. + + + + nix-daemon now respects + . + + + + nix run now respects + nix-support/propagated-user-env-packages. + + + + +This release has contributions from + +Adrien Devresse, +Aleksandr Pashkov, +Alexandre Esteves, +Amine Chikhaoui, +Andrew Dunham, +Asad Saeeduddin, +aszlig, +Ben Challenor, +Ben Gamari, +Benjamin Hipple, +Bogdan Seniuc, +Corey O'Connor, +Daiderd Jordan, +Daniel Peebles, +Daniel Poelzleithner, +Danylo Hlynskyi, +Dmitry Kalinkin, +Domen Kožar, +Doug Beardsley, +Eelco Dolstra, +Erik Arvstedt, +Félix Baylac-Jacqué, +Gleb Peregud, +Graham Christensen, +Guillaume Maudoux, +Ivan Kozik, +John Arnold, +Justin Humm, +Linus Heckemann, +Lorenzo Manacorda, +Matthew Justin Bauer, +Matthew O'Gorman, +Maximilian Bosch, +Michael Bishop, +Michael Fiano, +Michael Mercier, +Michael Raskin, +Michael Weiss, +Nicolas Dudebout, +Peter Simons, +Ryan Trinkle, +Samuel Dionne-Riel, +Sean Seefried, +Shea Levy, +Symphorien Gibol, +Tim Engler, +Tim Sears, +Tuomas Tynkkynen, +volth, +Will Dietz, +Yorick van Pelt and +zimbatm. + + +
diff --git a/doc/manual/release-notes/rl-2.2.xml b/doc/manual/release-notes/rl-2.2.xml new file mode 100644 index 00000000000..d29eb87e82c --- /dev/null +++ b/doc/manual/release-notes/rl-2.2.xml @@ -0,0 +1,143 @@ +
+ +Release 2.2 (2019-01-11) + +This is primarily a bug fix release. It also has the following +changes: + + + + + In derivations that use structured attributes (i.e. that + specify set the __structuredAttrs attribute to + true to cause all attributes to be passed to + the builder in JSON format), you can now specify closure checks + per output, e.g.: + + +outputChecks."out" = { + # The closure of 'out' must not be larger than 256 MiB. + maxClosureSize = 256 * 1024 * 1024; + + # It must not refer to C compiler or to the 'dev' output. + disallowedRequisites = [ stdenv.cc "dev" ]; +}; + +outputChecks."dev" = { + # The 'dev' output must not be larger than 128 KiB. + maxSize = 128 * 1024; +}; + + + + + + + + The derivation attribute + requiredSystemFeatures is now enforced for + local builds, and not just to route builds to remote builders. + The supported features of a machine can be specified through the + configuration setting system-features. + + By default, system-features includes + kvm if /dev/kvm + exists. For compatibility, it also includes the pseudo-features + nixos-test, benchmark and + big-parallel which are used by Nixpkgs to route + builds to particular Hydra build machines. + + + + + Sandbox builds are now enabled by default on Linux. + + + + The new command nix doctor shows + potential issues with your Nix installation. + + + + The fetchGit builtin function now uses a + caching scheme that puts different remote repositories in distinct + local repositories, rather than a single shared repository. This + may require more disk space but is faster. + + + + The dirOf builtin function now works on + relative paths. + + + + Nix now supports SRI hashes, + allowing the hash algorithm and hash to be specified in a single + string. For example, you can write: + + +import <nix/fetchurl.nix> { + url = https://nixos.org/releases/nix/nix-2.1.3/nix-2.1.3.tar.xz; + hash = "sha256-XSLa0FjVyADWWhFfkZ2iKTjFDda6mMXjoYMXLRSYQKQ="; +}; + + + instead of + + +import <nix/fetchurl.nix> { + url = https://nixos.org/releases/nix/nix-2.1.3/nix-2.1.3.tar.xz; + sha256 = "5d22dad058d5c800d65a115f919da22938c50dd6ba98c5e3a183172d149840a4"; +}; + + + + + In fixed-output derivations, the + outputHashAlgo attribute is no longer mandatory + if outputHash specifies the hash. + + nix hash-file and nix + hash-path now print hashes in SRI format by + default. They also use SHA-256 by default instead of SHA-512 + because that's what we use most of the time in Nixpkgs. + + + + Integers are now 64 bits on all platforms. + + + + The evaluator now prints profiling statistics (enabled via + the NIX_SHOW_STATS and + NIX_COUNT_CALLS environment variables) in JSON + format. + + + + The option in nix-store + --query has been removed. Instead, there now is an + option to output the dependency graph + in GraphML format. + + + + All nix-* commands are now symlinks to + nix. This saves a bit of disk space. + + + + nix repl now uses + libeditline or + libreadline. + + + + +
+ diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml new file mode 100644 index 00000000000..0ad7d641f87 --- /dev/null +++ b/doc/manual/release-notes/rl-2.3.xml @@ -0,0 +1,91 @@ +
+ +Release 2.3 (2019-09-04) + +This is primarily a bug fix release. However, it makes some +incompatible changes: + + + + + Nix now uses BSD file locks instead of POSIX file + locks. Because of this, you should not use Nix 2.3 and previous + releases at the same time on a Nix store. + + + + +It also has the following changes: + + + + + builtins.fetchGit's ref + argument now allows specifying an absolute remote ref. + Nix will automatically prefix ref with + refs/heads only if ref doesn't + already begin with refs/. + + + + + The installer now enables sandboxing by default on Linux when the + system has the necessary kernel support. + + + + + The max-jobs setting now defaults to 1. + + + + New builtin functions: + builtins.isPath, + builtins.hashFile. + + + + + The nix command has a new + () flag to + print build log output to stderr, rather than showing the last log + line in the progress bar. To distinguish between concurrent + builds, log lines are prefixed by the name of the package. + + + + + Builds are now executed in a pseudo-terminal, and the + TERM environment variable is set to + xterm-256color. This allows many programs + (e.g. gcc, clang, + cmake) to print colorized log output. + + + + Add convenience flag. This flag + disables substituters; sets the tarball-ttl + setting to infinity (ensuring that any previously downloaded files + are considered current); and disables retrying downloads and sets + the connection timeout to the minimum. This flag is enabled + automatically if there are no configured non-loopback network + interfaces. + + + + Add a post-build-hook setting to run a + program after a build has succeeded. + + + + Add a trace-function-calls setting to log + the duration of Nix function calls to stderr. + + + + +
diff --git a/doc/manual/style.css b/doc/manual/style.css deleted file mode 100644 index 53fd9d5709c..00000000000 --- a/doc/manual/style.css +++ /dev/null @@ -1,271 +0,0 @@ -/* Copied from http://bakefile.sourceforge.net/, which appears - licensed under the GNU GPL. */ - - -/*************************************************************************** - Basic headers and text: - ***************************************************************************/ - -body -{ - font-family: "Nimbus Sans L", sans-serif; - background: white; - margin: 2em 1em 2em 1em; -} - -h1, h2, h3, h4 -{ - color: #005aa0; -} - -h1 /* title */ -{ - font-size: 200%; -} - -div.part h1 -{ - font-size: 240%; -} - -h2 /* chapters, appendices, subtitle */ -{ - font-size: 180%; -} - -div.part -{ - margin-top: 4em; -} - -/* Extra space between chapters, appendices. */ -div.chapter > div.titlepage h2, div.appendix > div.titlepage h2 -{ - margin-top: 1.5em; -} - -div.section > div.titlepage h2 /* sections */ -{ - font-size: 150%; - margin-top: 1.5em; -} - -h3 /* subsections */ -{ - font-size: 125%; -} - -div.simplesect h2 -{ - font-size: 110%; -} - -div.appendix h3 -{ - font-size: 150%; - margin-top: 1.5em; -} - -div.refentry\.separator -{ - margin-top: 2.5em; - margin-bottom: 2em; -} - -div.refnamediv h2, div.refsynopsisdiv h2, div.refsection h2 /* refentry parts */ -{ - margin-top: 1.4em; - font-size: 125%; -} - -div.refsection h3 -{ - font-size: 110%; -} - - -/*************************************************************************** - Examples: - ***************************************************************************/ - -div.example -{ - border: 1px solid #b0b0b0; - padding: 6px 6px; - margin-left: 1.5em; - margin-right: 1.5em; - background: #f4f4f8; - border-radius: 0.4em; - box-shadow: 0.4em 0.4em 0.5em #e0e0e0; -} - -div.example p.title -{ - margin-top: 0em; -} - -div.example pre -{ - box-shadow: none; -} - - -/*************************************************************************** - Screen dumps: - ***************************************************************************/ - -pre.screen, pre.programlisting -{ - border: 1px solid #b0b0b0; - padding: 3px 3px; - margin-left: 1.5em; - margin-right: 1.5em; - color: #600000; - background: #f4f4f8; - font-family: monospace; - border-radius: 0.4em; - box-shadow: 0.4em 0.4em 0.5em #e0e0e0; -} - -div.example pre.programlisting -{ - border: 0px; - padding: 0 0; - margin: 0 0 0 0; -} - - -/*************************************************************************** - Notes, warnings etc: - ***************************************************************************/ - -.note, .warning -{ - border: 1px solid #b0b0b0; - padding: 3px 3px; - margin-left: 1.5em; - margin-right: 1.5em; - margin-bottom: 1em; - padding: 0.3em 0.3em 0.3em 0.3em; - background: #fffff5; - border-radius: 0.4em; - box-shadow: 0.4em 0.4em 0.5em #e0e0e0; -} - -div.note, div.warning -{ - font-style: italic; -} - -div.note h3, div.warning h3 -{ - color: red; - font-size: 100%; - padding-right: 0.5em; - display: inline; -} - -div.note p, div.warning p -{ - margin-bottom: 0em; -} - -div.note h3 + p, div.warning h3 + p -{ - display: inline; -} - -div.note h3 -{ - color: blue; - font-size: 100%; -} - -div.navfooter * -{ - font-size: 90%; -} - - -/*************************************************************************** - Links colors and highlighting: - ***************************************************************************/ - -a { text-decoration: none; } -a:hover { text-decoration: underline; } -a:link { color: #0048b3; } -a:visited { color: #002a6a; } - - -/*************************************************************************** - Table of contents: - ***************************************************************************/ - -div.toc -{ - font-size: 90%; -} - -div.toc dl -{ - margin-top: 0em; - margin-bottom: 0em; -} - - -/*************************************************************************** - Special elements: - ***************************************************************************/ - -tt, code -{ - color: #400000; -} - -.term -{ - font-weight: bold; - -} - -div.variablelist dd p, div.glosslist dd p -{ - margin-top: 0em; -} - -div.variablelist dd, div.glosslist dd -{ - margin-left: 1.5em; -} - -div.glosslist dt -{ - font-style: italic; -} - -.varname -{ - color: #400000; -} - -span.command strong -{ - font-weight: normal; - color: #400000; -} - -div.calloutlist table -{ - box-shadow: none; -} - -table -{ - border-collapse: collapse; - box-shadow: 0.4em 0.4em 0.5em #e0e0e0; -} - -div.affiliation -{ - font-style: italic; -} \ No newline at end of file diff --git a/doc/manual/troubleshooting/collisions-nixenv.xml b/doc/manual/troubleshooting/collisions-nixenv.xml deleted file mode 100644 index 23cc43faf08..00000000000 --- a/doc/manual/troubleshooting/collisions-nixenv.xml +++ /dev/null @@ -1,38 +0,0 @@ -
- -Collisions in <command>nix-env</command> - -Symptom: when installing or upgrading, you get an error message such as - - -$ nix-env -i docbook-xml -... -adding /nix/store/s5hyxgm62gk2...-docbook-xml-4.2 -collision between `/nix/store/s5hyxgm62gk2...-docbook-xml-4.2/xml/dtd/docbook/calstblx.dtd' - and `/nix/store/06h377hr4b33...-docbook-xml-4.3/xml/dtd/docbook/calstblx.dtd' - at /nix/store/...-builder.pl line 62. - - - -The cause is that two installed packages in the user environment -have overlapping filenames (e.g., -xml/dtd/docbook/calstblx.dtd. This usually -happens when you accidentally try to install two versions of the same -package. For instance, in the example above, the Nix Packages -collection contains two versions of docbook-xml, so -nix-env -i will try to install both. The default -user environment builder has no way to way to resolve such conflicts, -so it just gives up. - -Solution: remove one of the offending packages from the user -environment (if already installed) using nix-env --e, or specify exactly which version should be installed -(e.g., nix-env -i docbook-xml-4.2). - - - -
diff --git a/doc/manual/troubleshooting/links-nix-store.xml b/doc/manual/troubleshooting/links-nix-store.xml deleted file mode 100644 index c768889567d..00000000000 --- a/doc/manual/troubleshooting/links-nix-store.xml +++ /dev/null @@ -1,43 +0,0 @@ -
- -<quote>Too many links</quote> Error in the Nix store - - -Symptom: when building something, you get an error message such as - - -... -mkdir: cannot create directory `/nix/store/name': Too many links - - - -This is usually because you have more than 32,000 subdirectories -in /nix/store, as can be seen using ls --l: - - -$ ls -ld /nix/store -drwxrwxrwt 32000 nix nix 4620288 Sep 8 15:08 store - -The ext2 file system is limited to an inode link -count of 32,000 (each subdirectory increasing the count by one). -Furthermore, the st_nlink field of the -stat system call is a 16-bit value. - -This only happens on very large Nix installations (such as build -machines). - -Quick solution: run the garbage collector. You may want to use -the option. - -Real solution: put the Nix store on a file system that supports -more than 32,000 subdirectories per directory, such as ext4. (This -doesn’t solve the st_nlink limit, but ext4 lies to -the kernel by reporting a link count of 1 if it exceeds the -limit.) - -
diff --git a/doc/manual/troubleshooting/troubleshooting.xml b/doc/manual/troubleshooting/troubleshooting.xml deleted file mode 100644 index 1e973a192b1..00000000000 --- a/doc/manual/troubleshooting/troubleshooting.xml +++ /dev/null @@ -1,16 +0,0 @@ - - -Troubleshooting - -This section provides solutions for some common problems. See -the Nix bug -tracker for a list of currently known issues. - - - - - diff --git a/local.mk b/local.mk index 2541f3f3229..d254c10fe9b 100644 --- a/local.mk +++ b/local.mk @@ -1,16 +1,16 @@ ifeq ($(MAKECMDGOALS), dist) - # Make sure we are in repo root with `--git-dir` - dist-files += $(shell git --git-dir=.git ls-files || find * -type f) + dist-files += $(shell cat .dist-files) endif -dist-files += configure config.h.in nix.spec +dist-files += configure config.h.in perl/configure clean-files += Makefile.config -GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr \ - -Wno-unneeded-internal-declaration +GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -$(foreach i, config.h $(call rwildcard, src/lib*, *.hh) src/nix-store/serve-protocol.hh, \ +$(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) -$(foreach i, $(call rwildcard, src/boost, *.hpp), $(eval $(call install-file-in, $(i), $(includedir)/nix/$(patsubst src/%/,%,$(dir $(i))), 0644))) +$(GCH) $(PCH): src/libutil/util.hh config.h + +GCH_CXXFLAGS = -I src/libutil diff --git a/m4/ax_cxx_compile_stdcxx.m4 b/m4/ax_cxx_compile_stdcxx.m4 new file mode 100644 index 00000000000..43087b2e688 --- /dev/null +++ b/m4/ax_cxx_compile_stdcxx.m4 @@ -0,0 +1,951 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CXX_COMPILE_STDCXX(VERSION, [ext|noext], [mandatory|optional]) +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the specified +# version of the C++ standard. If necessary, add switches to CXX and +# CXXCPP to enable support. VERSION may be '11' (for the C++11 standard) +# or '14' (for the C++14 standard). +# +# The second argument, if specified, indicates whether you insist on an +# extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g. +# -std=c++11). If neither is specified, you get whatever works, with +# preference for an extended mode. +# +# The third argument, if specified 'mandatory' or if left unspecified, +# indicates that baseline support for the specified C++ standard is +# required and that the macro should error out if no mode with that +# support is found. If specified 'optional', then configuration proceeds +# regardless, after defining HAVE_CXX${VERSION} if and only if a +# supporting mode is found. +# +# LICENSE +# +# Copyright (c) 2008 Benjamin Kosnik +# Copyright (c) 2012 Zack Weinberg +# Copyright (c) 2013 Roy Stogner +# Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov +# Copyright (c) 2015 Paul Norman +# Copyright (c) 2015 Moritz Klammler +# Copyright (c) 2016, 2018 Krzesimir Nowak +# Copyright (c) 2019 Enji Cooper +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 11 + +dnl This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro +dnl (serial version number 13). + +AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl + m4_if([$1], [11], [ax_cxx_compile_alternatives="11 0x"], + [$1], [14], [ax_cxx_compile_alternatives="14 1y"], + [$1], [17], [ax_cxx_compile_alternatives="17 1z"], + [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$2], [], [], + [$2], [ext], [], + [$2], [noext], [], + [m4_fatal([invalid second argument `$2' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$3], [], [ax_cxx_compile_cxx$1_required=true], + [$3], [mandatory], [ax_cxx_compile_cxx$1_required=true], + [$3], [optional], [ax_cxx_compile_cxx$1_required=false], + [m4_fatal([invalid third argument `$3' to AX_CXX_COMPILE_STDCXX])]) + AC_LANG_PUSH([C++])dnl + ac_success=no + + m4_if([$2], [noext], [], [dnl + if test x$ac_success = xno; then + for alternative in ${ax_cxx_compile_alternatives}; do + switch="-std=gnu++${alternative}" + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + fi]) + + m4_if([$2], [ext], [], [dnl + if test x$ac_success = xno; then + dnl HP's aCC needs +std=c++11 according to: + dnl http://h21007.www2.hp.com/portal/download/files/unprot/aCxx/PDF_Release_Notes/769149-001.pdf + dnl Cray's crayCC needs "-h std=c++11" + for alternative in ${ax_cxx_compile_alternatives}; do + for switch in -std=c++${alternative} +std=c++${alternative} "-h std=c++${alternative}"; do + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + if test x$ac_success = xyes; then + break + fi + done + fi]) + AC_LANG_POP([C++]) + if test x$ax_cxx_compile_cxx$1_required = xtrue; then + if test x$ac_success = xno; then + AC_MSG_ERROR([*** A compiler with support for C++$1 language features is required.]) + fi + fi + if test x$ac_success = xno; then + HAVE_CXX$1=0 + AC_MSG_NOTICE([No compiler with C++$1 support was found]) + else + HAVE_CXX$1=1 + AC_DEFINE(HAVE_CXX$1,1, + [define if the compiler supports basic C++$1 syntax]) + fi + AC_SUBST(HAVE_CXX$1) +]) + + +dnl Test body for checking C++11 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_11], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 +) + + +dnl Test body for checking C++14 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 +) + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_17], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_17 +) + +dnl Tests for new features in C++11 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_11], [[ + +// If the compiler admits that it is not ready for C++11, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201103L + +#error "This is not a C++11 compiler" + +#else + +namespace cxx11 +{ + + namespace test_static_assert + { + + template + struct check + { + static_assert(sizeof(int) <= sizeof(T), "not big enough"); + }; + + } + + namespace test_final_override + { + + struct Base + { + virtual ~Base() {} + virtual void f() {} + }; + + struct Derived : public Base + { + virtual ~Derived() override {} + virtual void f() override {} + }; + + } + + namespace test_double_right_angle_brackets + { + + template < typename T > + struct check {}; + + typedef check single_type; + typedef check> double_type; + typedef check>> triple_type; + typedef check>>> quadruple_type; + + } + + namespace test_decltype + { + + int + f() + { + int a = 1; + decltype(a) b = 2; + return a + b; + } + + } + + namespace test_type_deduction + { + + template < typename T1, typename T2 > + struct is_same + { + static const bool value = false; + }; + + template < typename T > + struct is_same + { + static const bool value = true; + }; + + template < typename T1, typename T2 > + auto + add(T1 a1, T2 a2) -> decltype(a1 + a2) + { + return a1 + a2; + } + + int + test(const int c, volatile int v) + { + static_assert(is_same::value == true, ""); + static_assert(is_same::value == false, ""); + static_assert(is_same::value == false, ""); + auto ac = c; + auto av = v; + auto sumi = ac + av + 'x'; + auto sumf = ac + av + 1.0; + static_assert(is_same::value == true, ""); + static_assert(is_same::value == true, ""); + static_assert(is_same::value == true, ""); + static_assert(is_same::value == false, ""); + static_assert(is_same::value == true, ""); + return (sumf > 0.0) ? sumi : add(c, v); + } + + } + + namespace test_noexcept + { + + int f() { return 0; } + int g() noexcept { return 0; } + + static_assert(noexcept(f()) == false, ""); + static_assert(noexcept(g()) == true, ""); + + } + + namespace test_constexpr + { + + template < typename CharT > + unsigned long constexpr + strlen_c_r(const CharT *const s, const unsigned long acc) noexcept + { + return *s ? strlen_c_r(s + 1, acc + 1) : acc; + } + + template < typename CharT > + unsigned long constexpr + strlen_c(const CharT *const s) noexcept + { + return strlen_c_r(s, 0UL); + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("1") == 1UL, ""); + static_assert(strlen_c("example") == 7UL, ""); + static_assert(strlen_c("another\0example") == 7UL, ""); + + } + + namespace test_rvalue_references + { + + template < int N > + struct answer + { + static constexpr int value = N; + }; + + answer<1> f(int&) { return answer<1>(); } + answer<2> f(const int&) { return answer<2>(); } + answer<3> f(int&&) { return answer<3>(); } + + void + test() + { + int i = 0; + const int c = 0; + static_assert(decltype(f(i))::value == 1, ""); + static_assert(decltype(f(c))::value == 2, ""); + static_assert(decltype(f(0))::value == 3, ""); + } + + } + + namespace test_uniform_initialization + { + + struct test + { + static const int zero {}; + static const int one {1}; + }; + + static_assert(test::zero == 0, ""); + static_assert(test::one == 1, ""); + + } + + namespace test_lambdas + { + + void + test1() + { + auto lambda1 = [](){}; + auto lambda2 = lambda1; + lambda1(); + lambda2(); + } + + int + test2() + { + auto a = [](int i, int j){ return i + j; }(1, 2); + auto b = []() -> int { return '0'; }(); + auto c = [=](){ return a + b; }(); + auto d = [&](){ return c; }(); + auto e = [a, &b](int x) mutable { + const auto identity = [](int y){ return y; }; + for (auto i = 0; i < a; ++i) + a += b--; + return x + identity(a + b); + }(0); + return a + b + c + d + e; + } + + int + test3() + { + const auto nullary = [](){ return 0; }; + const auto unary = [](int x){ return x; }; + using nullary_t = decltype(nullary); + using unary_t = decltype(unary); + const auto higher1st = [](nullary_t f){ return f(); }; + const auto higher2nd = [unary](nullary_t f1){ + return [unary, f1](unary_t f2){ return f2(unary(f1())); }; + }; + return higher1st(nullary) + higher2nd(nullary)(unary); + } + + } + + namespace test_variadic_templates + { + + template + struct sum; + + template + struct sum + { + static constexpr auto value = N0 + sum::value; + }; + + template <> + struct sum<> + { + static constexpr auto value = 0; + }; + + static_assert(sum<>::value == 0, ""); + static_assert(sum<1>::value == 1, ""); + static_assert(sum<23>::value == 23, ""); + static_assert(sum<1, 2>::value == 3, ""); + static_assert(sum<5, 5, 11>::value == 21, ""); + static_assert(sum<2, 3, 5, 7, 11, 13>::value == 41, ""); + + } + + // http://stackoverflow.com/questions/13728184/template-aliases-and-sfinae + // Clang 3.1 fails with headers of libstd++ 4.8.3 when using std::function + // because of this. + namespace test_template_alias_sfinae + { + + struct foo {}; + + template + using member = typename T::member_type; + + template + void func(...) {} + + template + void func(member*) {} + + void test(); + + void test() { func(0); } + + } + +} // namespace cxx11 + +#endif // __cplusplus >= 201103L + +]]) + + +dnl Tests for new features in C++14 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_14], [[ + +// If the compiler admits that it is not ready for C++14, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201402L + +#error "This is not a C++14 compiler" + +#else + +namespace cxx14 +{ + + namespace test_polymorphic_lambdas + { + + int + test() + { + const auto lambda = [](auto&&... args){ + const auto istiny = [](auto x){ + return (sizeof(x) == 1UL) ? 1 : 0; + }; + const int aretiny[] = { istiny(args)... }; + return aretiny[0]; + }; + return lambda(1, 1L, 1.0f, '1'); + } + + } + + namespace test_binary_literals + { + + constexpr auto ivii = 0b0000000000101010; + static_assert(ivii == 42, "wrong value"); + + } + + namespace test_generalized_constexpr + { + + template < typename CharT > + constexpr unsigned long + strlen_c(const CharT *const s) noexcept + { + auto length = 0UL; + for (auto p = s; *p; ++p) + ++length; + return length; + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("x") == 1UL, ""); + static_assert(strlen_c("test") == 4UL, ""); + static_assert(strlen_c("another\0test") == 7UL, ""); + + } + + namespace test_lambda_init_capture + { + + int + test() + { + auto x = 0; + const auto lambda1 = [a = x](int b){ return a + b; }; + const auto lambda2 = [a = lambda1(x)](){ return a; }; + return lambda2(); + } + + } + + namespace test_digit_separators + { + + constexpr auto ten_million = 100'000'000; + static_assert(ten_million == 100000000, ""); + + } + + namespace test_return_type_deduction + { + + auto f(int& x) { return x; } + decltype(auto) g(int& x) { return x; } + + template < typename T1, typename T2 > + struct is_same + { + static constexpr auto value = false; + }; + + template < typename T > + struct is_same + { + static constexpr auto value = true; + }; + + int + test() + { + auto x = 0; + static_assert(is_same::value, ""); + static_assert(is_same::value, ""); + return x; + } + + } + +} // namespace cxx14 + +#endif // __cplusplus >= 201402L + +]]) + + +dnl Tests for new features in C++17 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_17], [[ + +// If the compiler admits that it is not ready for C++17, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201703L + +#error "This is not a C++17 compiler" + +#else + +#include +#include +#include + +namespace cxx17 +{ + + namespace test_constexpr_lambdas + { + + constexpr int foo = [](){return 42;}(); + + } + + namespace test::nested_namespace::definitions + { + + } + + namespace test_fold_expression + { + + template + int multiply(Args... args) + { + return (args * ... * 1); + } + + template + bool all(Args... args) + { + return (args && ...); + } + + } + + namespace test_extended_static_assert + { + + static_assert (true); + + } + + namespace test_auto_brace_init_list + { + + auto foo = {5}; + auto bar {5}; + + static_assert(std::is_same, decltype(foo)>::value); + static_assert(std::is_same::value); + } + + namespace test_typename_in_template_template_parameter + { + + template typename X> struct D; + + } + + namespace test_fallthrough_nodiscard_maybe_unused_attributes + { + + int f1() + { + return 42; + } + + [[nodiscard]] int f2() + { + [[maybe_unused]] auto unused = f1(); + + switch (f1()) + { + case 17: + f1(); + [[fallthrough]]; + case 42: + f1(); + } + return f1(); + } + + } + + namespace test_extended_aggregate_initialization + { + + struct base1 + { + int b1, b2 = 42; + }; + + struct base2 + { + base2() { + b3 = 42; + } + int b3; + }; + + struct derived : base1, base2 + { + int d; + }; + + derived d1 {{1, 2}, {}, 4}; // full initialization + derived d2 {{}, {}, 4}; // value-initialized bases + + } + + namespace test_general_range_based_for_loop + { + + struct iter + { + int i; + + int& operator* () + { + return i; + } + + const int& operator* () const + { + return i; + } + + iter& operator++() + { + ++i; + return *this; + } + }; + + struct sentinel + { + int i; + }; + + bool operator== (const iter& i, const sentinel& s) + { + return i.i == s.i; + } + + bool operator!= (const iter& i, const sentinel& s) + { + return !(i == s); + } + + struct range + { + iter begin() const + { + return {0}; + } + + sentinel end() const + { + return {5}; + } + }; + + void f() + { + range r {}; + + for (auto i : r) + { + [[maybe_unused]] auto v = i; + } + } + + } + + namespace test_lambda_capture_asterisk_this_by_value + { + + struct t + { + int i; + int foo() + { + return [*this]() + { + return i; + }(); + } + }; + + } + + namespace test_enum_class_construction + { + + enum class byte : unsigned char + {}; + + byte foo {42}; + + } + + namespace test_constexpr_if + { + + template + int f () + { + if constexpr(cond) + { + return 13; + } + else + { + return 42; + } + } + + } + + namespace test_selection_statement_with_initializer + { + + int f() + { + return 13; + } + + int f2() + { + if (auto i = f(); i > 0) + { + return 3; + } + + switch (auto i = f(); i + 4) + { + case 17: + return 2; + + default: + return 1; + } + } + + } + + namespace test_template_argument_deduction_for_class_templates + { + + template + struct pair + { + pair (T1 p1, T2 p2) + : m1 {p1}, + m2 {p2} + {} + + T1 m1; + T2 m2; + }; + + void f() + { + [[maybe_unused]] auto p = pair{13, 42u}; + } + + } + + namespace test_non_type_auto_template_parameters + { + + template + struct B + {}; + + B<5> b1; + B<'a'> b2; + + } + + namespace test_structured_bindings + { + + int arr[2] = { 1, 2 }; + std::pair pr = { 1, 2 }; + + auto f1() -> int(&)[2] + { + return arr; + } + + auto f2() -> std::pair& + { + return pr; + } + + struct S + { + int x1 : 2; + volatile double y1; + }; + + S f3() + { + return {}; + } + + auto [ x1, y1 ] = f1(); + auto& [ xr1, yr1 ] = f1(); + auto [ x2, y2 ] = f2(); + auto& [ xr2, yr2 ] = f2(); + const auto [ x3, y3 ] = f3(); + + } + + namespace test_exception_spec_type_system + { + + struct Good {}; + struct Bad {}; + + void g1() noexcept; + void g2(); + + template + Bad + f(T*, T*); + + template + Good + f(T1*, T2*); + + static_assert (std::is_same_v); + + } + + namespace test_inline_variables + { + + template void f(T) + {} + + template inline T g(T) + { + return T{}; + } + + template<> inline void f<>(int) + {} + + template<> int g<>(int) + { + return 5; + } + + } + +} // namespace cxx17 + +#endif // __cplusplus < 201703L + +]]) diff --git a/m4/ax_cxx_compile_stdcxx_17.m4 b/m4/ax_cxx_compile_stdcxx_17.m4 new file mode 100644 index 00000000000..a6834171739 --- /dev/null +++ b/m4/ax_cxx_compile_stdcxx_17.m4 @@ -0,0 +1,35 @@ +# ============================================================================= +# https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx_17.html +# ============================================================================= +# +# SYNOPSIS +# +# AX_CXX_COMPILE_STDCXX_17([ext|noext], [mandatory|optional]) +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the C++17 +# standard; if necessary, add switches to CXX and CXXCPP to enable +# support. +# +# This macro is a convenience alias for calling the AX_CXX_COMPILE_STDCXX +# macro with the version set to C++17. The two optional arguments are +# forwarded literally as the second and third argument respectively. +# Please see the documentation for the AX_CXX_COMPILE_STDCXX macro for +# more information. If you want to use this macro, you also need to +# download the ax_cxx_compile_stdcxx.m4 file. +# +# LICENSE +# +# Copyright (c) 2015 Moritz Klammler +# Copyright (c) 2016 Krzesimir Nowak +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 2 + +AX_REQUIRE_DEFINED([AX_CXX_COMPILE_STDCXX]) +AC_DEFUN([AX_CXX_COMPILE_STDCXX_17], [AX_CXX_COMPILE_STDCXX([17], [$1], [$2])]) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 743829e3f5f..baefe0f123b 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -1,19 +1,24 @@ #! /usr/bin/env nix-shell -#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp gnupg1 +#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp perlPackages.NetAmazonS3 gnupg1 use strict; use Data::Dumper; use File::Basename; use File::Path; use File::Slurp; +use File::Copy; use JSON::PP; use LWP::UserAgent; +use Net::Amazon::S3; my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n"; -my $releasesDir = "/home/eelco/mnt/releases"; +my $releasesBucketName = "nix-releases"; +my $channelsBucketName = "nix-channels"; my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine"; +my $TMPDIR = $ENV{'TMPDIR'} // "/tmp"; + # FIXME: cut&paste from nixos-channel-scripts. sub fetch { my ($url, $type) = @_; @@ -41,49 +46,83 @@ sub fetch { print STDERR "Nix revision is $nixRev, version is $version\n"; -File::Path::make_path($releasesDir); -if (system("mountpoint -q $releasesDir") != 0) { - system("sshfs hydra-mirror:/releases $releasesDir") == 0 or die; -} +my $releaseDir = "nix/$releaseName"; + +my $tmpDir = "$TMPDIR/nix-release/$releaseName"; +File::Path::make_path($tmpDir); + +# S3 setup. +my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "No AWS_ACCESS_KEY_ID given."; +my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "No AWS_SECRET_ACCESS_KEY given."; + +my $s3 = Net::Amazon::S3->new( + { aws_access_key_id => $aws_access_key_id, + aws_secret_access_key => $aws_secret_access_key, + retry => 1, + host => "s3-eu-west-1.amazonaws.com", + }); -my $releaseDir = "$releasesDir/nix/$releaseName"; -File::Path::make_path($releaseDir); +my $releasesBucket = $s3->bucket($releasesBucketName) or die; + +my $s3_us = Net::Amazon::S3->new( + { aws_access_key_id => $aws_access_key_id, + aws_secret_access_key => $aws_secret_access_key, + retry => 1, + }); + +my $channelsBucket = $s3_us->bucket($channelsBucketName) or die; sub downloadFile { my ($jobName, $productNr, $dstName) = @_; my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); - my $srcFile = $buildInfo->{buildproducts}->{$productNr}->{path} or die; + my $srcFile = $buildInfo->{buildproducts}->{$productNr}->{path} or die "job '$jobName' lacks product $productNr\n"; $dstName //= basename($srcFile); - my $dstFile = "$releaseDir/" . $dstName; + my $tmpFile = "$tmpDir/$dstName"; - if (! -e $dstFile) { - print STDERR "downloading $srcFile to $dstFile...\n"; - system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$dstFile.tmp'") == 0 + if (!-e $tmpFile) { + print STDERR "downloading $srcFile to $tmpFile...\n"; + system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$tmpFile'") == 0 or die "unable to fetch $srcFile\n"; - rename("$dstFile.tmp", $dstFile) or die; } my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die; - my $sha256_actual = `nix hash-file --type sha256 '$dstFile'`; + my $sha256_actual = `nix hash-file --base16 --type sha256 '$tmpFile'`; chomp $sha256_actual; if ($sha256_expected ne $sha256_actual) { - print STDERR "file $dstFile is corrupt\n"; + print STDERR "file $tmpFile is corrupt, got $sha256_actual, expected $sha256_expected\n"; exit 1; } - write_file("$dstFile.sha256", $sha256_expected); + write_file("$tmpFile.sha256", $sha256_expected); + + if (! -e "$tmpFile.asc") { + system("gpg2 --detach-sign --armor $tmpFile") == 0 or die "unable to sign $tmpFile\n"; + } + + return $sha256_expected; +} - return ($dstFile, $sha256_expected); +downloadFile("tarball", "2"); # .tar.bz2 +my $tarballHash = downloadFile("tarball", "3"); # .tar.xz +downloadFile("binaryTarball.i686-linux", "1"); +downloadFile("binaryTarball.x86_64-linux", "1"); +downloadFile("binaryTarball.aarch64-linux", "1"); +downloadFile("binaryTarball.x86_64-darwin", "1"); +downloadFile("installerScript", "1"); + +for my $fn (glob "$tmpDir/*") { + my $name = basename($fn); + my $dstKey = "$releaseDir/" . $name; + unless (defined $releasesBucket->head_key($dstKey)) { + print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n"; + $releasesBucket->add_key_filename($dstKey, $fn) + or die $releasesBucket->err . ": " . $releasesBucket->errstr; + } } -downloadFile("tarball", "2"); # PDF -downloadFile("tarball", "3"); # .tar.bz2 -my ($tarball, $tarballHash) = downloadFile("tarball", "4"); # .tar.xz -my ($tarball_i686_linux, $tarball_i686_linux_hash) = downloadFile("binaryTarball.i686-linux", "1"); -my ($tarball_x86_64_linux, $tarball_x86_64_linux_hash) = downloadFile("binaryTarball.x86_64-linux", "1"); -my ($tarball_x86_64_darwin, $tarball_x86_64_darwin_hash) = downloadFile("binaryTarball.x86_64-darwin", "1"); +exit if $version =~ /pre/; # Update Nixpkgs in a very hacky way. system("cd $nixpkgsDir && git pull") == 0 or die; @@ -103,31 +142,29 @@ sub downloadFile { sub getStorePath { my ($jobName) = @_; my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); - die unless $buildInfo->{buildproducts}->{1}->{type} eq "nix-build"; - return $buildInfo->{buildproducts}->{1}->{path}; + for my $product (values %{$buildInfo->{buildproducts}}) { + next unless $product->{type} eq "nix-build"; + next if $product->{path} =~ /[a-z]+$/; + return $product->{path}; + } + die; } write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix", "{\n" . " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . + " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . "}\n"); system("cd $nixpkgsDir && git commit -a -m 'nix: $oldName -> $version'") == 0 or die; -# Extract the HTML manual. -File::Path::make_path("$releaseDir/manual"); - -system("tar xvf $tarball --strip-components=3 -C $releaseDir/manual --wildcards '*/doc/manual/*.html' '*/doc/manual/*.css' '*/doc/manual/*.gif' '*/doc/manual/*.png'") == 0 or die; - -if (! -e "$releaseDir/manual/index.html") { - symlink("manual.html", "$releaseDir/manual/index.html") or die; -} - # Update the "latest" symlink. -symlink("$releaseName", "$releasesDir/nix/latest-tmp") or die; -rename("$releasesDir/nix/latest-tmp", "$releasesDir/nix/latest") or die; +$channelsBucket->add_key( + "nix-latest/install", "", + { "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" }) + or die $channelsBucket->err . ": " . $channelsBucket->errstr; # Tag the release in Git. chdir("/home/eelco/Dev/nix-pristine") or die; @@ -142,11 +179,6 @@ sub getStorePath { write_file("$siteDir/nix-release.tt", "[%-\n" . "latestNixVersion = \"$version\"\n" . - "nix_hash_i686_linux = \"$tarball_i686_linux_hash\"\n" . - "nix_hash_x86_64_linux = \"$tarball_x86_64_linux_hash\"\n" . - "nix_hash_x86_64_darwin = \"$tarball_x86_64_darwin_hash\"\n" . "-%]\n"); -system("cd $siteDir && nix-shell --run 'make nix/install nix/install.sig'") == 0 or die; - system("cd $siteDir && git commit -a -m 'Nix $version released'") == 0 or die; diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile deleted file mode 100644 index 7b2865c946d..00000000000 --- a/misc/docker/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -FROM alpine - -RUN wget -O- http://nixos.org/releases/nix/nix-1.11.2/nix-1.11.2-x86_64-linux.tar.bz2 | bzcat - | tar xf - \ - && echo "nixbld:x:30000:nixbld1,nixbld2,nixbld3,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld30" >> /etc/group \ - && for i in $(seq 1 30); do echo "nixbld$i:x:$((30000 + $i)):30000:::" >> /etc/passwd; done \ - && mkdir -m 0755 /nix && USER=root sh nix-*-x86_64-linux/install \ - && echo ". /root/.nix-profile/etc/profile.d/nix.sh" >> /etc/profile \ - && rm -r /nix-*-x86_64-linux \ - && apk --update add bash tar \ - && rm -rf /var/cache/apk/* - -ONBUILD ENV \ - ENV=/etc/profile \ - PATH=/root/.nix-profile/bin:/root/.nix-profile/sbin:/bin:/sbin:/usr/bin:/usr/sbin \ - GIT_SSL_CAINFO=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt \ - NIX_SSL_CERT_FILE=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt - -ENV \ - ENV=/etc/profile \ - PATH=/root/.nix-profile/bin:/root/.nix-profile/sbin:/bin:/sbin:/usr/bin:/usr/sbin \ - GIT_SSL_CAINFO=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt \ - NIX_SSL_CERT_FILE=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt \ - NIX_PATH=/nix/var/nix/profiles/per-user/root/channels/ diff --git a/misc/emacs/README b/misc/emacs/README deleted file mode 100644 index 8c87f67d571..00000000000 --- a/misc/emacs/README +++ /dev/null @@ -1,10 +0,0 @@ -The Nix Emacs mode supports syntax highlighting, somewhat sensible -indenting, and refilling of comments. - -To enable Nix mode in Emacs, add something like this to your ~/.emacs -file: - - (load "/nix/share/emacs/site-lisp/nix-mode.el") - -This automatically causes Nix mode to be activated for all files with -extension `.nix'. diff --git a/misc/emacs/local.mk b/misc/emacs/local.mk deleted file mode 100644 index 8e06b881bcd..00000000000 --- a/misc/emacs/local.mk +++ /dev/null @@ -1 +0,0 @@ -$(eval $(call install-data-in,$(d)/nix-mode.el,$(datadir)/emacs/site-lisp)) diff --git a/misc/emacs/nix-mode.el b/misc/emacs/nix-mode.el deleted file mode 100644 index e129e9efe1d..00000000000 --- a/misc/emacs/nix-mode.el +++ /dev/null @@ -1,177 +0,0 @@ -;;; nix-mode.el --- Major mode for editing Nix expressions - -;; Author: Eelco Dolstra -;; URL: https://github.com/NixOS/nix/tree/master/misc/emacs -;; Version: 1.0 - -;;; Commentary: - -;;; Code: - -(defun nix-syntax-match-antiquote (limit) - (let ((pos (next-single-char-property-change (point) 'nix-syntax-antiquote - nil limit))) - (when (and pos (> pos (point))) - (goto-char pos) - (let ((char (char-after pos))) - (pcase char - (`?$ - (forward-char 2)) - (`?} - (forward-char 1))) - (set-match-data (list pos (point))) - t)))) - -(defconst nix-font-lock-keywords - '("\\_" "\\_" "\\_" "\\_" "\\_" - "\\_" "\\_" "\\_" "\\_" "\\_" - ("\\_" . font-lock-builtin-face) - ("\\_" . font-lock-builtin-face) - ("\\_" . font-lock-builtin-face) - ("\\_" . font-lock-builtin-face) - ("\\_" . font-lock-builtin-face) - ("\\_" . font-lock-builtin-face) - ("\\_" . font-lock-builtin-face) - ("\\_" . font-lock-builtin-face) - ("[a-zA-Z][a-zA-Z0-9\\+-\\.]*:[a-zA-Z0-9%/\\?:@&=\\+\\$,_\\.!~\\*'-]+" - . font-lock-constant-face) - ("\\<\\([a-zA-Z_][a-zA-Z0-9_'\-\.]*\\)[ \t]*=" - (1 font-lock-variable-name-face nil nil)) - ("<[a-zA-Z0-9._\\+-]+\\(/[a-zA-Z0-9._\\+-]+\\)*>" - . font-lock-constant-face) - ("[a-zA-Z0-9._\\+-]*\\(/[a-zA-Z0-9._\\+-]+\\)+" - . font-lock-constant-face) - (nix-syntax-match-antiquote 0 font-lock-preprocessor-face t)) - "Font lock keywords for nix.") - -(defvar nix-mode-syntax-table - (let ((table (make-syntax-table))) - (modify-syntax-entry ?/ ". 14" table) - (modify-syntax-entry ?* ". 23" table) - (modify-syntax-entry ?# "< b" table) - (modify-syntax-entry ?\n "> b" table) - table) - "Syntax table for Nix mode.") - -(defun nix-syntax-propertize-escaped-antiquote () - "Set syntax properies for escaped antiquote marks." - nil) - -(defun nix-syntax-propertize-multiline-string () - "Set syntax properies for multiline string delimiters." - (let* ((start (match-beginning 0)) - (end (match-end 0)) - (context (save-excursion (save-match-data (syntax-ppss start)))) - (string-type (nth 3 context))) - (pcase string-type - (`t - ;; inside a multiline string - ;; ending multi-line string delimiter - (put-text-property (1- end) end - 'syntax-table (string-to-syntax "|"))) - (`nil - ;; beginning multi-line string delimiter - (put-text-property start (1+ start) - 'syntax-table (string-to-syntax "|")))))) - -(defun nix-syntax-propertize-antiquote () - "Set syntax properties for antiquote marks." - (let* ((start (match-beginning 0))) - (put-text-property start (1+ start) - 'syntax-table (string-to-syntax "|")) - (put-text-property start (+ start 2) - 'nix-syntax-antiquote t))) - -(defun nix-syntax-propertize-close-brace () - "Set syntax properties for close braces. -If a close brace `}' ends an antiquote, the next character begins a string." - (let* ((start (match-beginning 0)) - (end (match-end 0)) - (context (save-excursion (save-match-data (syntax-ppss start)))) - (open (nth 1 context))) - (when open ;; a corresponding open-brace was found - (let* ((antiquote (get-text-property open 'nix-syntax-antiquote))) - (when antiquote - (put-text-property (+ start 1) (+ start 2) - 'syntax-table (string-to-syntax "|")) - (put-text-property start (1+ start) - 'nix-syntax-antiquote t)))))) - -(defun nix-syntax-propertize (start end) - "Special syntax properties for Nix." - ;; search for multi-line string delimiters - (goto-char start) - (remove-text-properties start end '(syntax-table nil nix-syntax-antiquote nil)) - (funcall - (syntax-propertize-rules - ("''\\${" - (0 (ignore (nix-syntax-propertize-escaped-antiquote)))) - ("''" - (0 (ignore (nix-syntax-propertize-multiline-string)))) - ("\\${" - (0 (ignore (nix-syntax-propertize-antiquote)))) - ("}" - (0 (ignore (nix-syntax-propertize-close-brace))))) - start end)) - -(defun nix-indent-line () - "Indent current line in a Nix expression." - (interactive) - (indent-relative-maybe)) - - -;;;###autoload -(define-derived-mode nix-mode prog-mode "Nix" - "Major mode for editing Nix expressions. - -The following commands may be useful: - - '\\[newline-and-indent]' - Insert a newline and move the cursor to align with the previous - non-empty line. - - '\\[fill-paragraph]' - Refill a paragraph so that all lines are at most `fill-column' - lines long. This should do the right thing for comments beginning - with `#'. However, this command doesn't work properly yet if the - comment is adjacent to code (i.e., no intervening empty lines). - In that case, select the text to be refilled and use - `\\[fill-region]' instead. - -The hook `nix-mode-hook' is run when Nix mode is started. - -\\{nix-mode-map} -" - (set-syntax-table nix-mode-syntax-table) - - ;; Font lock support. - (setq-local font-lock-defaults '(nix-font-lock-keywords nil nil nil nil)) - - ;; Special syntax properties for Nix - (setq-local syntax-propertize-function 'nix-syntax-propertize) - - ;; Look at text properties when parsing - (setq-local parse-sexp-lookup-properties t) - - ;; Automatic indentation [C-j]. - (set (make-local-variable 'indent-line-function) 'nix-indent-line) - - ;; Indenting of comments. - (set (make-local-variable 'comment-start) "# ") - (set (make-local-variable 'comment-end) "") - (set (make-local-variable 'comment-start-skip) "\\(^\\|\\s-\\);?#+ *") - - ;; Filling of comments. - (set (make-local-variable 'adaptive-fill-mode) t) - (set (make-local-variable 'paragraph-start) "[ \t]*\\(#+[ \t]*\\)?$") - (set (make-local-variable 'paragraph-separate) paragraph-start)) - - -;;;###autoload -(progn - (add-to-list 'auto-mode-alist '("\\.nix\\'" . nix-mode)) - (add-to-list 'auto-mode-alist '("\\.nix.in\\'" . nix-mode))) - -(provide 'nix-mode) - -;;; nix-mode.el ends here diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index c5ef97ee9a3..9f26296a9d7 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -2,20 +2,26 @@ + EnvironmentVariables + + OBJC_DISABLE_INITIALIZE_FORK_SAFETY + YES + Label org.nixos.nix-daemon + KeepAlive + RunAtLoad - Program - @bindir@/nix-daemon + ProgramArguments + + /bin/sh + -c + /bin/wait4path /nix/var/nix/profiles/default/bin/nix-daemon && /nix/var/nix/profiles/default/bin/nix-daemon + StandardErrorPath /var/log/nix-daemon.log StandardOutPath /dev/null - EnvironmentVariables - - NIX_SSL_CERT_FILE - /nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt - diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in index fcd799e177d..25655204d4d 100644 --- a/misc/systemd/nix-daemon.service.in +++ b/misc/systemd/nix-daemon.service.in @@ -7,4 +7,6 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket [Service] ExecStart=@@bindir@/nix-daemon nix-daemon --daemon KillMode=process -Environment=XDG_CACHE_HOME=/root/.cache + +[Install] +WantedBy=multi-user.target diff --git a/mk/README.md b/mk/README.md deleted file mode 100644 index e4cd742b4c7..00000000000 --- a/mk/README.md +++ /dev/null @@ -1,6 +0,0 @@ -This is a set of helper Makefiles for doing non-recursive builds with -GNU Make. The canonical source can be found at -https://github.com/edolstra/make-rules. You should copy the files -into the `mk` subdirectory of your project. - -TODO: write more documentation. diff --git a/mk/lib.mk b/mk/lib.mk index bb82801d3b4..1da51d87973 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -53,8 +53,8 @@ BUILD_SHARED_LIBS ?= 1 ifeq ($(BUILD_SHARED_LIBS), 1) ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) - GLOBAL_CFLAGS += -U__STRICT_ANSI__ - GLOBAL_CXXFLAGS += -U__STRICT_ANSI__ + GLOBAL_CFLAGS += -U__STRICT_ANSI__ -D_GNU_SOURCE + GLOBAL_CXXFLAGS += -U__STRICT_ANSI__ -D_GNU_SOURCE else GLOBAL_CFLAGS += -fPIC GLOBAL_CXXFLAGS += -fPIC diff --git a/mk/libraries.mk b/mk/libraries.mk index 3cd7a53107b..e6ef2e3ec7d 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -45,6 +45,11 @@ endif # - $(1)_INSTALL_DIR: the directory where the library will be # installed. Defaults to $(libdir). # +# - $(1)_EXCLUDE_FROM_LIBRARY_LIST: if defined, the library will not +# be automatically marked as a dependency of the top-level all +# target andwill not be listed in the make help output. This is +# useful for libraries built solely for testing, for example. +# # - BUILD_SHARED_LIBS: if equal to ‘1’, a dynamic library will be # built, otherwise a static library. define build-library @@ -86,7 +91,7 @@ define build-library $(1)_PATH := $$(_d)/$$($(1)_NAME).$(SO_EXT) $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/ - $$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED) + $$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED) ifneq ($(OS), Darwin) $(1)_LDFLAGS_USE += -Wl,-rpath,$$(abspath $$(_d)) @@ -100,7 +105,7 @@ define build-library $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) $$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ - $$(trace-ld) $(CXX) -o $$@ -shared $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) + $$(trace-ld) $(CXX) -o $$@ -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) $(1)_LDFLAGS_USE_INSTALLED += -L$$(DESTDIR)$$($(1)_INSTALL_DIR) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME))) ifneq ($(OS), Darwin) @@ -120,7 +125,8 @@ define build-library $(1)_PATH := $$(_d)/$$($(1)_NAME).a $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/ - $(trace-ar) ar crs $$@ $$? + $(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$? + $(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS) @@ -149,7 +155,9 @@ define build-library $(1)_DEPS := $$(foreach fn, $$($(1)_OBJS), $$(call filename-to-dep, $$(fn))) -include $$($(1)_DEPS) + ifndef $(1)_EXCLUDE_FROM_LIBRARY_LIST libs-list += $$($(1)_PATH) + endif clean-files += $$(_d)/*.a $$(_d)/*.$(SO_EXT) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS) dist-files += $$(_srcs) endef diff --git a/mk/patterns.mk b/mk/patterns.mk index 3219d9629fb..7319f4cddee 100644 --- a/mk/patterns.mk +++ b/mk/patterns.mk @@ -1,10 +1,10 @@ $(buildprefix)%.o: %.cc @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP $(buildprefix)%.o: %.cpp @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP $(buildprefix)%.o: %.c @mkdir -p "$(dir $@)" diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk new file mode 100644 index 00000000000..1c0452dc2a5 --- /dev/null +++ b/mk/precompiled-headers.mk @@ -0,0 +1,42 @@ +PRECOMPILE_HEADERS ?= 1 + +print-var-help += \ + echo " PRECOMPILE_HEADERS ($(PRECOMPILE_HEADERS)): Whether to use precompiled headers to speed up the build"; + +GCH = $(buildprefix)precompiled-headers.h.gch + +$(GCH): precompiled-headers.h + @rm -f $@ + @mkdir -p "$(dir $@)" + $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) + +PCH = $(buildprefix)precompiled-headers.h.pch + +$(PCH): precompiled-headers.h + @rm -f $@ + @mkdir -p "$(dir $@)" + $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) + +clean-files += $(GCH) $(PCH) + +ifeq ($(PRECOMPILE_HEADERS), 1) + + ifeq ($(CXX), g++) + + GLOBAL_CXXFLAGS_PCH += -include $(buildprefix)precompiled-headers.h -Winvalid-pch + + GLOBAL_ORDER_AFTER += $(GCH) + + else ifeq ($(CXX), clang++) + + GLOBAL_CXXFLAGS_PCH += -include-pch $(PCH) -Winvalid-pch + + GLOBAL_ORDER_AFTER += $(PCH) + + else + + $(error Don't know how to precompile headers on $(CXX)) + + endif + +endif diff --git a/mk/programs.mk b/mk/programs.mk index 3ac64494e3a..3fa9685c39d 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -32,27 +32,31 @@ define build-program $$(eval $$(call create-dir, $$(_d))) $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/ - $$(trace-ld) $(CXX) -o $$@ $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) + $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $(1)_INSTALL_DIR ?= $$(bindir) - $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1) - $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) + ifdef $(1)_INSTALL_DIR - install: $(DESTDIR)$$($(1)_INSTALL_PATH) + $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1) - ifeq ($(BUILD_SHARED_LIBS), 1) + $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) - _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH)) + install: $(DESTDIR)$$($(1)_INSTALL_PATH) - $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ - $$(trace-ld) $(CXX) -o $$@ $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) + ifeq ($(BUILD_SHARED_LIBS), 1) - else + _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH)) - $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ - install -t $$($(1)_INSTALL_DIR) $$< + $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ + $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) + else + + $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ + install -t $(DESTDIR)$$($(1)_INSTALL_DIR) $$< + + endif endif # Propagate CFLAGS and CXXFLAGS to the individual object files. @@ -76,4 +80,10 @@ define build-program programs-list += $$($(1)_PATH) clean-files += $$($(1)_PATH) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS) dist-files += $$(_srcs) + + # Phony target to run this program (typically as a dependency of 'check'). + .PHONY: $(1)_RUN + $(1)_RUN: $$($(1)_PATH) + $(trace-test) $$($(1)_PATH) + endef diff --git a/mk/tests.mk b/mk/tests.mk index 004a4802861..70c30661b95 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -7,20 +7,39 @@ define run-install-test endef +# Color code from https://unix.stackexchange.com/a/10065 installcheck: - @total=0; failed=0; for i in $(_installcheck-list); do \ + @total=0; failed=0; \ + red=""; \ + green=""; \ + yellow=""; \ + normal=""; \ + if [ -t 1 ]; then \ + red=""; \ + green=""; \ + yellow=""; \ + normal=""; \ + fi; \ + for i in $(_installcheck-list); do \ total=$$((total + 1)); \ - echo "running test $$i"; \ - if (cd $$(dirname $$i) && $(tests-environment) $$(basename $$i)); then \ - echo "PASS: $$i"; \ + printf "running test $$i..."; \ + log="$$(cd $$(dirname $$i) && $(tests-environment) $$(basename $$i) 2>&1)"; \ + status=$$?; \ + if [ $$status -eq 0 ]; then \ + echo " [$${green}PASS$$normal]"; \ + elif [ $$status -eq 99 ]; then \ + echo " [$${yellow}SKIP$$normal]"; \ else \ - echo "FAIL: $$i"; \ + echo " [$${red}FAIL$$normal]"; \ + echo "$$log" | sed 's/^/ /'; \ failed=$$((failed + 1)); \ fi; \ done; \ if [ "$$failed" != 0 ]; then \ - echo "$$failed out of $$total tests failed "; \ + echo "$${red}$$failed out of $$total tests failed $$normal"; \ exit 1; \ + else \ + echo "$${green}All tests succeeded$$normal"; \ fi .PHONY: check installcheck diff --git a/mk/tracing.mk b/mk/tracing.mk index 13912d3c782..54c77ab6075 100644 --- a/mk/tracing.mk +++ b/mk/tracing.mk @@ -11,6 +11,7 @@ ifeq ($(V), 0) trace-javac = @echo " JAVAC " $@; trace-jar = @echo " JAR " $@; trace-mkdir = @echo " MKDIR " $@; + trace-test = @echo " TEST " $@; suppress = @ diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock new file mode 100644 index 00000000000..957c01e5a51 --- /dev/null +++ b/nix-rust/Cargo.lock @@ -0,0 +1,399 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "assert_matches" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "autocfg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bit-set" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bit-vec" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "byteorder" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "c2-chacha" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fnv" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "getrandom" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hex" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.66" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "nix-rust" +version = "0.1.0" +dependencies = [ + "assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "proptest 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-traits" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "proptest" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", + "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "rusty-fork 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "quick-error" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_jitter 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_chacha" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "redox_syscall" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "regex-syntax" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "remove_dir_all" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rusty-fork" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wait-timeout 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tempfile" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", + "remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "wasi" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[metadata] +"checksum assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7deb0a829ca7bcfaf5da70b073a8d128619259a7be8216a355e23f00763059e5" +"checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" +"checksum bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e84c238982c4b1e1ee668d136c510c67a13465279c0cb367ea6baf6310620a80" +"checksum bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f59bbe95d4e52a6398ec21238d31577f2b28a9d86807f06ca59d191d8440d0bb" +"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" +"checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb" +"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" +"checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" +"checksum getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "e7db7ca94ed4cd01190ceee0d8a8052f08a247aa1b469a7f68c6a3b71afcf407" +"checksum hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77" +"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +"checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558" +"checksum num-traits 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c81ffc11c212fa327657cb19dd85eb7419e163b5b076bede2bdb5c974c07e4" +"checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" +"checksum proptest 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cf147e022eacf0c8a054ab864914a7602618adba841d800a9a9868a5237a529f" +"checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" +"checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +"checksum rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3ae1b169243eaf61759b8475a998f0a385e42042370f3a7dbaf35246eacc8412" +"checksum rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +"checksum rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "03a2a90da8c7523f554344f921aa97283eadf6ac484a6d2a7d0212fa7f8d6853" +"checksum rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +"checksum rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" +"checksum rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +"checksum rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +"checksum rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +"checksum rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +"checksum rand_jitter 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +"checksum rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +"checksum rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +"checksum rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +"checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" +"checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716" +"checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e" +"checksum rusty-fork 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3dd93264e10c577503e926bd1430193eeb5d21b059148910082245309b424fae" +"checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +"checksum wait-timeout 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +"checksum wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d" +"checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml new file mode 100644 index 00000000000..1372e5a7350 --- /dev/null +++ b/nix-rust/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "nix-rust" +version = "0.1.0" +authors = ["Eelco Dolstra "] +edition = "2018" + +[lib] +name = "nixrust" +crate-type = ["cdylib"] + +[dependencies] +libc = "0.2" +#futures-preview = { version = "=0.3.0-alpha.19" } +#hyper = "0.13.0-alpha.4" +#http = "0.1" +#tokio = { version = "0.2.0-alpha.6", default-features = false, features = ["rt-full"] } +lazy_static = "1.4" +#byteorder = "1.3" + +[dev-dependencies] +hex = "0.3" +assert_matches = "1.3" +proptest = "0.9" diff --git a/nix-rust/local.mk b/nix-rust/local.mk new file mode 100644 index 00000000000..e4bfde31b8b --- /dev/null +++ b/nix-rust/local.mk @@ -0,0 +1,45 @@ +ifeq ($(OPTIMIZE), 1) + RUST_MODE = --release + RUST_DIR = release +else + RUST_MODE = + RUST_DIR = debug +endif + +libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.$(SO_EXT) +libnixrust_INSTALL_PATH := $(libdir)/libnixrust.$(SO_EXT) +libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust -ldl +libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust -ldl + +ifeq ($(OS), Darwin) +libnixrust_BUILD_FLAGS = NIX_LDFLAGS="-undefined dynamic_lookup" +else +libnixrust_LDFLAGS_USE += -Wl,-rpath,$(abspath $(d)/target/$(RUST_DIR)) +libnixrust_LDFLAGS_USE_INSTALLED += -Wl,-rpath,$(libdir) +endif + +$(libnixrust_PATH): $(call rwildcard, $(d)/src, *.rs) $(d)/Cargo.toml + $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) \ + $(libnixrust_BUILD_FLAGS) \ + cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \ + && touch target/$(RUST_DIR)/libnixrust.$(SO_EXT) + +$(libnixrust_INSTALL_PATH): $(libnixrust_PATH) + $(target-gen) cp $^ $@ +ifeq ($(OS), Darwin) + install_name_tool -id $@ $@ +endif + +dist-files += $(d)/vendor + +clean: clean-rust + +clean-rust: + $(suppress) rm -rfv nix-rust/target + +ifneq ($(OS), Darwin) +check: rust-tests + +rust-tests: + $(trace-test) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo test --release $$(if [[ -d vendor ]]; then echo --offline; fi) +endif diff --git a/nix-rust/src/c.rs b/nix-rust/src/c.rs new file mode 100644 index 00000000000..c1358545fd1 --- /dev/null +++ b/nix-rust/src/c.rs @@ -0,0 +1,77 @@ +use super::{error, store::path, store::StorePath, util}; + +#[no_mangle] +pub unsafe extern "C" fn ffi_String_new(s: &str, out: *mut String) { + // FIXME: check whether 's' is valid UTF-8? + out.write(s.to_string()) +} + +#[no_mangle] +pub unsafe extern "C" fn ffi_String_drop(self_: *mut String) { + std::ptr::drop_in_place(self_); +} + +#[no_mangle] +pub extern "C" fn ffi_StorePath_new( + path: &str, + store_dir: &str, +) -> Result { + StorePath::new(std::path::Path::new(path), std::path::Path::new(store_dir)) + .map_err(|err| err.into()) +} + +#[no_mangle] +pub extern "C" fn ffi_StorePath_new2( + hash: &[u8; crate::store::path::STORE_PATH_HASH_BYTES], + name: &str, +) -> Result { + StorePath::from_parts(*hash, name).map_err(|err| err.into()) +} + +#[no_mangle] +pub extern "C" fn ffi_StorePath_fromBaseName( + base_name: &str, +) -> Result { + StorePath::new_from_base_name(base_name).map_err(|err| err.into()) +} + +#[no_mangle] +pub unsafe extern "C" fn ffi_StorePath_drop(self_: *mut StorePath) { + std::ptr::drop_in_place(self_); +} + +#[no_mangle] +pub extern "C" fn ffi_StorePath_to_string(self_: &StorePath) -> Vec { + let mut buf = vec![0; path::STORE_PATH_HASH_CHARS + 1 + self_.name.name().len()]; + util::base32::encode_into(self_.hash.hash(), &mut buf[0..path::STORE_PATH_HASH_CHARS]); + buf[path::STORE_PATH_HASH_CHARS] = b'-'; + buf[path::STORE_PATH_HASH_CHARS + 1..].clone_from_slice(self_.name.name().as_bytes()); + buf +} + +#[no_mangle] +pub extern "C" fn ffi_StorePath_less_than(a: &StorePath, b: &StorePath) -> bool { + a < b +} + +#[no_mangle] +pub extern "C" fn ffi_StorePath_eq(a: &StorePath, b: &StorePath) -> bool { + a == b +} + +#[no_mangle] +pub extern "C" fn ffi_StorePath_clone(self_: &StorePath) -> StorePath { + self_.clone() +} + +#[no_mangle] +pub extern "C" fn ffi_StorePath_name(self_: &StorePath) -> &str { + self_.name.name() +} + +#[no_mangle] +pub extern "C" fn ffi_StorePath_hash_data( + self_: &StorePath, +) -> &[u8; crate::store::path::STORE_PATH_HASH_BYTES] { + self_.hash.hash() +} diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs new file mode 100644 index 00000000000..bb0c9a9338d --- /dev/null +++ b/nix-rust/src/error.rs @@ -0,0 +1,118 @@ +use std::fmt; + +#[derive(Debug)] +pub enum Error { + InvalidPath(crate::store::StorePath), + BadStorePath(std::path::PathBuf), + NotInStore(std::path::PathBuf), + BadNarInfo, + BadBase32, + StorePathNameEmpty, + StorePathNameTooLong, + BadStorePathName, + NarSizeFieldTooBig, + BadNarString, + BadNarPadding, + BadNarVersionMagic, + MissingNarOpenTag, + MissingNarCloseTag, + MissingNarField, + BadNarField(String), + BadExecutableField, + IOError(std::io::Error), + #[cfg(unused)] + HttpError(hyper::error::Error), + Misc(String), + #[cfg(not(test))] + Foreign(CppException), + BadTarFileMemberName(String), +} + +impl From for Error { + fn from(err: std::io::Error) -> Self { + Error::IOError(err) + } +} + +#[cfg(unused)] +impl From for Error { + fn from(err: hyper::error::Error) -> Self { + Error::HttpError(err) + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Error::InvalidPath(_) => write!(f, "invalid path"), + Error::BadNarInfo => write!(f, ".narinfo file is corrupt"), + Error::BadStorePath(path) => write!(f, "path '{}' is not a store path", path.display()), + Error::NotInStore(path) => { + write!(f, "path '{}' is not in the Nix store", path.display()) + } + Error::BadBase32 => write!(f, "invalid base32 string"), + Error::StorePathNameEmpty => write!(f, "store path name is empty"), + Error::StorePathNameTooLong => { + write!(f, "store path name is longer than 211 characters") + } + Error::BadStorePathName => write!(f, "store path name contains forbidden character"), + Error::NarSizeFieldTooBig => write!(f, "size field in NAR is too big"), + Error::BadNarString => write!(f, "NAR string is not valid UTF-8"), + Error::BadNarPadding => write!(f, "NAR padding is not zero"), + Error::BadNarVersionMagic => write!(f, "unsupported NAR version"), + Error::MissingNarOpenTag => write!(f, "NAR open tag is missing"), + Error::MissingNarCloseTag => write!(f, "NAR close tag is missing"), + Error::MissingNarField => write!(f, "expected NAR field is missing"), + Error::BadNarField(s) => write!(f, "unrecognized NAR field '{}'", s), + Error::BadExecutableField => write!(f, "bad 'executable' field in NAR"), + Error::IOError(err) => write!(f, "I/O error: {}", err), + #[cfg(unused)] + Error::HttpError(err) => write!(f, "HTTP error: {}", err), + #[cfg(not(test))] + Error::Foreign(_) => write!(f, ""), // FIXME + Error::Misc(s) => write!(f, "{}", s), + Error::BadTarFileMemberName(s) => { + write!(f, "tar archive contains illegal file name '{}'", s) + } + } + } +} + +#[cfg(not(test))] +impl From for CppException { + fn from(err: Error) -> Self { + match err { + Error::Foreign(ex) => ex, + _ => CppException::new(&err.to_string()), + } + } +} + +#[cfg(not(test))] +#[repr(C)] +#[derive(Debug)] +pub struct CppException(*const libc::c_void); // == std::exception_ptr* + +#[cfg(not(test))] +impl CppException { + fn new(s: &str) -> Self { + Self(unsafe { make_error(s) }) + } +} + +#[cfg(not(test))] +impl Drop for CppException { + fn drop(&mut self) { + unsafe { + destroy_error(self.0); + } + } +} + +#[cfg(not(test))] +extern "C" { + #[allow(improper_ctypes)] // YOLO + fn make_error(s: &str) -> *const libc::c_void; + + fn destroy_error(exc: *const libc::c_void); +} diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs new file mode 100644 index 00000000000..27ea69fbdda --- /dev/null +++ b/nix-rust/src/lib.rs @@ -0,0 +1,9 @@ +#[cfg(not(test))] +mod c; +mod error; +#[cfg(unused)] +mod nar; +mod store; +mod util; + +pub use error::Error; diff --git a/nix-rust/src/nar.rs b/nix-rust/src/nar.rs new file mode 100644 index 00000000000..cb520935e63 --- /dev/null +++ b/nix-rust/src/nar.rs @@ -0,0 +1,126 @@ +use crate::Error; +use byteorder::{LittleEndian, ReadBytesExt}; +use std::convert::TryFrom; +use std::io::Read; + +pub fn parse(input: &mut R) -> Result<(), Error> { + if String::read(input)? != NAR_VERSION_MAGIC { + return Err(Error::BadNarVersionMagic); + } + + parse_file(input) +} + +const NAR_VERSION_MAGIC: &str = "nix-archive-1"; + +fn parse_file(input: &mut R) -> Result<(), Error> { + if String::read(input)? != "(" { + return Err(Error::MissingNarOpenTag); + } + + if String::read(input)? != "type" { + return Err(Error::MissingNarField); + } + + match String::read(input)?.as_ref() { + "regular" => { + let mut _executable = false; + let mut tag = String::read(input)?; + if tag == "executable" { + _executable = true; + if String::read(input)? != "" { + return Err(Error::BadExecutableField); + } + tag = String::read(input)?; + } + if tag != "contents" { + return Err(Error::MissingNarField); + } + let _contents = Vec::::read(input)?; + if String::read(input)? != ")" { + return Err(Error::MissingNarCloseTag); + } + } + "directory" => loop { + match String::read(input)?.as_ref() { + "entry" => { + if String::read(input)? != "(" { + return Err(Error::MissingNarOpenTag); + } + if String::read(input)? != "name" { + return Err(Error::MissingNarField); + } + let _name = String::read(input)?; + if String::read(input)? != "node" { + return Err(Error::MissingNarField); + } + parse_file(input)?; + let tag = String::read(input)?; + if tag != ")" { + return Err(Error::MissingNarCloseTag); + } + } + ")" => break, + s => return Err(Error::BadNarField(s.into())), + } + }, + "symlink" => { + if String::read(input)? != "target" { + return Err(Error::MissingNarField); + } + let _target = String::read(input)?; + if String::read(input)? != ")" { + return Err(Error::MissingNarCloseTag); + } + } + s => return Err(Error::BadNarField(s.into())), + } + + Ok(()) +} + +trait Deserialize: Sized { + fn read(input: &mut R) -> Result; +} + +impl Deserialize for String { + fn read(input: &mut R) -> Result { + let buf = Deserialize::read(input)?; + Ok(String::from_utf8(buf).map_err(|_| Error::BadNarString)?) + } +} + +impl Deserialize for Vec { + fn read(input: &mut R) -> Result { + let n: usize = Deserialize::read(input)?; + let mut buf = vec![0; n]; + input.read_exact(&mut buf)?; + skip_padding(input, n)?; + Ok(buf) + } +} + +fn skip_padding(input: &mut R, len: usize) -> Result<(), Error> { + if len % 8 != 0 { + let mut buf = [0; 8]; + let buf = &mut buf[0..8 - (len % 8)]; + input.read_exact(buf)?; + if !buf.iter().all(|b| *b == 0) { + return Err(Error::BadNarPadding); + } + } + Ok(()) +} + +impl Deserialize for u64 { + fn read(input: &mut R) -> Result { + Ok(input.read_u64::()?) + } +} + +impl Deserialize for usize { + fn read(input: &mut R) -> Result { + let n: u64 = Deserialize::read(input)?; + Ok(usize::try_from(n).map_err(|_| Error::NarSizeFieldTooBig)?) + } +} diff --git a/nix-rust/src/store/binary_cache_store.rs b/nix-rust/src/store/binary_cache_store.rs new file mode 100644 index 00000000000..9e1e88b7c85 --- /dev/null +++ b/nix-rust/src/store/binary_cache_store.rs @@ -0,0 +1,48 @@ +use super::{PathInfo, Store, StorePath}; +use crate::Error; +use hyper::client::Client; + +pub struct BinaryCacheStore { + base_uri: String, + client: Client, +} + +impl BinaryCacheStore { + pub fn new(base_uri: String) -> Self { + Self { + base_uri, + client: Client::new(), + } + } +} + +impl Store for BinaryCacheStore { + fn query_path_info( + &self, + path: &StorePath, + ) -> std::pin::Pin> + Send>> { + let uri = format!("{}/{}.narinfo", self.base_uri.clone(), path.hash); + let path = path.clone(); + let client = self.client.clone(); + let store_dir = self.store_dir().to_string(); + + Box::pin(async move { + let response = client.get(uri.parse::().unwrap()).await?; + + if response.status() == hyper::StatusCode::NOT_FOUND + || response.status() == hyper::StatusCode::FORBIDDEN + { + return Err(Error::InvalidPath(path)); + } + + let mut body = response.into_body(); + + let mut bytes = Vec::new(); + while let Some(next) = body.next().await { + bytes.extend(next?); + } + + PathInfo::parse_nar_info(std::str::from_utf8(&bytes).unwrap(), &store_dir) + }) + } +} diff --git a/nix-rust/src/store/mod.rs b/nix-rust/src/store/mod.rs new file mode 100644 index 00000000000..da972482c61 --- /dev/null +++ b/nix-rust/src/store/mod.rs @@ -0,0 +1,17 @@ +pub mod path; + +#[cfg(unused)] +mod binary_cache_store; +#[cfg(unused)] +mod path_info; +#[cfg(unused)] +mod store; + +pub use path::{StorePath, StorePathHash, StorePathName}; + +#[cfg(unused)] +pub use binary_cache_store::BinaryCacheStore; +#[cfg(unused)] +pub use path_info::PathInfo; +#[cfg(unused)] +pub use store::Store; diff --git a/nix-rust/src/store/path.rs b/nix-rust/src/store/path.rs new file mode 100644 index 00000000000..47b5975c034 --- /dev/null +++ b/nix-rust/src/store/path.rs @@ -0,0 +1,225 @@ +use crate::error::Error; +use crate::util::base32; +use std::fmt; +use std::path::Path; + +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub struct StorePath { + pub hash: StorePathHash, + pub name: StorePathName, +} + +pub const STORE_PATH_HASH_BYTES: usize = 20; +pub const STORE_PATH_HASH_CHARS: usize = 32; + +impl StorePath { + pub fn new(path: &Path, store_dir: &Path) -> Result { + if path.parent() != Some(store_dir) { + return Err(Error::NotInStore(path.into())); + } + Self::new_from_base_name( + path.file_name() + .ok_or(Error::BadStorePath(path.into()))? + .to_str() + .ok_or(Error::BadStorePath(path.into()))?, + ) + } + + pub fn from_parts(hash: [u8; STORE_PATH_HASH_BYTES], name: &str) -> Result { + Ok(StorePath { + hash: StorePathHash(hash), + name: StorePathName::new(name)?, + }) + } + + pub fn new_from_base_name(base_name: &str) -> Result { + if base_name.len() < STORE_PATH_HASH_CHARS + 1 + || base_name.as_bytes()[STORE_PATH_HASH_CHARS] != '-' as u8 + { + return Err(Error::BadStorePath(base_name.into())); + } + + Ok(StorePath { + hash: StorePathHash::new(&base_name[0..STORE_PATH_HASH_CHARS])?, + name: StorePathName::new(&base_name[STORE_PATH_HASH_CHARS + 1..])?, + }) + } +} + +impl fmt::Display for StorePath { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}-{}", self.hash, self.name) + } +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct StorePathHash([u8; STORE_PATH_HASH_BYTES]); + +impl StorePathHash { + pub fn new(s: &str) -> Result { + assert_eq!(s.len(), STORE_PATH_HASH_CHARS); + let v = base32::decode(s)?; + assert_eq!(v.len(), STORE_PATH_HASH_BYTES); + let mut bytes: [u8; 20] = Default::default(); + bytes.copy_from_slice(&v[0..STORE_PATH_HASH_BYTES]); + Ok(Self(bytes)) + } + + pub fn hash<'a>(&'a self) -> &'a [u8; STORE_PATH_HASH_BYTES] { + &self.0 + } +} + +impl fmt::Display for StorePathHash { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut buf = vec![0; STORE_PATH_HASH_CHARS]; + base32::encode_into(&self.0, &mut buf); + f.write_str(std::str::from_utf8(&buf).unwrap()) + } +} + +impl Ord for StorePathHash { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + // Historically we've sorted store paths by their base32 + // serialization, but our base32 encodes bytes in reverse + // order. So compare them in reverse order as well. + self.0.iter().rev().cmp(other.0.iter().rev()) + } +} + +impl PartialOrd for StorePathHash { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub struct StorePathName(String); + +impl StorePathName { + pub fn new(s: &str) -> Result { + if s.len() == 0 { + return Err(Error::StorePathNameEmpty); + } + + if s.len() > 211 { + return Err(Error::StorePathNameTooLong); + } + + if s.starts_with('.') + || !s.chars().all(|c| { + c.is_ascii_alphabetic() + || c.is_ascii_digit() + || c == '+' + || c == '-' + || c == '.' + || c == '_' + || c == '?' + || c == '=' + }) + { + return Err(Error::BadStorePathName); + } + + Ok(Self(s.to_string())) + } + + pub fn name<'a>(&'a self) -> &'a str { + &self.0 + } +} + +impl fmt::Display for StorePathName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use assert_matches::assert_matches; + + #[test] + fn test_parse() { + let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-konsole-18.12.3"; + let p = StorePath::new_from_base_name(&s).unwrap(); + assert_eq!(p.name.0, "konsole-18.12.3"); + assert_eq!( + p.hash.0, + [ + 0x9f, 0x76, 0x49, 0x20, 0xf6, 0x5d, 0xe9, 0x71, 0xc4, 0xca, 0x46, 0x21, 0xab, 0xff, + 0x9b, 0x44, 0xef, 0x87, 0x0f, 0x3c + ] + ); + } + + #[test] + fn test_no_name() { + let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-"; + assert_matches!( + StorePath::new_from_base_name(&s), + Err(Error::StorePathNameEmpty) + ); + } + + #[test] + fn test_no_dash() { + let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz"; + assert_matches!( + StorePath::new_from_base_name(&s), + Err(Error::BadStorePath(_)) + ); + } + + #[test] + fn test_short_hash() { + let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxl-konsole-18.12.3"; + assert_matches!( + StorePath::new_from_base_name(&s), + Err(Error::BadStorePath(_)) + ); + } + + #[test] + fn test_invalid_hash() { + let s = "7h7qgvs4kgzsn8e6rb273saxyqh4jxlz-konsole-18.12.3"; + assert_matches!(StorePath::new_from_base_name(&s), Err(Error::BadBase32)); + } + + #[test] + fn test_long_name() { + let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"; + assert_matches!(StorePath::new_from_base_name(&s), Ok(_)); + } + + #[test] + fn test_too_long_name() { + let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"; + assert_matches!( + StorePath::new_from_base_name(&s), + Err(Error::StorePathNameTooLong) + ); + } + + #[test] + fn test_bad_name() { + let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-foo bar"; + assert_matches!( + StorePath::new_from_base_name(&s), + Err(Error::BadStorePathName) + ); + + let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-kónsole"; + assert_matches!( + StorePath::new_from_base_name(&s), + Err(Error::BadStorePathName) + ); + } + + #[test] + fn test_roundtrip() { + let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-konsole-18.12.3"; + assert_eq!(StorePath::new_from_base_name(&s).unwrap().to_string(), s); + } +} diff --git a/nix-rust/src/store/path_info.rs b/nix-rust/src/store/path_info.rs new file mode 100644 index 00000000000..c2903ed29dd --- /dev/null +++ b/nix-rust/src/store/path_info.rs @@ -0,0 +1,70 @@ +use crate::store::StorePath; +use crate::Error; +use std::collections::BTreeSet; + +#[derive(Clone, Debug)] +pub struct PathInfo { + pub path: StorePath, + pub references: BTreeSet, + pub nar_size: u64, + pub deriver: Option, + + // Additional binary cache info. + pub url: Option, + pub compression: Option, + pub file_size: Option, +} + +impl PathInfo { + pub fn parse_nar_info(nar_info: &str, store_dir: &str) -> Result { + let mut path = None; + let mut references = BTreeSet::new(); + let mut nar_size = None; + let mut deriver = None; + let mut url = None; + let mut compression = None; + let mut file_size = None; + + for line in nar_info.lines() { + let colon = line.find(':').ok_or(Error::BadNarInfo)?; + + let (name, value) = line.split_at(colon); + + if !value.starts_with(": ") { + return Err(Error::BadNarInfo); + } + + let value = &value[2..]; + + if name == "StorePath" { + path = Some(StorePath::new(std::path::Path::new(value), store_dir)?); + } else if name == "NarSize" { + nar_size = Some(u64::from_str_radix(value, 10).map_err(|_| Error::BadNarInfo)?); + } else if name == "References" { + if !value.is_empty() { + for r in value.split(' ') { + references.insert(StorePath::new_from_base_name(r)?); + } + } + } else if name == "Deriver" { + deriver = Some(StorePath::new_from_base_name(value)?); + } else if name == "URL" { + url = Some(value.into()); + } else if name == "Compression" { + compression = Some(value.into()); + } else if name == "FileSize" { + file_size = Some(u64::from_str_radix(value, 10).map_err(|_| Error::BadNarInfo)?); + } + } + + Ok(PathInfo { + path: path.ok_or(Error::BadNarInfo)?, + references, + nar_size: nar_size.ok_or(Error::BadNarInfo)?, + deriver, + url: Some(url.ok_or(Error::BadNarInfo)?), + compression, + file_size, + }) + } +} diff --git a/nix-rust/src/store/store.rs b/nix-rust/src/store/store.rs new file mode 100644 index 00000000000..c33dc4a90f1 --- /dev/null +++ b/nix-rust/src/store/store.rs @@ -0,0 +1,53 @@ +use super::{PathInfo, StorePath}; +use crate::Error; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +pub trait Store: Send + Sync { + fn store_dir(&self) -> &str { + "/nix/store" + } + + fn query_path_info( + &self, + store_path: &StorePath, + ) -> std::pin::Pin> + Send>>; +} + +impl dyn Store { + pub fn parse_store_path(&self, path: &Path) -> Result { + StorePath::new(path, self.store_dir()) + } + + pub async fn compute_path_closure( + &self, + roots: BTreeSet, + ) -> Result, Error> { + let mut done = BTreeSet::new(); + let mut result = BTreeMap::new(); + let mut pending = vec![]; + + for root in roots { + pending.push(self.query_path_info(&root)); + done.insert(root); + } + + while !pending.is_empty() { + let (info, _, remaining) = futures::future::select_all(pending).await; + pending = remaining; + + let info = info?; + + for path in &info.references { + if !done.contains(path) { + pending.push(self.query_path_info(&path)); + done.insert(path.clone()); + } + } + + result.insert(info.path.clone(), info); + } + + Ok(result) + } +} diff --git a/nix-rust/src/util/base32.rs b/nix-rust/src/util/base32.rs new file mode 100644 index 00000000000..efd4a290129 --- /dev/null +++ b/nix-rust/src/util/base32.rs @@ -0,0 +1,160 @@ +use crate::error::Error; +use lazy_static::lazy_static; + +pub fn encoded_len(input_len: usize) -> usize { + if input_len == 0 { + 0 + } else { + (input_len * 8 - 1) / 5 + 1 + } +} + +pub fn decoded_len(input_len: usize) -> usize { + input_len * 5 / 8 +} + +static BASE32_CHARS: &'static [u8; 32] = &b"0123456789abcdfghijklmnpqrsvwxyz"; + +lazy_static! { + static ref BASE32_CHARS_REVERSE: Box<[u8; 256]> = { + let mut xs = [0xffu8; 256]; + for (n, c) in BASE32_CHARS.iter().enumerate() { + xs[*c as usize] = n as u8; + } + Box::new(xs) + }; +} + +pub fn encode(input: &[u8]) -> String { + let mut buf = vec![0; encoded_len(input.len())]; + encode_into(input, &mut buf); + std::str::from_utf8(&buf).unwrap().to_string() +} + +pub fn encode_into(input: &[u8], output: &mut [u8]) { + let len = encoded_len(input.len()); + assert_eq!(len, output.len()); + + let mut nr_bits_left: usize = 0; + let mut bits_left: u16 = 0; + let mut pos = len; + + for b in input { + bits_left |= (*b as u16) << nr_bits_left; + nr_bits_left += 8; + while nr_bits_left > 5 { + output[pos - 1] = BASE32_CHARS[(bits_left & 0x1f) as usize]; + pos -= 1; + bits_left >>= 5; + nr_bits_left -= 5; + } + } + + if nr_bits_left > 0 { + output[pos - 1] = BASE32_CHARS[(bits_left & 0x1f) as usize]; + pos -= 1; + } + + assert_eq!(pos, 0); +} + +pub fn decode(input: &str) -> Result, crate::Error> { + let mut res = Vec::with_capacity(decoded_len(input.len())); + + let mut nr_bits_left: usize = 0; + let mut bits_left: u16 = 0; + + for c in input.chars().rev() { + let b = BASE32_CHARS_REVERSE[c as usize]; + if b == 0xff { + return Err(Error::BadBase32); + } + bits_left |= (b as u16) << nr_bits_left; + nr_bits_left += 5; + if nr_bits_left >= 8 { + res.push((bits_left & 0xff) as u8); + bits_left >>= 8; + nr_bits_left -= 8; + } + } + + if nr_bits_left > 0 && bits_left != 0 { + return Err(Error::BadBase32); + } + + Ok(res) +} + +#[cfg(test)] +mod tests { + use super::*; + use assert_matches::assert_matches; + use hex; + use proptest::proptest; + + #[test] + fn test_encode() { + assert_eq!(encode(&[]), ""); + + assert_eq!( + encode(&hex::decode("0839703786356bca59b0f4a32987eb2e6de43ae8").unwrap()), + "x0xf8v9fxf3jk8zln1cwlsrmhqvp0f88" + ); + + assert_eq!( + encode( + &hex::decode("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") + .unwrap() + ), + "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" + ); + + assert_eq!( + encode( + &hex::decode("ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f") + .unwrap() + ), + "2gs8k559z4rlahfx0y688s49m2vvszylcikrfinm30ly9rak69236nkam5ydvly1ai7xac99vxfc4ii84hawjbk876blyk1jfhkbbyx" + ); + } + + #[test] + fn test_decode() { + assert_eq!(hex::encode(decode("").unwrap()), ""); + + assert_eq!( + hex::encode(decode("x0xf8v9fxf3jk8zln1cwlsrmhqvp0f88").unwrap()), + "0839703786356bca59b0f4a32987eb2e6de43ae8" + ); + + assert_eq!( + hex::encode(decode("1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s").unwrap()), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + + assert_eq!( + hex::encode(decode("2gs8k559z4rlahfx0y688s49m2vvszylcikrfinm30ly9rak69236nkam5ydvly1ai7xac99vxfc4ii84hawjbk876blyk1jfhkbbyx").unwrap()), + "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f" + ); + + assert_matches!( + decode("xoxf8v9fxf3jk8zln1cwlsrmhqvp0f88"), + Err(Error::BadBase32) + ); + assert_matches!( + decode("2b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s"), + Err(Error::BadBase32) + ); + assert_matches!(decode("2"), Err(Error::BadBase32)); + assert_matches!(decode("2gs"), Err(Error::BadBase32)); + assert_matches!(decode("2gs8"), Err(Error::BadBase32)); + } + + proptest! { + + #[test] + fn roundtrip(s: Vec) { + assert_eq!(s, decode(&encode(&s)).unwrap()); + } + } +} diff --git a/nix-rust/src/util/mod.rs b/nix-rust/src/util/mod.rs new file mode 100644 index 00000000000..eaad9d40601 --- /dev/null +++ b/nix-rust/src/util/mod.rs @@ -0,0 +1 @@ +pub mod base32; diff --git a/nix.spec.in b/nix.spec.in deleted file mode 100644 index 401a2dc8a1f..00000000000 --- a/nix.spec.in +++ /dev/null @@ -1,198 +0,0 @@ -%global nixbld_user "nix-builder-" -%global nixbld_group "nixbld" - -Summary: The Nix software deployment system -Name: nix -Version: @PACKAGE_VERSION@ -Release: 2%{?dist} -License: LGPLv2+ -%if 0%{?rhel} && 0%{?rhel} < 7 -Group: Applications/System -%endif -URL: http://nixos.org/ -Source0: %{name}-%{version}.tar.bz2 -%if 0%{?el5} -BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) -%endif -BuildRequires: perl(DBD::SQLite) -BuildRequires: perl(DBI) -BuildRequires: perl(ExtUtils::ParseXS) -Requires: /usr/bin/perl -Requires: curl -Requires: perl-DBD-SQLite -Requires: bzip2 -Requires: gzip -Requires: xz -BuildRequires: bzip2-devel -BuildRequires: sqlite-devel -BuildRequires: libcurl-devel - -# Hack to make that shitty RPM scanning hack shut up. -Provides: perl(Nix::SSH) - -%description -Nix is a purely functional package manager. It allows multiple -versions of a package to be installed side-by-side, ensures that -dependency specifications are complete, supports atomic upgrades and -rollbacks, allows non-root users to install software, and has many -other features. It is the basis of the NixOS Linux distribution, but -it can be used equally well under other Unix systems. - -%package devel -Summary: Development files for %{name} -%if 0%{?rhel} && 0%{?rhel} < 7 -Group: Development/Libraries -%endif -Requires: %{name}%{?_isa} = %{version}-%{release} - -%description devel -The %{name}-devel package contains libraries and header files for -developing applications that use %{name}. - - -%package doc -Summary: Documentation files for %{name} -%if 0%{?rhel} && 0%{?rhel} < 7 -Group: Documentation -%endif -BuildArch: noarch -Requires: %{name} = %{version}-%{release} - -%description doc -The %{name}-doc package contains documentation files for %{name}. - - -%package -n emacs-%{name} -Summary: Nix mode for Emacs -%if 0%{?rhel} && 0%{?rhel} < 7 -Group: Applications/Editors -%endif -BuildArch: noarch -BuildRequires: emacs -Requires: emacs(bin) >= %{_emacs_version} - -%description -n emacs-%{name} -This package provides a major mode for editing Nix expressions. - -%package -n emacs-%{name}-el -Summary: Elisp source files for emacs-%{name} -%if 0%{?rhel} && 0%{?rhel} < 7 -Group: Applications/Editors -%endif -BuildArch: noarch -Requires: emacs-%{name} = %{version}-%{release} - -%description -n emacs-%{name}-el -This package contains the elisp source file for the Nix major mode for -GNU Emacs. You do not need to install this package to run Nix. Install -the emacs-%{name} package to edit Nix expressions with GNU Emacs. - - -%prep -%setup -q -# Install Perl modules to vendor_perl -# configure.ac need to be changed to make this global; however, this will -# also affect NixOS. Use discretion. -%{__sed} -i 's|perl5/site_perl/$perlversion/$perlarchname|perl5/vendor_perl|' \ - configure - - -%build -extraFlags= -# - override docdir so large documentation files are owned by the -# -doc subpackage -# - set localstatedir by hand to the preferred nix value -%configure --localstatedir=/nix/var \ - --docdir=%{_defaultdocdir}/%{name}-doc-%{version} \ - $extraFlags -make %{?_smp_flags} -%{_emacs_bytecompile} misc/emacs/nix-mode.el - - -%install -%if 0%{?el5} -rm -rf $RPM_BUILD_ROOT -%endif -make DESTDIR=$RPM_BUILD_ROOT install - -find $RPM_BUILD_ROOT -name '*.la' -exec rm -f {} ';' - -# make the store -mkdir -p $RPM_BUILD_ROOT/nix/store -chmod 1775 $RPM_BUILD_ROOT/nix/store - -# make per-user directories -for d in profiles gcroots; -do - mkdir -p $RPM_BUILD_ROOT/nix/var/nix/$d/per-user - chmod 1777 $RPM_BUILD_ROOT/nix/var/nix/$d/per-user -done - -# fix permission of nix profile -# (until this is fixed in the relevant Makefile) -chmod -x $RPM_BUILD_ROOT%{_sysconfdir}/profile.d/nix.sh - -# Copy the byte-compiled mode file by hand -cp -p misc/emacs/nix-mode.elc $RPM_BUILD_ROOT%{_emacs_sitelispdir}/ - -# we ship this file in the base package -rm -f $RPM_BUILD_ROOT%{_defaultdocdir}/%{name}-doc-%{version}/README - -# Get rid of Upstart job. -rm -rf $RPM_BUILD_ROOT%{_sysconfdir}/init - - -%clean -rm -rf $RPM_BUILD_ROOT - - -%pre -getent group %{nixbld_group} >/dev/null || groupadd -r %{nixbld_group} -for i in $(seq 10); -do - getent passwd %{nixbld_user}$i >/dev/null || \ - useradd -r -g %{nixbld_group} -G %{nixbld_group} -d /var/empty \ - -s %{_sbindir}/nologin \ - -c "Nix build user $i" %{nixbld_user}$i -done - -%post -chgrp %{nixbld_group} /nix/store -%if ! 0%{?rhel} || 0%{?rhel} >= 7 -# Enable and start Nix worker -systemctl enable nix-daemon.socket nix-daemon.service -systemctl start nix-daemon.socket -%endif - -%files -%{_bindir}/nix* -%{_libdir}/*.so -%{perl_vendorarch}/* -%exclude %dir %{perl_vendorarch}/auto/ -%{_prefix}/libexec/* -%if ! 0%{?rhel} || 0%{?rhel} >= 7 -%{_prefix}/lib/systemd/system/nix-daemon.socket -%{_prefix}/lib/systemd/system/nix-daemon.service -%endif -%{_datadir}/emacs/site-lisp/nix-mode.el -%{_datadir}/nix -%{_mandir}/man1/*.1* -%{_mandir}/man5/*.5* -%{_mandir}/man8/*.8* -%config(noreplace) %{_sysconfdir}/profile.d/nix.sh -/nix - -%files devel -%{_includedir}/nix -%{_prefix}/lib/pkgconfig/*.pc - -%files doc -%docdir %{_defaultdocdir}/%{name}-doc-%{version} -%{_defaultdocdir}/%{name}-doc-%{version} - -%files -n emacs-%{name} -%{_emacs_sitelispdir}/*.elc -#{_emacs_sitestartdir}/*.el - -%files -n emacs-%{name}-el -%{_emacs_sitelispdir}/*.el diff --git a/perl/Makefile b/perl/Makefile new file mode 100644 index 00000000000..7ddb0cf692c --- /dev/null +++ b/perl/Makefile @@ -0,0 +1,15 @@ +makefiles = local.mk + +GLOBAL_CXXFLAGS += -g -Wall + +-include Makefile.config + +OPTIMIZE = 1 + +ifeq ($(OPTIMIZE), 1) + GLOBAL_CXXFLAGS += -O3 +else + GLOBAL_CXXFLAGS += -O0 +endif + +include mk/lib.mk diff --git a/perl/Makefile.config.in b/perl/Makefile.config.in new file mode 100644 index 00000000000..c87d4817e17 --- /dev/null +++ b/perl/Makefile.config.in @@ -0,0 +1,18 @@ +CC = @CC@ +CFLAGS = @CFLAGS@ +CXX = @CXX@ +CXXFLAGS = @CXXFLAGS@ +HAVE_SODIUM = @HAVE_SODIUM@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +SODIUM_LIBS = @SODIUM_LIBS@ +NIX_CFLAGS = @NIX_CFLAGS@ +NIX_LIBS = @NIX_LIBS@ +nixbindir = @nixbindir@ +curl = @curl@ +nixlibexecdir = @nixlibexecdir@ +nixlocalstatedir = @nixlocalstatedir@ +perl = @perl@ +perllibdir = @perllibdir@ +nixstoredir = @nixstoredir@ +nixsysconfdir = @nixsysconfdir@ diff --git a/perl/configure.ac b/perl/configure.ac new file mode 100644 index 00000000000..c3769e1426c --- /dev/null +++ b/perl/configure.ac @@ -0,0 +1,99 @@ +AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX"])) +AC_CONFIG_SRCDIR(MANIFEST) +AC_CONFIG_AUX_DIR(../config) + +CFLAGS= +CXXFLAGS= +AC_PROG_CC +AC_PROG_CXX + +# Use 64-bit file system calls so that we can support files > 2 GiB. +AC_SYS_LARGEFILE + +AC_DEFUN([NEED_PROG], +[ +AC_PATH_PROG($1, $2) +if test -z "$$1"; then + AC_MSG_ERROR([$2 is required]) +fi +]) + +NEED_PROG(perl, perl) +NEED_PROG(curl, curl) +NEED_PROG(bzip2, bzip2) +NEED_PROG(xz, xz) + +# Test that Perl has the open/fork feature (Perl 5.8.0 and beyond). +AC_MSG_CHECKING([whether Perl is recent enough]) +if ! $perl -e 'open(FOO, "-|", "true"); while () { print; }; close FOO or die;'; then + AC_MSG_RESULT(no) + AC_MSG_ERROR([Your Perl version is too old. Nix requires Perl 5.8.0 or newer.]) +fi +AC_MSG_RESULT(yes) + + +# Figure out where to install Perl modules. +AC_MSG_CHECKING([for the Perl installation prefix]) +perlversion=$($perl -e 'use Config; print $Config{version};') +perlarchname=$($perl -e 'use Config; print $Config{archname};') +AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname]) +AC_MSG_RESULT($perllibdir) + +# Look for libsodium, an optional dependency. +PKG_CHECK_MODULES([SODIUM], [libsodium], + [AC_DEFINE([HAVE_SODIUM], [1], [Whether to use libsodium for cryptography.]) + CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS" + have_sodium=1], [have_sodium=]) +AC_SUBST(HAVE_SODIUM, [$have_sodium]) + +# Check for the required Perl dependencies (DBI and DBD::SQLite). +perlFlags="-I$perllibdir" + +AC_ARG_WITH(dbi, AC_HELP_STRING([--with-dbi=PATH], + [prefix of the Perl DBI library]), + perlFlags="$perlFlags -I$withval") + +AC_ARG_WITH(dbd-sqlite, AC_HELP_STRING([--with-dbd-sqlite=PATH], + [prefix of the Perl DBD::SQLite library]), + perlFlags="$perlFlags -I$withval") + +AC_MSG_CHECKING([whether DBD::SQLite works]) +if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then + AC_MSG_RESULT(no) + AC_MSG_FAILURE([The Perl modules DBI and/or DBD::SQLite are missing.]) +fi +AC_MSG_RESULT(yes) + +AC_SUBST(perlFlags) + +PKG_CHECK_MODULES([NIX], [nix-store]) + +NEED_PROG([NIX], [nix]) + +# Get nix configure values +export NIX_REMOTE=daemon +nixbindir=$("$NIX" --experimental-features nix-command eval --raw -f '' nixBinDir) +nixlibexecdir=$("$NIX" --experimental-features nix-command eval --raw -f '' nixLibexecDir) +nixlocalstatedir=$("$NIX" --experimental-features nix-command eval --raw -f '' nixLocalstateDir) +nixsysconfdir=$("$NIX" --experimental-features nix-command eval --raw -f '' nixSysconfDir) +nixstoredir=$("$NIX" --experimental-features nix-command eval --raw -f '' nixStoreDir) +AC_SUBST(nixbindir) +AC_SUBST(nixlibexecdir) +AC_SUBST(nixlocalstatedir) +AC_SUBST(nixsysconfdir) +AC_SUBST(nixstoredir) + +# Expand all variables in config.status. +test "$prefix" = NONE && prefix=$ac_default_prefix +test "$exec_prefix" = NONE && exec_prefix='${prefix}' +for name in $ac_subst_vars; do + declare $name="$(eval echo "${!name}")" + declare $name="$(eval echo "${!name}")" + declare $name="$(eval echo "${!name}")" +done + +rm -f Makefile.config +ln -sfn ../mk mk + +AC_CONFIG_FILES([]) +AC_OUTPUT diff --git a/perl/lib/Nix/Config.pm.in b/perl/lib/Nix/Config.pm.in index 3575d99cb67..bc1749e601e 100644 --- a/perl/lib/Nix/Config.pm.in +++ b/perl/lib/Nix/Config.pm.in @@ -4,48 +4,27 @@ use MIME::Base64; $version = "@PACKAGE_VERSION@"; -$binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@"; -$libexecDir = $ENV{"NIX_LIBEXEC_DIR"} || "@libexecdir@"; -$stateDir = $ENV{"NIX_STATE_DIR"} || "@localstatedir@/nix"; -$logDir = $ENV{"NIX_LOG_DIR"} || "@localstatedir@/log/nix"; -$confDir = $ENV{"NIX_CONF_DIR"} || "@sysconfdir@/nix"; -$storeDir = $ENV{"NIX_STORE_DIR"} || "@storedir@"; +$binDir = $ENV{"NIX_BIN_DIR"} || "@nixbindir@"; +$libexecDir = $ENV{"NIX_LIBEXEC_DIR"} || "@nixlibexecdir@"; +$stateDir = $ENV{"NIX_STATE_DIR"} || "@nixlocalstatedir@/nix"; +$logDir = $ENV{"NIX_LOG_DIR"} || "@nixlocalstatedir@/log/nix"; +$confDir = $ENV{"NIX_CONF_DIR"} || "@nixsysconfdir@/nix"; +$storeDir = $ENV{"NIX_STORE_DIR"} || "@nixstoredir@"; -$bzip2 = "@bzip2@"; -$xz = "@xz@"; -$curl = "@curl@"; - -$useBindings = "@perlbindings@" eq "yes"; +$useBindings = 1; %config = (); -%binaryCachePublicKeys = (); - -$defaultPublicKeys = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="; - sub readConfig { - if (defined $ENV{'_NIX_OPTIONS'}) { - foreach my $s (split '\n', $ENV{'_NIX_OPTIONS'}) { - my ($n, $v) = split '=', $s, 2; - $config{$n} = $v; - } - } else { - my $config = "$confDir/nix.conf"; - return unless -f $config; - - open CONFIG, "<$config" or die "cannot open ‘$config’"; - while () { - /^\s*([\w\-\.]+)\s*=\s*(.*)$/ or next; - $config{$1} = $2; - } - close CONFIG; - } + my $config = "$confDir/nix.conf"; + return unless -f $config; - foreach my $s (split(/ /, $config{"binary-cache-public-keys"} // $defaultPublicKeys)) { - my ($keyName, $publicKey) = split ":", $s; - next unless defined $keyName && defined $publicKey; - $binaryCachePublicKeys{$keyName} = decode_base64($publicKey); + open CONFIG, "<$config" or die "cannot open '$config'"; + while () { + /^\s*([\w\-\.]+)\s*=\s*(.*)$/ or next; + $config{$1} = $2; } + close CONFIG; } return 1; diff --git a/perl/lib/Nix/CopyClosure.pm b/perl/lib/Nix/CopyClosure.pm index affb3ea524a..902ee1a1bc9 100644 --- a/perl/lib/Nix/CopyClosure.pm +++ b/perl/lib/Nix/CopyClosure.pm @@ -35,14 +35,14 @@ sub copyToOpen { my $missingSize = 0; $missingSize += (queryPathInfo($_, 1))[3] foreach @missing; - printf STDERR "copying %d missing paths (%.2f MiB) to ‘$sshHost’...\n", + printf STDERR "copying %d missing paths (%.2f MiB) to '$sshHost'...\n", scalar(@missing), $missingSize / (1024**2); return if $dryRun; # Send the "import paths" command. syswrite($to, pack("L store() static std::shared_ptr _store; if (!_store) { try { - logger = makeDefaultLogger(); - settings.processEnvironment(); - settings.loadConfFile(); - settings.update(); + loadConfFile(); settings.lockCPU = false; _store = openStore(); } catch (Error & e) { @@ -60,7 +59,7 @@ void setVerbosity(int level) int isValidPath(char * path) CODE: try { - RETVAL = store()->isValidPath(path); + RETVAL = store()->isValidPath(store()->parseStorePath(path)); } catch (Error & e) { croak("%s", e.what()); } @@ -71,9 +70,8 @@ int isValidPath(char * path) SV * queryReferences(char * path) PPCODE: try { - PathSet paths = store()->queryPathInfo(path)->references; - for (PathSet::iterator i = paths.begin(); i != paths.end(); ++i) - XPUSHs(sv_2mortal(newSVpv(i->c_str(), 0))); + for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->references) + XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } @@ -82,8 +80,7 @@ SV * queryReferences(char * path) SV * queryPathHash(char * path) PPCODE: try { - auto hash = store()->queryPathInfo(path)->narHash; - string s = "sha256:" + printHash32(hash); + auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -93,9 +90,9 @@ SV * queryPathHash(char * path) SV * queryDeriver(char * path) PPCODE: try { - auto deriver = store()->queryPathInfo(path)->deriver; - if (deriver == "") XSRETURN_UNDEF; - XPUSHs(sv_2mortal(newSVpv(deriver.c_str(), 0))); + auto info = store()->queryPathInfo(store()->parseStorePath(path)); + if (!info->deriver) XSRETURN_UNDEF; + XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } @@ -104,18 +101,18 @@ SV * queryDeriver(char * path) SV * queryPathInfo(char * path, int base32) PPCODE: try { - auto info = store()->queryPathInfo(path); - if (info->deriver == "") + auto info = store()->queryPathInfo(store()->parseStorePath(path)); + if (!info->deriver) XPUSHs(&PL_sv_undef); else - XPUSHs(sv_2mortal(newSVpv(info->deriver.c_str(), 0))); - string s = "sha256:" + (base32 ? printHash32(info->narHash) : printHash(info->narHash)); + XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); + auto s = info->narHash.to_string(base32 ? Base32 : Base16, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); AV * arr = newAV(); - for (PathSet::iterator i = info->references.begin(); i != info->references.end(); ++i) - av_push(arr, newSVpv(i->c_str(), 0)); + for (auto & i : info->references) + av_push(arr, newSVpv(store()->printStorePath(i).c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) arr))); } catch (Error & e) { croak("%s", e.what()); @@ -125,8 +122,8 @@ SV * queryPathInfo(char * path, int base32) SV * queryPathFromHashPart(char * hashPart) PPCODE: try { - Path path = store()->queryPathFromHashPart(hashPart); - XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0))); + auto path = store()->queryPathFromHashPart(hashPart); + XPUSHs(sv_2mortal(newSVpv(path ? store()->printStorePath(*path).c_str() : "", 0))); } catch (Error & e) { croak("%s", e.what()); } @@ -135,11 +132,11 @@ SV * queryPathFromHashPart(char * hashPart) SV * computeFSClosure(int flipDirection, int includeOutputs, ...) PPCODE: try { - PathSet paths; + StorePathSet paths; for (int n = 2; n < items; ++n) - store()->computeFSClosure(SvPV_nolen(ST(n)), paths, flipDirection, includeOutputs); - for (PathSet::iterator i = paths.begin(); i != paths.end(); ++i) - XPUSHs(sv_2mortal(newSVpv(i->c_str(), 0))); + store()->computeFSClosure(store()->parseStorePath(SvPV_nolen(ST(n))), paths, flipDirection, includeOutputs); + for (auto & i : paths) + XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } @@ -148,11 +145,11 @@ SV * computeFSClosure(int flipDirection, int includeOutputs, ...) SV * topoSortPaths(...) PPCODE: try { - PathSet paths; - for (int n = 0; n < items; ++n) paths.insert(SvPV_nolen(ST(n))); - Paths sorted = store()->topoSortPaths(paths); - for (Paths::iterator i = sorted.begin(); i != sorted.end(); ++i) - XPUSHs(sv_2mortal(newSVpv(i->c_str(), 0))); + StorePathSet paths; + for (int n = 0; n < items; ++n) paths.insert(store()->parseStorePath(SvPV_nolen(ST(n)))); + auto sorted = store()->topoSortPaths(paths); + for (auto & i : sorted) + XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } @@ -161,7 +158,7 @@ SV * topoSortPaths(...) SV * followLinksToStorePath(char * path) CODE: try { - RETVAL = newSVpv(store()->followLinksToStorePath(path).c_str(), 0); + RETVAL = newSVpv(store()->printStorePath(store()->followLinksToStorePath(path)).c_str(), 0); } catch (Error & e) { croak("%s", e.what()); } @@ -172,8 +169,8 @@ SV * followLinksToStorePath(char * path) void exportPaths(int fd, ...) PPCODE: try { - Paths paths; - for (int n = 1; n < items; ++n) paths.push_back(SvPV_nolen(ST(n))); + StorePathSet paths; + for (int n = 1; n < items; ++n) paths.insert(store()->parseStorePath(SvPV_nolen(ST(n)))); FdSink sink(fd); store()->exportPaths(paths, sink); } catch (Error & e) { @@ -185,7 +182,7 @@ void importPaths(int fd, int dontCheckSigs) PPCODE: try { FdSource source(fd); - store()->importPaths(source, 0, dontCheckSigs); + store()->importPaths(source, nullptr, dontCheckSigs ? NoCheckSigs : CheckSigs); } catch (Error & e) { croak("%s", e.what()); } @@ -195,7 +192,7 @@ SV * hashPath(char * algo, int base32, char * path) PPCODE: try { Hash h = hashPath(parseHashType(algo), path).first; - string s = base32 ? printHash32(h) : printHash(h); + auto s = h.to_string(base32 ? Base32 : Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -206,7 +203,7 @@ SV * hashFile(char * algo, int base32, char * path) PPCODE: try { Hash h = hashFile(parseHashType(algo), path); - string s = base32 ? printHash32(h) : printHash(h); + auto s = h.to_string(base32 ? Base32 : Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -217,7 +214,7 @@ SV * hashString(char * algo, int base32, char * s) PPCODE: try { Hash h = hashString(parseHashType(algo), s); - string s = base32 ? printHash32(h) : printHash(h); + auto s = h.to_string(base32 ? Base32 : Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -227,8 +224,8 @@ SV * hashString(char * algo, int base32, char * s) SV * convertHash(char * algo, char * s, int toBase32) PPCODE: try { - Hash h = parseHash16or32(parseHashType(algo), s); - string s = toBase32 ? printHash32(h) : printHash(h); + Hash h(s, parseHashType(algo)); + string s = h.to_string(toBase32 ? Base32 : Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -277,8 +274,9 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg) SV * addToStore(char * srcPath, int recursive, char * algo) PPCODE: try { - Path path = store()->addToStore(baseNameOf(srcPath), srcPath, recursive, parseHashType(algo)); - XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0))); + auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashType(algo)); + XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } @@ -287,10 +285,10 @@ SV * addToStore(char * srcPath, int recursive, char * algo) SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) PPCODE: try { - HashType ht = parseHashType(algo); - Hash h = parseHash16or32(ht, hash); - Path path = store()->makeFixedOutputPath(recursive, h, name); - XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0))); + Hash h(hash, parseHashType(algo)); + auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + auto path = store()->makeFixedOutputPath(method, h, name); + XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } @@ -301,35 +299,35 @@ SV * derivationFromPath(char * drvPath) HV *hash; CODE: try { - Derivation drv = store()->derivationFromPath(drvPath); + Derivation drv = store()->derivationFromPath(store()->parseStorePath(drvPath)); hash = newHV(); HV * outputs = newHV(); - for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i) - hv_store(outputs, i->first.c_str(), i->first.size(), newSVpv(i->second.path.c_str(), 0), 0); + for (auto & i : drv.outputs) + hv_store(outputs, i.first.c_str(), i.first.size(), newSVpv(store()->printStorePath(i.second.path).c_str(), 0), 0); hv_stores(hash, "outputs", newRV((SV *) outputs)); AV * inputDrvs = newAV(); - for (DerivationInputs::iterator i = drv.inputDrvs.begin(); i != drv.inputDrvs.end(); ++i) - av_push(inputDrvs, newSVpv(i->first.c_str(), 0)); // !!! ignores i->second + for (auto & i : drv.inputDrvs) + av_push(inputDrvs, newSVpv(store()->printStorePath(i.first).c_str(), 0)); // !!! ignores i->second hv_stores(hash, "inputDrvs", newRV((SV *) inputDrvs)); AV * inputSrcs = newAV(); - for (PathSet::iterator i = drv.inputSrcs.begin(); i != drv.inputSrcs.end(); ++i) - av_push(inputSrcs, newSVpv(i->c_str(), 0)); + for (auto & i : drv.inputSrcs) + av_push(inputSrcs, newSVpv(store()->printStorePath(i).c_str(), 0)); hv_stores(hash, "inputSrcs", newRV((SV *) inputSrcs)); hv_stores(hash, "platform", newSVpv(drv.platform.c_str(), 0)); hv_stores(hash, "builder", newSVpv(drv.builder.c_str(), 0)); AV * args = newAV(); - for (Strings::iterator i = drv.args.begin(); i != drv.args.end(); ++i) - av_push(args, newSVpv(i->c_str(), 0)); + for (auto & i : drv.args) + av_push(args, newSVpv(i.c_str(), 0)); hv_stores(hash, "args", newRV((SV *) args)); HV * env = newHV(); - for (StringPairs::iterator i = drv.env.begin(); i != drv.env.end(); ++i) - hv_store(env, i->first.c_str(), i->first.size(), newSVpv(i->second.c_str(), 0), 0); + for (auto & i : drv.env) + hv_store(env, i.first.c_str(), i.first.size(), newSVpv(i.second.c_str(), 0), 0); hv_stores(hash, "env", newRV((SV *) env)); RETVAL = newRV_noinc((SV *)hash); @@ -343,7 +341,7 @@ SV * derivationFromPath(char * drvPath) void addTempRoot(char * storePath) PPCODE: try { - store()->addTempRoot(storePath); + store()->addTempRoot(store()->parseStorePath(storePath)); } catch (Error & e) { croak("%s", e.what()); } diff --git a/perl/lib/Nix/Utils.pm b/perl/lib/Nix/Utils.pm index 392c45f2fff..44955a70698 100644 --- a/perl/lib/Nix/Utils.pm +++ b/perl/lib/Nix/Utils.pm @@ -10,7 +10,7 @@ $urlRE = "(?: [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\! sub checkURL { my ($url) = @_; - die "invalid URL ‘$url’\n" unless $url =~ /^ $urlRE $ /x; + die "invalid URL '$url'\n" unless $url =~ /^ $urlRE $ /x; } sub uniq { @@ -26,7 +26,7 @@ sub uniq { sub writeFile { my ($fn, $s) = @_; - open TMP, ">$fn" or die "cannot create file ‘$fn’: $!"; + open TMP, ">$fn" or die "cannot create file '$fn': $!"; print TMP "$s" or die; close TMP or die; } @@ -34,7 +34,7 @@ sub writeFile { sub readFile { local $/ = undef; my ($fn) = @_; - open TMP, "<$fn" or die "cannot open file ‘$fn’: $!"; + open TMP, "<$fn" or die "cannot open file '$fn': $!"; my $s = ; close TMP or die; return $s; diff --git a/perl/local.mk b/perl/local.mk index 5b43c4b717f..b13d4c0d639 100644 --- a/perl/local.mk +++ b/perl/local.mk @@ -1,48 +1,43 @@ nix_perl_sources := \ - $(d)/lib/Nix/Store.pm \ - $(d)/lib/Nix/Manifest.pm \ - $(d)/lib/Nix/SSH.pm \ - $(d)/lib/Nix/CopyClosure.pm \ - $(d)/lib/Nix/Config.pm.in \ - $(d)/lib/Nix/Utils.pm + lib/Nix/Store.pm \ + lib/Nix/Manifest.pm \ + lib/Nix/SSH.pm \ + lib/Nix/CopyClosure.pm \ + lib/Nix/Config.pm.in \ + lib/Nix/Utils.pm nix_perl_modules := $(nix_perl_sources:.in=) $(foreach x, $(nix_perl_modules), $(eval $(call install-data-in, $(x), $(perllibdir)/Nix))) -ifeq ($(perlbindings), yes) - - $(d)/lib/Nix/Store.cc: $(d)/lib/Nix/Store.xs +lib/Nix/Store.cc: lib/Nix/Store.xs $(trace-gen) xsubpp $^ -output $@ - libraries += Store - - Store_DIR := $(d)/lib/Nix - - Store_SOURCES := $(Store_DIR)/Store.cc +libraries += Store - Store_CXXFLAGS = \ - -I$(shell $(perl) -e 'use Config; print $$Config{archlibexp};')/CORE \ - -D_FILE_OFFSET_BITS=64 \ - -Wno-unknown-warning-option -Wno-unused-variable -Wno-literal-suffix \ - -Wno-reserved-user-defined-literal -Wno-duplicate-decl-specifier -Wno-pointer-bool-conversion +Store_DIR := lib/Nix - Store_LIBS = libstore libutil +Store_SOURCES := $(Store_DIR)/Store.cc - Store_LDFLAGS := $(SODIUM_LIBS) +Store_CXXFLAGS = \ + $(NIX_CFLAGS) \ + -I$(shell perl -e 'use Config; print $$Config{archlibexp};')/CORE \ + -D_FILE_OFFSET_BITS=64 \ + -Wno-unknown-warning-option -Wno-unused-variable -Wno-literal-suffix \ + -Wno-reserved-user-defined-literal -Wno-duplicate-decl-specifier -Wno-pointer-bool-conversion - ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) - archlib = $(shell perl -E 'use Config; print $$Config{archlib};') - libperl = $(shell perl -E 'use Config; print $$Config{libperl};') - Store_LDFLAGS += $(shell find ${archlib} -name ${libperl}) - endif +Store_LDFLAGS := $(SODIUM_LIBS) $(NIX_LIBS) - Store_ALLOW_UNDEFINED = 1 +ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) + archlib = $(shell perl -E 'use Config; print $$Config{archlib};') + libperl = $(shell perl -E 'use Config; print $$Config{libperl};') + Store_LDFLAGS += $(shell find ${archlib} -name ${libperl}) +endif - Store_FORCE_INSTALL = 1 +Store_ALLOW_UNDEFINED = 1 - Store_INSTALL_DIR = $(perllibdir)/auto/Nix/Store +Store_FORCE_INSTALL = 1 -endif +Store_INSTALL_DIR = $(perllibdir)/auto/Nix/Store -clean-files += $(d)/lib/Nix/Config.pm $(d)/lib/Nix/Store.cc +clean-files += lib/Nix/Config.pm lib/Nix/Store.cc Makefile.config diff --git a/precompiled-headers.h b/precompiled-headers.h new file mode 100644 index 00000000000..079aa496e27 --- /dev/null +++ b/precompiled-headers.h @@ -0,0 +1,58 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include diff --git a/release-common.nix b/release-common.nix new file mode 100644 index 00000000000..7e7de005d2f --- /dev/null +++ b/release-common.nix @@ -0,0 +1,83 @@ +{ pkgs }: + +with pkgs; + +rec { + # Use "busybox-sandbox-shell" if present, + # if not (legacy) fallback and hope it's sufficient. + sh = pkgs.busybox-sandbox-shell or (busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y + + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y + + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y + ''; + }); + + configureFlags = + lib.optionals stdenv.isLinux [ + "--with-sandbox-shell=${sh}/bin/busybox" + ]; + + buildDeps = + [ bison + flex + libxml2 + libxslt + docbook5 + docbook_xsl_ns + autoconf-archive + autoreconfHook + + curl + bzip2 xz brotli zlib editline + openssl pkgconfig sqlite + libarchive + boost + nlohmann_json + rustc cargo + + # Tests + git + mercurial + gmock + ] + ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal] + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) + ((aws-sdk-cpp.override { + apis = ["s3" "transfer"]; + customMemoryManagement = false; + }).overrideDerivation (args: { + /* + patches = args.patches or [] ++ [ (fetchpatch { + url = https://github.com/edolstra/aws-sdk-cpp/commit/3e07e1f1aae41b4c8b340735ff9e8c735f0c063f.patch; + sha256 = "1pij0v449p166f9l29x7ppzk8j7g9k9mp15ilh5qxp29c7fnvxy2"; + }) ]; + */ + })); + + propagatedDeps = + [ (boehmgc.override { enableLargeConfig = true; }) + ]; + + perlDeps = + [ perl + perlPackages.DBDSQLite + ]; +} diff --git a/release.nix b/release.nix index d825dd58376..2a320e1c3bc 100644 --- a/release.nix +++ b/release.nix @@ -1,161 +1,260 @@ -{ nix ? { outPath = ./.; revCount = 1234; shortRev = "abcdef"; } -, nixpkgs ? { outPath = ; revCount = 1234; shortRev = "abcdef"; } +{ nix ? builtins.fetchGit ./. +, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz , officialRelease ? false +, systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] }: let - pkgs = import {}; + pkgs = import nixpkgs { system = builtins.currentSystem or "x86_64-linux"; }; - systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" /* "x86_64-freebsd" "i686-freebsd" */ ]; + version = + builtins.readFile ./.version + + (if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}"); + # Create a "vendor" directory that contains the crates listed in + # Cargo.lock. This allows Nix to be built without network access. + vendoredCrates' = + let + lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock); - jobs = rec { + files = map (pkg: import { + url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download"; + sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)"; + }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package); + in pkgs.runCommand "cargo-vendor-dir" {} + '' + mkdir -p $out/vendor - tarball = - with pkgs; + cat > $out/vendor/config < "$dir/.cargo-checksum.json" - postUnpack = '' - # Clean up when building from a working tree. - if [[ -d $sourceRoot/.git ]]; then - git -C $sourceRoot clean -fd + # Clean up some cruft from the winapi crates. FIXME: find + # a way to remove winapi* from our dependencies. + if [[ $dir =~ /winapi ]]; then + find $dir -name "*.a" -print0 | xargs -0 rm -f -- fi - ''; - preConfigure = '' - # TeX needs a writable font cache. - export VARTEXFONTS=$TMPDIR/texfonts + mv "$dir" $out/vendor/ + + rm -rf $out/vendor/tmp + '') files)} + ''; + + jobs = rec { + + vendoredCrates = + with pkgs; + runCommand "vendored-crates" {} + '' + mkdir -p $out/nix-support + name=nix-vendored-crates-${version} + fn=$out/$name.tar.xz + tar cvfJ $fn -C ${vendoredCrates'} vendor \ + --owner=0 --group=0 --mode=u+rw,uga+r \ + --transform "s,vendor,$name," + echo "file crates-tarball $fn" >> $out/nix-support/hydra-build-products ''; - distPhase = + build = pkgs.lib.genAttrs systems (system: + + let pkgs = import nixpkgs { inherit system; }; in + + with pkgs; + + with import ./release-common.nix { inherit pkgs; }; + + stdenv.mkDerivation { + name = "nix-${version}"; + + src = nix; + + outputs = [ "out" "dev" "doc" ]; + + buildInputs = buildDeps; + + propagatedBuildInputs = propagatedDeps; + + preConfigure = '' - runHook preDist - make dist - mkdir -p $out/tarballs - cp *.tar.* $out/tarballs + # Copy libboost_context so we don't get all of Boost in our closure. + # https://github.com/NixOS/nixpkgs/issues/45462 + mkdir -p $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib + rm -f $out/lib/*.a + ${lib.optionalString stdenv.isLinux '' + chmod u+w $out/lib/*.so.* + patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* + ''} + + ln -sfn ${vendoredCrates'}/vendor/ nix-rust/vendor + + (cd perl; autoreconf --install --force --verbose) ''; - preDist = '' - make install docdir=$out/share/doc/nix makefiles=doc/manual/local.mk - echo "doc manual $out/share/doc/nix/manual" >> $out/nix-support/hydra-build-products + configureFlags = configureFlags ++ + [ "--sysconfdir=/etc" ]; + + enableParallelBuilding = true; + + makeFlags = "profiledir=$(out)/etc/profile.d"; + + installFlags = "sysconfdir=$(out)/etc"; + + postInstall = '' + mkdir -p $doc/nix-support + echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products ''; - }; + doCheck = true; - build = pkgs.lib.genAttrs systems (system: + doInstallCheck = true; + installCheckFlags = "sysconfdir=$(out)/etc"; + + separateDebugInfo = true; + }); + + + perlBindings = pkgs.lib.genAttrs systems (system: - with import { inherit system; }; + let pkgs = import nixpkgs { inherit system; }; in with pkgs; releaseTools.nixBuild { - name = "nix"; - src = tarball; + name = "nix-perl-${version}"; + + src = nix; buildInputs = - [ curl perl bzip2 xz openssl pkgconfig sqlite boehmgc ] - ++ lib.optional stdenv.isLinux libsodium - ++ lib.optional stdenv.isLinux - (aws-sdk-cpp.override { - apis = ["s3"]; - customMemoryManagement = false; - }); + [ autoconf-archive + autoreconfHook + jobs.build.${system} + curl + bzip2 + xz + pkgconfig + pkgs.perl + boost + ] + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium; configureFlags = '' - --disable-init-state - --with-dbi=${perlPackages.DBI}/${perl.libPrefix} - --with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix} - --enable-gc - --sysconfdir=/etc + --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix} + --with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix} ''; enableParallelBuilding = true; - makeFlags = "profiledir=$(out)/etc/profile.d"; - - preBuild = "unset NIX_INDENT_MAKE"; - - installFlags = "sysconfdir=$(out)/etc"; - - doInstallCheck = true; - installCheckFlags = "sysconfdir=$(out)/etc"; + postUnpack = "sourceRoot=$sourceRoot/perl"; }); binaryTarball = pkgs.lib.genAttrs systems (system: - # FIXME: temporarily use a different branch for the Darwin build. - with import { inherit system; }; + with import nixpkgs { inherit system; }; let toplevel = builtins.getAttr system jobs.build; - version = toplevel.src.version; + installerClosureInfo = closureInfo { rootPaths = [ toplevel cacert ]; }; in runCommand "nix-binary-tarball-${version}" - { exportReferencesGraph = [ "closure1" toplevel "closure2" cacert ]; - buildInputs = [ perl ]; + { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; } '' - storePaths=$(perl ${pathsFromGraph} ./closure1 ./closure2) - printRegistration=1 perl ${pathsFromGraph} ./closure1 ./closure2 > $TMPDIR/reginfo + cp ${installerClosureInfo}/registration $TMPDIR/reginfo + cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ --subst-var-by nix ${toplevel} \ --subst-var-by cacert ${cacert} + substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ + --subst-var-by nix ${toplevel} \ + --subst-var-by cacert ${cacert} + substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ + --subst-var-by nix ${toplevel} \ + --subst-var-by cacert ${cacert} + substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \ + --subst-var-by nix ${toplevel} \ + --subst-var-by cacert ${cacert} + + if type -p shellcheck; then + # SC1090: Don't worry about not being able to find + # $nix/etc/profile.d/nix.sh + shellcheck --exclude SC1090 $TMPDIR/install + shellcheck $TMPDIR/create-darwin-volume.sh + shellcheck $TMPDIR/install-darwin-multi-user.sh + shellcheck $TMPDIR/install-systemd-multi-user.sh + + # SC1091: Don't panic about not being able to source + # /etc/profile + # SC2002: Ignore "useless cat" "error", when loading + # .reginfo, as the cat is a much cleaner + # implementation, even though it is "useless" + # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving + # root's home directory + shellcheck --external-sources \ + --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user + fi + chmod +x $TMPDIR/install + chmod +x $TMPDIR/create-darwin-volume.sh + chmod +x $TMPDIR/install-darwin-multi-user.sh + chmod +x $TMPDIR/install-systemd-multi-user.sh + chmod +x $TMPDIR/install-multi-user dir=nix-${version}-${system} - fn=$out/$dir.tar.bz2 + fn=$out/$dir.tar.xz mkdir -p $out/nix-support echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products - tar cvfj $fn \ + tar cvfJ $fn \ --owner=0 --group=0 --mode=u+rw,uga+r \ --absolute-names \ --hard-dereference \ --transform "s,$TMPDIR/install,$dir/install," \ + --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \ --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ --transform "s,$NIX_STORE,$dir/store,S" \ - $TMPDIR/install $TMPDIR/reginfo $storePaths + $TMPDIR/install \ + $TMPDIR/create-darwin-volume.sh \ + $TMPDIR/install-darwin-multi-user.sh \ + $TMPDIR/install-systemd-multi-user.sh \ + $TMPDIR/install-multi-user \ + $TMPDIR/reginfo \ + $(cat ${installerClosureInfo}/store-paths) ''); coverage = - with import { system = "x86_64-linux"; }; + with pkgs; + + with import ./release-common.nix { inherit pkgs; }; releaseTools.coverageAnalysis { - name = "nix-build"; - src = tarball; + name = "nix-coverage-${version}"; - buildInputs = - [ curl perl bzip2 openssl pkgconfig sqlite xz libsodium - # These are for "make check" only: - graphviz libxml2 libxslt - ]; + src = nix; - configureFlags = '' - --disable-init-state - --with-dbi=${perlPackages.DBI}/${perl.libPrefix} - --with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix} - ''; + preConfigure = + '' + ln -sfn ${vendoredCrates'}/vendor/ nix-rust/vendor + ''; + + enableParallelBuilding = true; + + buildInputs = buildDeps ++ propagatedDeps; dontInstall = false; @@ -167,41 +266,38 @@ let # syntax-check generated dot files, it still requires some # fonts. So provide those. FONTCONFIG_FILE = texFunctions.fontsConf; - }; - - - rpm_fedora21i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora21i386) [ "libsodium-devel" ]; - rpm_fedora21x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora21x86_64) [ "libsodium-devel" ]; - - - deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium13" ]; - deb_debian8x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian8x86_64) [ "libsodium-dev" ] [ "libsodium13" ]; - deb_ubuntu1410i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1410i386) [] []; - deb_ubuntu1410x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1410x86_64) [] []; - deb_ubuntu1504i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1504i386) [ "libsodium-dev" ] [ "libsodium13" ]; - deb_ubuntu1504x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1504x86_64) [ "libsodium-dev" ] [ "libsodium13" ]; - deb_ubuntu1510i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1510i386) [ "libsodium-dev" ] [ "libsodium13"]; - deb_ubuntu1510x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1510x86_64) [ "libsodium-dev" ] [ "libsodium13" ]; - deb_ubuntu1604i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1604i386) [ "libsodium-dev" ] [ "libsodium18" ]; - deb_ubuntu1604x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1604x86_64) [ "libsodium-dev" ] [ "libsodium18" ]; + # To test building without precompiled headers. + makeFlagsArray = [ "PRECOMPILE_HEADERS=0" ]; + }; # System tests. tests.remoteBuilds = (import ./tests/remote-builds.nix rec { + inherit nixpkgs; nix = build.x86_64-linux; system = "x86_64-linux"; }); tests.nix-copy-closure = (import ./tests/nix-copy-closure.nix rec { + inherit nixpkgs; nix = build.x86_64-linux; system = "x86_64-linux"; }); + tests.setuid = pkgs.lib.genAttrs + ["i686-linux" "x86_64-linux"] + (system: + import ./tests/setuid.nix rec { + inherit nixpkgs; + nix = build.${system}; inherit system; + }); + tests.binaryTarball = - with import { system = "x86_64-linux"; }; + with import nixpkgs { system = "x86_64-linux"; }; vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test" { diskImage = vmTools.diskImages.ubuntu1204x86_64; } '' + set -x useradd -m alice su - alice -c 'tar xf ${binaryTarball.x86_64-linux}/*.tar.*' mkdir /dest-nix @@ -210,13 +306,25 @@ let su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install' su - alice -c 'nix-store --verify' su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}' + + # Check whether 'nix upgrade-nix' works. + cat > /tmp/paths.nix < { + import (nixpkgs + "/pkgs/top-level/make-tarball.nix") { inherit nixpkgs; inherit pkgs; nix = build.x86_64-linux; @@ -227,93 +335,32 @@ let pkgs.runCommand "eval-nixos" { buildInputs = [ build.x86_64-linux ]; } '' export NIX_STATE_DIR=$TMPDIR - nix-store --init - nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run + nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run \ + --arg nixpkgs '{ outPath = ${nixpkgs}; revCount = 123; shortRev = "abcdefgh"; }' touch $out ''; + */ - # Aggregate job containing the release-critical jobs. - release = pkgs.releaseTools.aggregate { - name = "nix-${tarball.version}"; - meta.description = "Release-critical builds"; - constituents = - [ tarball - #build.i686-freebsd - build.i686-linux - build.x86_64-darwin - #build.x86_64-freebsd - build.x86_64-linux - #binaryTarball.i686-freebsd - binaryTarball.i686-linux - binaryTarball.x86_64-darwin - #binaryTarball.x86_64-freebsd - binaryTarball.x86_64-linux - deb_debian8i386 - deb_debian8x86_64 - deb_ubuntu1504i386 - deb_ubuntu1504x86_64 - rpm_fedora21i386 - rpm_fedora21x86_64 - tests.remoteBuilds - tests.nix-copy-closure - tests.binaryTarball - tests.evalNixpkgs - tests.evalNixOS - ]; - }; + installerScript = + pkgs.runCommand "installer-script" + { buildInputs = [ build.${builtins.currentSystem or "x86_64-linux"} ]; } + '' + mkdir -p $out/nix-support - }; + substitute ${./scripts/install.in} $out/install \ + ${pkgs.lib.concatMapStrings + (system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ") + systems + } \ + --replace '@nixVersion@' ${version} + echo "file installer $out/install" >> $out/nix-support/hydra-build-products + ''; - makeRPM_i686 = makeRPM "i686-linux"; - makeRPM_x86_64 = makeRPM "x86_64-linux"; - - makeRPM = - system: diskImageFun: extraPackages: - - with import { inherit system; }; - - releaseTools.rpmBuild rec { - name = "nix-rpm"; - src = jobs.tarball; - diskImage = (diskImageFun vmTools.diskImageFuns) - { extraPackages = - [ "perl-DBD-SQLite" "perl-devel" "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "libcurl-devel" "openssl-devel" "xz-devel" ] - ++ extraPackages; }; - memSize = 1024; - meta.schedulingPriority = 50; - postRPMInstall = "cd /tmp/rpmout/BUILD/nix-* && make installcheck"; - }; - - - makeDeb_i686 = makeDeb "i686-linux"; - makeDeb_x86_64 = makeDeb "x86_64-linux"; - - makeDeb = - system: diskImageFun: extraPackages: extraDebPackages: - - with import { inherit system; }; - - releaseTools.debBuild { - name = "nix-deb"; - src = jobs.tarball; - diskImage = (diskImageFun vmTools.diskImageFuns) - { extraPackages = - [ "libdbd-sqlite3-perl" "libsqlite3-dev" "libbz2-dev" "libwww-curl-perl" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" ] - ++ extraPackages; }; - memSize = 1024; - meta.schedulingPriority = 50; - postInstall = "make installcheck"; - configureFlags = "--sysconfdir=/etc"; - debRequires = - [ "curl" "libdbd-sqlite3-perl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libwww-curl-perl" "libssl1.0.0" "liblzma5" ] - ++ extraDebPackages; - debMaintainer = "Eelco Dolstra "; - doInstallCheck = true; - }; + }; in jobs diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in deleted file mode 100755 index b5fc629eb49..00000000000 --- a/scripts/build-remote.pl.in +++ /dev/null @@ -1,275 +0,0 @@ -#! @perl@ -w @perlFlags@ - -use utf8; -use Fcntl qw(:DEFAULT :flock); -use English '-no_match_vars'; -use IO::Handle; -use Nix::Config; -use Nix::SSH; -use Nix::CopyClosure; -use Nix::Store; -use Encode; -no warnings('once'); - -STDERR->autoflush(1); -binmode STDERR, ":encoding(utf8)"; - -my $debug = defined $ENV{NIX_DEBUG_HOOK}; - - -# General operation: -# -# Try to find a free machine of type $neededSystem. We do this as -# follows: -# - We acquire an exclusive lock on $currentLoad/main-lock. -# - For each machine $machine of type $neededSystem and for each $slot -# less than the maximum load for that machine, we try to get an -# exclusive lock on $currentLoad/$machine-$slot (without blocking). -# If we get such a lock, we send "accept" to the caller. Otherwise, -# we send "postpone" and exit. -# - We release the exclusive lock on $currentLoad/main-lock. -# - We perform the build on $neededSystem. -# - We release the exclusive lock on $currentLoad/$machine-$slot. -# -# The nice thing about this scheme is that if we die prematurely, the -# locks are released automatically. - - -# Make sure that we don't get any SSH passphrase or host key popups - -# if there is any problem it should fail, not do something -# interactive. -$ENV{"DISPLAY"} = ""; -$ENV{"SSH_ASKPASS"} = ""; - - -sub sendReply { - my $reply = shift; - print STDERR "# $reply\n"; -} - -sub all { $_ || return 0 for @_; 1 } - - -# Initialisation. -my $loadIncreased = 0; - -my ($localSystem, $maxSilentTime, $buildTimeout) = @ARGV; - -my $currentLoad = $ENV{"NIX_CURRENT_LOAD"} // "/run/nix/current-load"; -my $conf = $ENV{"NIX_REMOTE_SYSTEMS"} // "@sysconfdir@/nix/machines"; - - -sub openSlotLock { - my ($machine, $slot) = @_; - my $slotLockFn = "$currentLoad/" . (join '+', @{$machine->{systemTypes}}) . "-" . $machine->{hostName} . "-$slot"; - my $slotLock = new IO::Handle; - sysopen $slotLock, "$slotLockFn", O_RDWR|O_CREAT, 0600 or die; - return $slotLock; -} - - -# Read the list of machines. -my @machines; -if (defined $conf && -e $conf) { - open CONF, "<$conf" or die; - while () { - chomp; - s/\#.*$//g; - next if /^\s*$/; - my @tokens = split /\s/, $_; - my @supportedFeatures = split(/,/, $tokens[5] || ""); - my @mandatoryFeatures = split(/,/, $tokens[6] || ""); - push @machines, - { hostName => $tokens[0] - , systemTypes => [ split(/,/, $tokens[1]) ] - , sshKey => $tokens[2] - , maxJobs => int($tokens[3]) - , speedFactor => 1.0 * (defined $tokens[4] ? int($tokens[4]) : 1) - , supportedFeatures => [ @supportedFeatures, @mandatoryFeatures ] - , mandatoryFeatures => [ @mandatoryFeatures ] - , enabled => 1 - }; - } - close CONF; -} - - - -# Wait for the calling process to ask us whether we can build some derivation. -my ($drvPath, $hostName, $slotLock); -my ($from, $to); - -REQ: while (1) { - $_ = || exit 0; - (my $amWilling, my $neededSystem, $drvPath, my $requiredFeatures) = split; - my @requiredFeatures = split /,/, $requiredFeatures; - - my $canBuildLocally = $amWilling && ($localSystem eq $neededSystem); - - if (!defined $currentLoad) { - sendReply "decline"; - next; - } - - # Acquire the exclusive lock on $currentLoad/main-lock. - mkdir $currentLoad, 0777 or die unless -d $currentLoad; - my $mainLock = "$currentLoad/main-lock"; - sysopen MAINLOCK, "$mainLock", O_RDWR|O_CREAT, 0600 or die; - flock(MAINLOCK, LOCK_EX) or die; - - - while (1) { - # Find all machine that can execute this build, i.e., that - # support builds for the given platform and features, and are - # not at their job limit. - my $rightType = 0; - my @available = (); - LOOP: foreach my $cur (@machines) { - if ($cur->{enabled} - && (grep { $neededSystem eq $_ } @{$cur->{systemTypes}}) - && all(map { my $f = $_; 0 != grep { $f eq $_ } @{$cur->{supportedFeatures}} } (@requiredFeatures, @mandatoryFeatures)) - && all(map { my $f = $_; 0 != grep { $f eq $_ } @requiredFeatures } @{$cur->{mandatoryFeatures}}) - ) - { - $rightType = 1; - - # We have a machine of the right type. Determine the load on - # the machine. - my $slot = 0; - my $load = 0; - my $free; - while ($slot < $cur->{maxJobs}) { - my $slotLock = openSlotLock($cur, $slot); - if (flock($slotLock, LOCK_EX | LOCK_NB)) { - $free = $slot unless defined $free; - flock($slotLock, LOCK_UN) or die; - } else { - $load++; - } - close $slotLock; - $slot++; - } - - push @available, { machine => $cur, load => $load, free => $free } - if $load < $cur->{maxJobs}; - } - } - - if ($debug) { - print STDERR "load on " . $_->{machine}->{hostName} . " = " . $_->{load} . "\n" - foreach @available; - } - - - # Didn't find any available machine? Then decline or postpone. - if (scalar @available == 0) { - # Postpone if we have a machine of the right type, except - # if the local system can and wants to do the build. - if ($rightType && !$canBuildLocally) { - sendReply "postpone"; - } else { - sendReply "decline"; - } - close MAINLOCK; - next REQ; - } - - - # Prioritise the available machines as follows: - # - First by load divided by speed factor, rounded to the nearest - # integer. This causes fast machines to be preferred over slow - # machines with similar loads. - # - Then by speed factor. - # - Finally by load. - sub lf { my $x = shift; return int($x->{load} / $x->{machine}->{speedFactor} + 0.4999); } - @available = sort - { lf($a) <=> lf($b) - || $b->{machine}->{speedFactor} <=> $a->{machine}->{speedFactor} - || $a->{load} <=> $b->{load} - } @available; - - - # Select the best available machine and lock a free slot. - my $selected = $available[0]; - my $machine = $selected->{machine}; - - $slotLock = openSlotLock($machine, $selected->{free}); - flock($slotLock, LOCK_EX | LOCK_NB) or die; - utime undef, undef, $slotLock; - - close MAINLOCK; - - - # Connect to the selected machine. - my @sshOpts = ("-i", $machine->{sshKey}); - $hostName = $machine->{hostName}; - eval { - ($from, $to) = connectToRemoteNix($hostName, \@sshOpts, "2>&4"); - # FIXME: check if builds are inhibited. - }; - last REQ unless $@; - print STDERR "$@"; - warn "unable to open SSH connection to ‘$hostName’, trying other available machines...\n"; - $from = undef; - $to = undef; - $machine->{enabled} = 0; - } -} - - -# Tell Nix we've accepted the build. -sendReply "accept"; -my @inputs = split /\s/, readline(STDIN); -my @outputs = split /\s/, readline(STDIN); - - -# Copy the derivation and its dependencies to the build machine. This -# is guarded by an exclusive lock per machine to prevent multiple -# build-remote instances from copying to a machine simultaneously. -# That's undesirable because we may end up with N instances uploading -# the same missing path simultaneously, causing the effective network -# bandwidth and target disk speed to be divided by N. -my $uploadLock = "$currentLoad/$hostName.upload-lock"; -sysopen UPLOADLOCK, "$uploadLock", O_RDWR|O_CREAT, 0600 or die; -eval { - local $SIG{ALRM} = sub { die "alarm\n" }; - # Don't wait forever, so that a process that gets stuck while - # holding the lock doesn't block everybody else indefinitely. - # It's safe to continue after a timeout, just (potentially) - # inefficient. - alarm 15 * 60; - flock(UPLOADLOCK, LOCK_EX); - alarm 0; -}; -if ($@) { - die unless $@ eq "alarm\n"; - print STDERR "somebody is hogging $uploadLock, continuing...\n"; - unlink $uploadLock; -} -Nix::CopyClosure::copyToOpen($from, $to, $hostName, [ $drvPath, @inputs ], 0, 0); -close UPLOADLOCK; - - -# Perform the build. -print STDERR "building ‘$drvPath’ on ‘$hostName’\n"; -writeInt(6, $to) or die; # == cmdBuildPaths -writeStrings([$drvPath], $to); -writeInt($maxSilentTime, $to); -writeInt($buildTimeout, $to); -my $res = readInt($from); -if ($res != 0) { - my $msg = decode("utf-8", readString($from)); - print STDERR "error: $msg on ‘$hostName’\n"; - exit $res; -} - - -# Copy the output from the build machine. -my @outputs2 = grep { !isValidPath($_) } @outputs; -if (scalar @outputs2 > 0) { - writeInt(5, $to); # == cmdExportPaths - writeInt(0, $to); # don't sign - writeStrings(\@outputs2, $to); - $ENV{'NIX_HELD_LOCKS'} = "@outputs2"; # FIXME: ugly - importPaths(fileno($from), 1); -} diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh new file mode 100755 index 00000000000..dac30d72d73 --- /dev/null +++ b/scripts/create-darwin-volume.sh @@ -0,0 +1,185 @@ +#!/bin/sh +set -e + +root_disk() { + diskutil info -plist / +} + +apfs_volumes_for() { + disk=$1 + diskutil apfs list -plist "$disk" +} + +disk_identifier() { + xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" 2>/dev/null +} + +volume_list_true() { + key=$1 + xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict/key[text()='$key']/following-sibling::true[1]" 2> /dev/null +} + +volume_get_string() { + key=$1 i=$2 + xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict[$i]/key[text()='$key']/following-sibling::string[1]/text()" 2> /dev/null +} + +find_nix_volume() { + disk=$1 + i=1 + volumes=$(apfs_volumes_for "$disk") + while true; do + name=$(echo "$volumes" | volume_get_string "Name" "$i") + if [ -z "$name" ]; then + break + fi + case "$name" in + [Nn]ix*) + echo "$name" + break + ;; + esac + i=$((i+1)) + done +} + +test_fstab() { + grep -q "/nix apfs rw" /etc/fstab 2>/dev/null +} + +test_nix_symlink() { + [ -L "/nix" ] || grep -q "^nix." /etc/synthetic.conf 2>/dev/null +} + +test_synthetic_conf() { + grep -q "^nix$" /etc/synthetic.conf 2>/dev/null +} + +test_nix() { + test -d "/nix" +} + +test_t2_chip_present(){ + # Use xartutil to see if system has a t2 chip. + # + # This isn't well-documented on its own; until it is, + # let's keep track of knowledge/assumptions. + # + # Warnings: + # - Don't search "xart" if porn will cause you trouble :) + # - Other xartutil flags do dangerous things. Don't run them + # naively. If you must, search "xartutil" first. + # + # Assumptions: + # - the "xART session seeds recovery utility" + # appears to interact with xartstorageremoted + # - `sudo xartutil --list` lists xART sessions + # and their seeds and exits 0 if successful. If + # not, it exits 1 and prints an error such as: + # xartutil: ERROR: No supported link to the SEP present + # - xART sessions/seeds are present when a T2 chip is + # (and not, otherwise) + # - the presence of a T2 chip means a newly-created + # volume on the primary drive will be + # encrypted at rest + # - all together: `sudo xartutil --list` + # should exit 0 if a new Nix Store volume will + # be encrypted at rest, and exit 1 if not. + sudo xartutil --list >/dev/null 2>/dev/null +} + +test_filevault_in_use() { + disk=$1 + # list vols on disk | get value of Filevault key | value is true + apfs_volumes_for "$disk" | volume_list_true FileVault | grep -q true +} + +# use after error msg for conditions we don't understand +suggest_report_error(){ + # ex "error: something sad happened :(" >&2 + echo " please report this @ https://github.com/nixos/nix/issues" >&2 +} + +main() { + ( + echo "" + echo " ------------------------------------------------------------------ " + echo " | This installer will create a volume for the nix store and |" + echo " | configure it to mount at /nix. Follow these steps to uninstall. |" + echo " ------------------------------------------------------------------ " + echo "" + echo " 1. Remove the entry from fstab using 'sudo vifs'" + echo " 2. Destroy the data volume using 'diskutil apfs deleteVolume'" + echo " 3. Remove the 'nix' line from /etc/synthetic.conf or the file" + echo "" + ) >&2 + + if test_nix_symlink; then + echo "error: /nix is a symlink, please remove it and make sure it's not in synthetic.conf (in which case a reboot is required)" >&2 + echo " /nix -> $(readlink "/nix")" >&2 + exit 2 + fi + + if ! test_synthetic_conf; then + echo "Configuring /etc/synthetic.conf..." >&2 + echo nix | sudo tee -a /etc/synthetic.conf + if ! test_synthetic_conf; then + echo "error: failed to configure synthetic.conf;" >&2 + suggest_report_error + exit 1 + fi + fi + + if ! test_nix; then + echo "Creating mountpoint for /nix..." >&2 + /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B || true + if ! test_nix; then + sudo mkdir -p /nix 2>/dev/null || true + fi + if ! test_nix; then + echo "error: failed to bootstrap /nix; if a reboot doesn't help," >&2 + suggest_report_error + exit 1 + fi + fi + + disk=$(root_disk | disk_identifier) + volume=$(find_nix_volume "$disk") + if [ -z "$volume" ]; then + echo "Creating a Nix Store volume..." >&2 + + if test_filevault_in_use "$disk"; then + # TODO: Not sure if it's in-scope now, but `diskutil apfs list` + # shows both filevault and encrypted at rest status, and it + # may be the more semantic way to test for this? It'll show + # `FileVault: No (Encrypted at rest)` + # `FileVault: No` + # `FileVault: Yes (Unlocked)` + # and so on. + if test_t2_chip_present; then + echo "warning: boot volume is FileVault-encrypted, but the Nix store volume" >&2 + echo " is only encrypted at rest." >&2 + echo " See https://nixos.org/nix/manual/#sect-macos-installation" >&2 + else + echo "error: refusing to create Nix store volume because the boot volume is" >&2 + echo " FileVault encrypted, but encryption-at-rest is not available." >&2 + echo " Manually create a volume for the store and re-run this script." >&2 + echo " See https://nixos.org/nix/manual/#sect-macos-installation" >&2 + exit 1 + fi + fi + + sudo diskutil apfs addVolume "$disk" APFS 'Nix Store' -mountpoint /nix + volume="Nix Store" + else + echo "Using existing '$volume' volume" >&2 + fi + + if ! test_fstab; then + echo "Configuring /etc/fstab..." >&2 + label=$(echo "$volume" | sed 's/ /\\040/g') + printf "\$a\nLABEL=%s /nix apfs rw,nobrowse\n.\nwq\n" "$label" | EDITOR=ed sudo vifs + fi +} + +main "$@" diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh new file mode 100644 index 00000000000..49076bd5c03 --- /dev/null +++ b/scripts/install-darwin-multi-user.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +readonly PLIST_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist + +dsclattr() { + /usr/bin/dscl . -read "$1" \ + | awk "/$2/ { print \$2 }" +} + +poly_validate_assumptions() { + if [ "$(uname -s)" != "Darwin" ]; then + failure "This script is for use with macOS!" + fi +} + +poly_service_installed_check() { + [ -e "$PLIST_DEST" ] +} + +poly_service_uninstall_directions() { + cat < /dev/null 2>&1 +} + +poly_group_id_get() { + dsclattr "/Groups/$1" "PrimaryGroupID" +} + +poly_create_build_group() { + _sudo "Create the Nix build group, $NIX_BUILD_GROUP_NAME" \ + /usr/sbin/dseditgroup -o create \ + -r "Nix build group for nix-daemon" \ + -i "$NIX_BUILD_GROUP_ID" \ + "$NIX_BUILD_GROUP_NAME" >&2 +} + +poly_user_exists() { + /usr/bin/dscl . -read "/Users/$1" > /dev/null 2>&1 +} + +poly_user_id_get() { + dsclattr "/Users/$1" "UniqueID" +} + +poly_user_hidden_get() { + dsclattr "/Users/$1" "IsHidden" +} + +poly_user_hidden_set() { + _sudo "in order to make $1 a hidden user" \ + /usr/bin/dscl . -create "/Users/$1" "IsHidden" "1" +} + +poly_user_home_get() { + dsclattr "/Users/$1" "NFSHomeDirectory" +} + +poly_user_home_set() { + _sudo "in order to give $1 a safe home directory" \ + /usr/bin/dscl . -create "/Users/$1" "NFSHomeDirectory" "$2" +} + +poly_user_note_get() { + dsclattr "/Users/$1" "RealName" +} + +poly_user_note_set() { + _sudo "in order to give $username a useful note" \ + /usr/bin/dscl . -create "/Users/$1" "RealName" "$2" +} + +poly_user_shell_get() { + dsclattr "/Users/$1" "UserShell" +} + +poly_user_shell_set() { + _sudo "in order to give $1 a safe home directory" \ + /usr/bin/dscl . -create "/Users/$1" "UserShell" "$2" +} + +poly_user_in_group_check() { + username=$1 + group=$2 + dseditgroup -o checkmember -m "$username" "$group" > /dev/null 2>&1 +} + +poly_user_in_group_set() { + username=$1 + group=$2 + + _sudo "Add $username to the $group group"\ + /usr/sbin/dseditgroup -o edit -t user \ + -a "$username" "$group" +} + +poly_user_primary_group_get() { + dsclattr "/Users/$1" "PrimaryGroupID" +} + +poly_user_primary_group_set() { + _sudo "to let the nix daemon use this user for builds (this might seem redundant, but there are two concepts of group membership)" \ + /usr/bin/dscl . -create "/Users/$1" "PrimaryGroupID" "$2" +} + +poly_create_build_user() { + username=$1 + uid=$2 + builder_num=$3 + + _sudo "Creating the Nix build user (#$builder_num), $username" \ + /usr/bin/dscl . create "/Users/$username" \ + UniqueID "${uid}" +} diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh new file mode 100644 index 00000000000..157e8ddb45e --- /dev/null +++ b/scripts/install-multi-user.sh @@ -0,0 +1,711 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +# Sourced from: +# - https://github.com/LnL7/nix-darwin/blob/8c29d0985d74b4a990238497c47a2542a5616b3c/bootstrap.sh +# - https://gist.github.com/expipiplus1/e571ce88c608a1e83547c918591b149f/ac504c6c1b96e65505fbda437a28ce563408ecb0 +# - https://github.com/NixOS/nixos-org-configurations/blob/a122f418797713d519aadf02e677fce0dc1cb446/delft/scripts/nix-mac-installer.sh +# - https://github.com/matthewbauer/macNixOS/blob/f6045394f9153edea417be90c216788e754feaba/install-macNixOS.sh +# - https://gist.github.com/LnL7/9717bd6cdcb30b086fd7f2093e5f8494/86b26f852ce563e973acd30f796a9a416248c34a +# +# however tracking which bits came from which would be impossible. + +readonly ESC='\033[0m' +readonly BOLD='\033[1m' +readonly BLUE='\033[34m' +readonly BLUE_UL='\033[4;34m' +readonly GREEN='\033[32m' +readonly GREEN_UL='\033[4;32m' +readonly RED='\033[31m' + +# installer allows overriding build user count to speed up installation +# as creating each user takes non-trivial amount of time on macos +readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32} +readonly NIX_BUILD_GROUP_ID="30000" +readonly NIX_BUILD_GROUP_NAME="nixbld" +readonly NIX_FIRST_BUILD_UID="30001" +# Please don't change this. We don't support it, because the +# default shell profile that comes with Nix doesn't support it. +readonly NIX_ROOT="/nix" +readonly NIX_EXTRA_CONF=${NIX_EXTRA_CONF:-} + +readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshenv") +readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix" +readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" + +readonly NIX_INSTALLED_NIX="@nix@" +readonly NIX_INSTALLED_CACERT="@cacert@" +readonly EXTRACTED_NIX_PATH="$(dirname "$0")" + +readonly ROOT_HOME=$(echo ~root) + +if [ -t 0 ]; then + readonly IS_HEADLESS='no' +else + readonly IS_HEADLESS='yes' +fi + +headless() { + if [ "$IS_HEADLESS" = "yes" ]; then + return 0 + else + return 1 + fi +} + +contactme() { + echo "We'd love to help if you need it." + echo "" + echo "If you can, open an issue at https://github.com/nixos/nix/issues" + echo "" + echo "Or feel free to contact the team," + echo " - on IRC #nixos on irc.freenode.net" + echo " - on twitter @nixos_org" +} + +uninstall_directions() { + subheader "Uninstalling nix:" + local step=0 + + if poly_service_installed_check; then + step=$((step + 1)) + poly_service_uninstall_directions "$step" + fi + + for profile_target in "${PROFILE_TARGETS[@]}"; do + if [ -e "$profile_target" ] && [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then + step=$((step + 1)) + cat < $1" +} + +bold() { + echo "$BOLD$*$ESC" +} + +ok() { + _textout "$GREEN" "$@" +} + +warning() { + warningheader "warning!" + cat + echo "" +} + +failure() { + header "oh no!" + _textout "$RED" "$@" + echo "" + _textout "$RED" "$(contactme)" + trap finish_cleanup EXIT + exit 1 +} + +ui_confirm() { + _textout "$GREEN$GREEN_UL" "$1" + + if headless; then + echo "No TTY, assuming you would say yes :)" + return 0 + fi + + local prompt="[y/n] " + echo -n "$prompt" + while read -r y; do + if [ "$y" = "y" ]; then + echo "" + return 0 + elif [ "$y" = "n" ]; then + echo "" + return 1 + else + _textout "$RED" "Sorry, I didn't understand. I can only understand answers of y or n" + echo -n "$prompt" + fi + done + echo "" + return 1 +} + +__sudo() { + local expl="$1" + local cmd="$2" + shift + header "sudo execution" + + echo "I am executing:" + echo "" + printf " $ sudo %s\\n" "$cmd" + echo "" + echo "$expl" + echo "" + + return 0 +} + +_sudo() { + local expl="$1" + shift + if ! headless; then + __sudo "$expl" "$*" + fi + sudo "$@" +} + + +readonly SCRATCH=$(mktemp -d -t tmp.XXXXXXXXXX) +function finish_cleanup { + rm -rf "$SCRATCH" +} + +function finish_fail { + finish_cleanup + + failure < /dev/null >&2; then + warning < "$SCRATCH/.nix-channels" + _sudo "to set up the default system channel (part 1)" \ + install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels" + fi +} + +welcome_to_nix() { + ok "Welcome to the Multi-User Nix Installation" + + cat < "$SCRATCH/nix.conf" +$NIX_EXTRA_CONF +build-users-group = $NIX_BUILD_GROUP_NAME +EOF + _sudo "to place the default nix daemon configuration (part 2)" \ + install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf +} + +main() { + if [ "$(uname -s)" = "Darwin" ]; then + # shellcheck source=./install-darwin-multi-user.sh + . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh" + elif [ "$(uname -s)" = "Linux" ]; then + if [ -e /run/systemd/system ]; then + # shellcheck source=./install-systemd-multi-user.sh + . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" + else + failure "Sorry, the multi-user installation requires systemd on Linux (detected using /run/systemd/system)" + fi + else + failure "Sorry, I don't know what to do on $(uname)" + fi + + welcome_to_nix + chat_about_sudo + + validate_starting_assumptions + + setup_report + + if ! ui_confirm "Ready to continue?"; then + ok "Alright, no changes have been made :)" + contactme + trap finish_cleanup EXIT + exit 1 + fi + + create_build_group + create_build_users + create_directories + place_channel_configuration + install_from_extracted_nix + + configure_shell_profile + + set +eu + . /etc/profile + set -eu + + setup_default_profile + place_nix_configuration + poly_configure_nix_daemon_service + + trap finish_success EXIT +} + + +main diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index d4eb1c6fbeb..826ca8b8c1a 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -8,22 +8,125 @@ nix="@nix@" cacert="@cacert@" -# macOS support for 10.10 or higher -if [[ "$(uname -s)" = "Darwin" && $(($(sw_vers -productVersion | cut -d '.' -f 2))) -lt 10 ]]; then - echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.10 or higher" - exit 1 -fi - if ! [ -e "$self/.reginfo" ]; then echo "$0: incomplete installer (.reginfo is missing)" >&2 - exit 1 fi -if [ -z "$USER" ]; then +if [ -z "$USER" ] && ! USER=$(id -u -n); then echo "$0: \$USER is not set" >&2 exit 1 fi +if [ -z "$HOME" ]; then + echo "$0: \$HOME is not set" >&2 + exit 1 +fi + +# macOS support for 10.12.6 or higher +if [ "$(uname -s)" = "Darwin" ]; then + macos_major=$(sw_vers -productVersion | cut -d '.' -f 2) + macos_minor=$(sw_vers -productVersion | cut -d '.' -f 3) + if [ "$macos_major" -lt 12 ] || { [ "$macos_major" -eq 12 ] && [ "$macos_minor" -lt 6 ]; }; then + echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher" + exit 1 + fi +fi + +# Determine if we could use the multi-user installer or not +if [ "$(uname -s)" = "Darwin" ]; then + echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2 +elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then + echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2 +fi + +INSTALL_MODE=no-daemon +CREATE_DARWIN_VOLUME=0 +# handle the command line flags +while [ $# -gt 0 ]; do + case $1 in + --daemon) + INSTALL_MODE=daemon;; + --no-daemon) + INSTALL_MODE=no-daemon;; + --no-channel-add) + export NIX_INSTALLER_NO_CHANNEL_ADD=1;; + --daemon-user-count) + export NIX_USER_COUNT=$2 + shift;; + --no-modify-profile) + NIX_INSTALLER_NO_MODIFY_PROFILE=1;; + --darwin-use-unencrypted-nix-store-volume) + CREATE_DARWIN_VOLUME=1;; + --nix-extra-conf-file) + export NIX_EXTRA_CONF="$(cat $2)" + shift;; + *) + ( + echo "Nix Installer [--daemon|--no-daemon] [--daemon-user-count INT] [--no-channel-add] [--no-modify-profile] [--darwin-use-unencrypted-nix-store-volume] [--nix-extra-conf-file FILE]" + + echo "Choose installation method." + echo "" + echo " --daemon: Installs and configures a background daemon that manages the store," + echo " providing multi-user support and better isolation for local builds." + echo " Both for security and reproducibility, this method is recommended if" + echo " supported on your platform." + echo " See https://nixos.org/nix/manual/#sect-multi-user-installation" + echo "" + echo " --no-daemon: Simple, single-user installation that does not require root and is" + echo " trivial to uninstall." + echo " (default)" + echo "" + echo " --no-channel-add: Don't add any channels. nixpkgs-unstable is installed by default." + echo "" + echo " --no-modify-profile: Skip channel installation. When not provided nixpkgs-unstable" + echo " is installed by default." + echo "" + echo " --daemon-user-count: Number of build users to create. Defaults to 32." + echo "" + echo " --nix-extra-conf-file: Path to nix.conf to prepend when installing /etc/nix.conf" + echo "" + ) >&2 + + # darwin and Catalina+ + if [ "$(uname -s)" = "Darwin" ] && [ "$macos_major" -gt 14 ]; then + ( + echo " --darwin-use-unencrypted-nix-store-volume: Create an APFS volume for the Nix" + echo " store and mount it at /nix. This is the recommended way to create" + echo " /nix with a read-only / on macOS >=10.15." + echo " See: https://nixos.org/nix/manual/#sect-macos-installation" + echo "" + ) >&2 + fi + exit;; + esac + shift +done + +if [ "$(uname -s)" = "Darwin" ]; then + if [ "$CREATE_DARWIN_VOLUME" = 1 ]; then + printf '\e[1;31mCreating volume and mountpoint /nix.\e[0m\n' + "$self/create-darwin-volume.sh" + fi + + info=$(diskutil info -plist / | xpath "/plist/dict/key[text()='Writable']/following-sibling::true[1]" 2> /dev/null) + if ! [ -e $dest ] && [ -n "$info" ] && [ "$macos_major" -gt 14 ]; then + ( + echo "" + echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume." + echo "Use sh <(curl https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume or run the preparation steps manually." + echo "See https://nixos.org/nix/manual/#sect-macos-installation" + echo "" + ) >&2 + exit 1 + fi +fi + +if [ "$INSTALL_MODE" = "daemon" ]; then + printf '\e[1;31mSwitching to the Daemon-based Installer\e[0m\n' + exec "$self/install-multi-user" + exit 0 +fi + if [ "$(id -u)" -eq 0 ]; then printf '\e[1;31mwarning: installing Nix as root is not supported by this script!\e[0m\n' fi @@ -34,13 +137,13 @@ if ! [ -e $dest ]; then cmd="mkdir -m 0755 $dest && chown $USER $dest" echo "directory $dest does not exist; creating it by running '$cmd' using sudo" >&2 if ! sudo sh -c "$cmd"; then - echo "$0: please manually run ‘$cmd’ as root to create $dest" >&2 + echo "$0: please manually run '$cmd' as root to create $dest" >&2 exit 1 fi fi if ! [ -w $dest ]; then - echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see http://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run ‘chown -R $USER $dest’ as root." >&2 + echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2 exit 1 fi @@ -55,7 +158,7 @@ for i in $(cd "$self/store" >/dev/null && echo ./*); do rm -rf "$i_tmp" fi if ! [ -e "$dest/store/$i" ]; then - cp -Rp "$self/store/$i" "$i_tmp" + cp -RPp "$self/store/$i" "$i_tmp" chmod -R a-w "$i_tmp" chmod +w "$i_tmp" mv "$i_tmp" "$dest/store/$i" @@ -64,12 +167,6 @@ for i in $(cd "$self/store" >/dev/null && echo ./*); do done echo "" >&2 -echo "initialising Nix database..." >&2 -if ! $nix/bin/nix-store --init; then - echo "$0: failed to initialize the Nix database" >&2 - exit 1 -fi - if ! "$nix/bin/nix-store" --load-db < "$self/.reginfo"; then echo "$0: unable to register valid paths" >&2 exit 1 @@ -89,19 +186,22 @@ if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then fi # Subscribe the user to the Nixpkgs channel and fetch it. -if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then - $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable -fi -if [ -z "$_NIX_INSTALLER_TEST" ]; then - $nix/bin/nix-channel --update nixpkgs +if [ -z "$NIX_INSTALLER_NO_CHANNEL_ADD" ]; then + if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then + $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable + fi + if [ -z "$_NIX_INSTALLER_TEST" ]; then + if ! $nix/bin/nix-channel --update nixpkgs; then + echo "Fetching the nixpkgs channel failed. (Are you offline?)" + echo "To try again later, run \"nix-channel --update nixpkgs\"." + fi + fi fi added= +p=$HOME/.nix-profile/etc/profile.d/nix.sh if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then - # Make the shell source nix.sh during login. - p=$HOME/.nix-profile/etc/profile.d/nix.sh - for i in .bash_profile .bash_login .profile; do fn="$HOME/$i" if [ -w "$fn" ]; then @@ -113,7 +213,17 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then break fi done - + for i in .zshenv .zshrc; do + fn="$HOME/$i" + if [ -w "$fn" ]; then + if ! grep -q "$p" "$fn"; then + echo "modifying $fn..." >&2 + echo "if [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn" + fi + added=1 + break + fi + done fi if [ -z "$added" ]; then diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh new file mode 100755 index 00000000000..e0201d53bd7 --- /dev/null +++ b/scripts/install-systemd-multi-user.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +readonly SERVICE_SRC=/lib/systemd/system/nix-daemon.service +readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service + +readonly SOCKET_SRC=/lib/systemd/system/nix-daemon.socket +readonly SOCKET_DEST=/etc/systemd/system/nix-daemon.socket + + +# Path for the systemd override unit file to contain the proxy settings +readonly SERVICE_OVERRIDE=${SERVICE_DEST}.d/override.conf + +create_systemd_override() { + header "Configuring proxy for the nix-daemon service" + _sudo "create directory for systemd unit override" mkdir -p "$(dirname $SERVICE_OVERRIDE)" + cat < /dev/null 2>&1 +} + +poly_group_id_get() { + getent group "$1" | cut -d: -f3 +} + +poly_create_build_group() { + _sudo "Create the Nix build group, $NIX_BUILD_GROUP_NAME" \ + groupadd -g "$NIX_BUILD_GROUP_ID" --system \ + "$NIX_BUILD_GROUP_NAME" >&2 +} + +poly_user_exists() { + getent passwd "$1" > /dev/null 2>&1 +} + +poly_user_id_get() { + getent passwd "$1" | cut -d: -f3 +} + +poly_user_hidden_get() { + echo "1" +} + +poly_user_hidden_set() { + true +} + +poly_user_home_get() { + getent passwd "$1" | cut -d: -f6 +} + +poly_user_home_set() { + _sudo "in order to give $1 a safe home directory" \ + usermod --home "$2" "$1" +} + +poly_user_note_get() { + getent passwd "$1" | cut -d: -f5 +} + +poly_user_note_set() { + _sudo "in order to give $1 a useful comment" \ + usermod --comment "$2" "$1" +} + +poly_user_shell_get() { + getent passwd "$1" | cut -d: -f7 +} + +poly_user_shell_set() { + _sudo "in order to prevent $1 from logging in" \ + usermod --shell "$2" "$1" +} + +poly_user_in_group_check() { + groups "$1" | grep -q "$2" > /dev/null 2>&1 +} + +poly_user_in_group_set() { + _sudo "Add $1 to the $2 group"\ + usermod --append --groups "$2" "$1" +} + +poly_user_primary_group_get() { + getent passwd "$1" | cut -d: -f4 +} + +poly_user_primary_group_set() { + _sudo "to let the nix daemon use this user for builds (this might seem redundant, but there are two concepts of group membership)" \ + usermod --gid "$2" "$1" + +} + +poly_create_build_user() { + username=$1 + uid=$2 + builder_num=$3 + + _sudo "Creating the Nix build user, $username" \ + useradd \ + --home-dir /var/empty \ + --comment "Nix build user $builder_num" \ + --gid "$NIX_BUILD_GROUP_ID" \ + --groups "$NIX_BUILD_GROUP_NAME" \ + --no-user-group \ + --system \ + --shell /sbin/nologin \ + --uid "$uid" \ + --password "!" \ + "$username" +} diff --git a/scripts/install.in b/scripts/install.in new file mode 100644 index 00000000000..1d26c4ff0eb --- /dev/null +++ b/scripts/install.in @@ -0,0 +1,69 @@ +#!/bin/sh + +# This script installs the Nix package manager on your system by +# downloading a binary distribution and running its installer script +# (which in turn creates and populates /nix). + +{ # Prevent execution if this script was only partially downloaded +oops() { + echo "$0:" "$@" >&2 + exit 1 +} + +tmpDir="$(mktemp -d -t nix-binary-tarball-unpack.XXXXXXXXXX || \ + oops "Can't create temporary directory for downloading the Nix binary tarball")" +cleanup() { + rm -rf "$tmpDir" +} +trap cleanup EXIT INT QUIT TERM + +require_util() { + command -v "$1" > /dev/null 2>&1 || + oops "you do not have '$1' installed, which I need to $2" +} + +case "$(uname -s).$(uname -m)" in + Linux.x86_64) system=x86_64-linux; hash=@binaryTarball_x86_64-linux@;; + Linux.i?86) system=i686-linux; hash=@binaryTarball_i686-linux@;; + Linux.aarch64) system=aarch64-linux; hash=@binaryTarball_aarch64-linux@;; + Darwin.x86_64) system=x86_64-darwin; hash=@binaryTarball_x86_64-darwin@;; + *) oops "sorry, there is no binary distribution of Nix for your platform";; +esac + +url="https://releases.nixos.org/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz" + +tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")" + +require_util curl "download the binary tarball" +require_util tar "unpack the binary tarball" +if [ "$(uname -s)" != "Darwin" ]; then + require_util xz "unpack the binary tarball" +fi + +echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..." +curl -L "$url" -o "$tarball" || oops "failed to download '$url'" + +if command -v sha256sum > /dev/null 2>&1; then + hash2="$(sha256sum -b "$tarball" | cut -c1-64)" +elif command -v shasum > /dev/null 2>&1; then + hash2="$(shasum -a 256 -b "$tarball" | cut -c1-64)" +elif command -v openssl > /dev/null 2>&1; then + hash2="$(openssl dgst -r -sha256 "$tarball" | cut -c1-64)" +else + oops "cannot verify the SHA-256 hash of '$url'; you need one of 'shasum', 'sha256sum', or 'openssl'" +fi + +if [ "$hash" != "$hash2" ]; then + oops "SHA-256 hash mismatch in '$url'; expected $hash, got $hash2" +fi + +unpack=$tmpDir/unpack +mkdir -p "$unpack" +tar -xJf "$tarball" -C "$unpack" || oops "failed to unpack '$url'" + +script=$(echo "$unpack"/*/install) + +[ -e "$script" ] || oops "installation script is missing from the binary tarball!" +"$script" "$@" + +} # End of wrapping diff --git a/scripts/local.mk b/scripts/local.mk index ee8ae6845dc..2a00558521b 100644 --- a/scripts/local.mk +++ b/scripts/local.mk @@ -1,10 +1,4 @@ -nix_bin_scripts := \ - $(d)/nix-copy-closure \ - -bin-scripts += $(nix_bin_scripts) - nix_noinst_scripts := \ - $(d)/build-remote.pl \ $(d)/nix-http-export.cgi \ $(d)/nix-profile.sh \ $(d)/nix-reduce-build @@ -14,6 +8,6 @@ noinst-scripts += $(nix_noinst_scripts) profiledir = $(sysconfdir)/profile.d $(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644)) -$(eval $(call install-program-in, $(d)/build-remote.pl, $(libexecdir)/nix)) +$(eval $(call install-file-as, $(d)/nix-profile-daemon.sh, $(profiledir)/nix-daemon.sh, 0644)) -clean-files += $(nix_bin_scripts) $(nix_noinst_scripts) +clean-files += $(nix_noinst_scripts) diff --git a/scripts/nix-copy-closure.in b/scripts/nix-copy-closure.in deleted file mode 100755 index af1d3091926..00000000000 --- a/scripts/nix-copy-closure.in +++ /dev/null @@ -1,103 +0,0 @@ -#! @perl@ -w @perlFlags@ - -use utf8; -use strict; -use Nix::SSH; -use Nix::Config; -use Nix::Store; -use Nix::CopyClosure; -use List::Util qw(sum); - -binmode STDERR, ":encoding(utf8)"; - -if (scalar @ARGV < 1) { - print STDERR < 0) { - print STDERR "copying ", scalar @missing, " missing paths from ‘$sshHost’...\n"; - writeInt(5, $to); # == cmdExportPaths - writeInt(0, $to); # obsolete - writeStrings(\@missing, $to); - importPaths(fileno($from), 1); - } - -} diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in new file mode 100644 index 00000000000..4bc7c6fc593 --- /dev/null +++ b/scripts/nix-profile-daemon.sh.in @@ -0,0 +1,27 @@ +# Only execute this file once per shell. +if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi +__ETC_PROFILE_NIX_SOURCED=1 + +export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" + +# Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. +if [ ! -z "${NIX_SSL_CERT_FILE:-}" ]; then + : # Allow users to override the NIX_SSL_CERT_FILE +elif [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch + export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt +elif [ -e /etc/ssl/ca-bundle.pem ]; then # openSUSE Tumbleweed + export NIX_SSL_CERT_FILE=/etc/ssl/ca-bundle.pem +elif [ -e /etc/ssl/certs/ca-bundle.crt ]; then # Old NixOS + export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt +elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS + export NIX_SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt +else + # Fall back to what is in the nix profiles, favouring whatever is defined last. + for i in $NIX_PROFILES; do + if [ -e $i/etc/ssl/certs/ca-bundle.crt ]; then + export NIX_SSL_CERT_FILE=$i/etc/ssl/certs/ca-bundle.crt + fi + done +fi + +export PATH="$HOME/.nix-profile/bin:@localstatedir@/nix/profiles/default/bin:$PATH" diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 3cdf431041c..8cba1c522fa 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -1,70 +1,13 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then - __savedpath="$PATH" - export PATH=@coreutils@ # Set up the per-user profile. # This part should be kept in sync with nixpkgs:nixos/modules/programs/shell.nix NIX_LINK=$HOME/.nix-profile - NIX_USER_PROFILE_DIR=@localstatedir@/nix/profiles/per-user/$USER - - mkdir -m 0755 -p "$NIX_USER_PROFILE_DIR" - - if [ "$(stat --printf '%u' "$NIX_USER_PROFILE_DIR")" != "$(id -u)" ]; then - echo "Nix: WARNING: bad ownership on "$NIX_USER_PROFILE_DIR", should be $(id -u)" >&2 - fi - - if [ -w "$HOME" ]; then - if ! [ -L "$NIX_LINK" ]; then - echo "Nix: creating $NIX_LINK" >&2 - if [ "$USER" != root ]; then - if ! ln -s "$NIX_USER_PROFILE_DIR"/profile "$NIX_LINK"; then - echo "Nix: WARNING: could not create $NIX_LINK -> $NIX_USER_PROFILE_DIR/profile" >&2 - fi - else - # Root installs in the system-wide profile by default. - ln -s @localstatedir@/nix/profiles/default "$NIX_LINK" - fi - fi - - # Subscribe the user to the unstable Nixpkgs channel by default. - if [ ! -e "$HOME/.nix-channels" ]; then - echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$HOME/.nix-channels" - fi - - # Create the per-user garbage collector roots directory. - __user_gcroots=@localstatedir@/nix/gcroots/per-user/"$USER" - mkdir -m 0755 -p "$__user_gcroots" - if [ "$(stat --printf '%u' "$__user_gcroots")" != "$(id -u)" ]; then - echo "Nix: WARNING: bad ownership on $__user_gcroots, should be $(id -u)" >&2 - fi - unset __user_gcroots - - # Set up a default Nix expression from which to install stuff. - __nix_defexpr="$HOME"/.nix-defexpr - [ -L "$__nix_defexpr" ] && rm -f "$__nix_defexpr" - mkdir -m 0755 -p "$__nix_defexpr" - if [ "$USER" != root ] && [ ! -L "$__nix_defexpr"/channels_root ]; then - ln -s @localstatedir@/nix/profiles/per-user/root/channels "$__nix_defexpr"/channels_root - fi - unset __nix_defexpr - fi - - # Append ~/.nix-defexpr/channels/nixpkgs to $NIX_PATH so that - # paths work when the user has fetched the Nixpkgs - # channel. - export NIX_PATH="${NIX_PATH:+$NIX_PATH:}nixpkgs=$HOME/.nix-defexpr/channels/nixpkgs" - # Set up environment. # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix - NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_USER_PROFILE_DIR" - - for i in $NIX_PROFILES; do - if [ -d "$i/lib/aspell" ]; then - export ASPELL_CONF="dict-dir $i/lib/aspell" - fi - done + export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch @@ -81,10 +24,10 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then export NIX_SSL_CERT_FILE="$NIX_LINK/etc/ca-bundle.crt" fi - if [ -n ${MANPATH} ]; then + if [ -n "${MANPATH-}" ]; then export MANPATH="$NIX_LINK/share/man:$MANPATH" fi - export PATH="$NIX_LINK/bin:$NIX_LINK/sbin:$__savedpath" - unset __savedpath NIX_LINK NIX_USER_PROFILE_DIR NIX_PROFILES + export PATH="$NIX_LINK/bin:$PATH" + unset NIX_LINK fi diff --git a/shell.nix b/shell.nix index 4c1608230ce..e3b422c7c9d 100644 --- a/shell.nix +++ b/shell.nix @@ -1,29 +1,15 @@ { useClang ? false }: -with import {}; +with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz) {}; + +with import ./release-common.nix { inherit pkgs; }; (if useClang then clangStdenv else stdenv).mkDerivation { name = "nix"; - buildInputs = - [ curl bison flex perl libxml2 libxslt bzip2 xz - pkgconfig sqlite libsodium boehmgc - docbook5 docbook5_xsl - autoconf-archive - (aws-sdk-cpp.override { - apis = ["s3"]; - customMemoryManagement = false; - }) - autoreconfHook - perlPackages.DBDSQLite - ]; - - configureFlags = - [ "--disable-init-state" - "--enable-gc" - "--with-dbi=${perlPackages.DBI}/${perl.libPrefix}" - "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}" - ]; + buildInputs = buildDeps ++ propagatedDeps ++ perlDeps ++ [ pkgs.rustfmt ]; + + inherit configureFlags; enableParallelBuilding = true; @@ -31,6 +17,9 @@ with import {}; shellHook = '' - configureFlags+=" --prefix=$(pwd)/inst" + export prefix=$(pwd)/inst + configureFlags+=" --prefix=$prefix" + PKG_CONFIG_PATH=$prefix/lib/pkgconfig:$PKG_CONFIG_PATH + PATH=$prefix/bin:$PATH ''; } diff --git a/src/boost/assert.hpp b/src/boost/assert.hpp deleted file mode 100644 index 754ebb954bc..00000000000 --- a/src/boost/assert.hpp +++ /dev/null @@ -1,38 +0,0 @@ -// -// boost/assert.hpp - BOOST_ASSERT(expr) -// -// Copyright (c) 2001, 2002 Peter Dimov and Multi Media Ltd. -// -// Permission to copy, use, modify, sell and distribute this software -// is granted provided this copyright notice appears in all copies. -// This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. -// -// Note: There are no include guards. This is intentional. -// -// See http://www.boost.org/libs/utility/assert.html for documentation. -// - -#undef BOOST_ASSERT - -#if defined(BOOST_DISABLE_ASSERTS) - -# define BOOST_ASSERT(expr) ((void)0) - -#elif defined(BOOST_ENABLE_ASSERT_HANDLER) - -#include - -namespace boost -{ - -void assertion_failed(char const * expr, char const * function, char const * file, long line); // user defined - -} // namespace boost - -#define BOOST_ASSERT(expr) ((expr)? ((void)0): ::boost::assertion_failed(#expr, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__)) - -#else -# include -# define BOOST_ASSERT(expr) assert(expr) -#endif diff --git a/src/boost/format.hpp b/src/boost/format.hpp deleted file mode 100644 index f965f0f33e9..00000000000 --- a/src/boost/format.hpp +++ /dev/null @@ -1,64 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rdiger Loos's format class -// and Karl Nelson's ofstream - -// ---------------------------------------------------------------------------- -// format.hpp : primary header -// ---------------------------------------------------------------------------- - -#ifndef BOOST_FORMAT_HPP -#define BOOST_FORMAT_HPP - -#include -#include -#include -#include - -#if HAVE_LOCALE -#include -#else -#define BOOST_NO_STD_LOCALE -#define BOOST_NO_LOCALE_ISIDIGIT -#include -#endif - -#include - - -// **** Forward declarations ---------------------------------- -#include // basic_format, and other frontends -#include // misc forward declarations for internal use - - -// **** Auxiliary structs (stream_format_state , and format_item ) -#include - -// **** Format class interface -------------------------------- -#include - -// **** Exceptions ----------------------------------------------- -#include - -// **** Implementation ------------------------------------------- -//#include // member functions - -#include // class for grouping arguments - -#include // argument-feeding functions -//#include // format-string parsing (member-)functions - -// **** Implementation of the free functions ---------------------- -//#include - - -#endif // BOOST_FORMAT_HPP diff --git a/src/boost/format/exceptions.hpp b/src/boost/format/exceptions.hpp deleted file mode 100644 index a7641458c95..00000000000 --- a/src/boost/format/exceptions.hpp +++ /dev/null @@ -1,96 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rdiger Loos's format class -// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) - -// ------------------------------------------------------------------------------ -// exceptions.hpp -// ------------------------------------------------------------------------------ - - -#ifndef BOOST_FORMAT_EXCEPTIONS_HPP -#define BOOST_FORMAT_EXCEPTIONS_HPP - - -#include - - -namespace boost { - -namespace io { - -// **** exceptions ----------------------------------------------- - -class format_error : public std::exception -{ -public: - format_error() { abort(); } - virtual const char *what() const throw() - { - return "boost::format_error: " - "format generic failure"; - } -}; - -class bad_format_string : public format_error -{ -public: - bad_format_string() { abort(); } - virtual const char *what() const throw() - { - return "boost::bad_format_string: " - "format-string is ill-formed"; - } -}; - -class too_few_args : public format_error -{ -public: - too_few_args() { abort(); } - virtual const char *what() const throw() - { - return "boost::too_few_args: " - "format-string refered to more arguments than were passed"; - } -}; - -class too_many_args : public format_error -{ -public: - too_many_args() { abort(); } - virtual const char *what() const throw() - { - return "boost::too_many_args: " - "format-string refered to less arguments than were passed"; - } -}; - - -class out_of_range : public format_error -{ -public: - out_of_range() { abort(); } - virtual const char *what() const throw() - { - return "boost::out_of_range: " - "tried to refer to an argument (or item) number which is out of range, " - "according to the format string."; - } -}; - - -} // namespace io - -} // namespace boost - - -#endif // BOOST_FORMAT_EXCEPTIONS_HPP diff --git a/src/boost/format/feed_args.hpp b/src/boost/format/feed_args.hpp deleted file mode 100644 index 3d0b47b4a12..00000000000 --- a/src/boost/format/feed_args.hpp +++ /dev/null @@ -1,247 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rdiger Loos's format class -// and Karl Nelson's ofstream - -// ---------------------------------------------------------------------------- -// feed_args.hpp : functions for processing each argument -// (feed, feed_manip, and distribute) -// ---------------------------------------------------------------------------- - - -#ifndef BOOST_FORMAT_FEED_ARGS_HPP -#define BOOST_FORMAT_FEED_ARGS_HPP - -#include "boost/format/format_class.hpp" -#include "boost/format/group.hpp" - -#include "boost/throw_exception.hpp" - -namespace boost { -namespace io { -namespace detail { -namespace { - - inline - void empty_buf(BOOST_IO_STD ostringstream & os) { - static const std::string emptyStr; - os.str(emptyStr); - } - - void do_pad( std::string & s, - std::streamsize w, - const char c, - std::ios::fmtflags f, - bool center) - // applies centered / left / right padding to the string s. - // Effects : string s is padded. - { - std::streamsize n=w-s.size(); - if(n<=0) { - return; - } - if(center) - { - s.reserve(w); // allocate once for the 2 inserts - const std::streamsize n1 = n /2, n0 = n - n1; - s.insert(s.begin(), n0, c); - s.append(n1, c); - } - else - { - if(f & std::ios::left) { - s.append(n, c); - } - else { - s.insert(s.begin(), n, c); - } - } - } // -do_pad(..) - - - template inline - void put_head(BOOST_IO_STD ostream& , const T& ) { - } - - template inline - void put_head( BOOST_IO_STD ostream& os, const group1& x ) { - os << group_head(x.a1_); // send the first N-1 items, not the last - } - - template inline - void put_last( BOOST_IO_STD ostream& os, const T& x ) { - os << x ; - } - - template inline - void put_last( BOOST_IO_STD ostream& os, const group1& x ) { - os << group_last(x.a1_); // this selects the last element - } - -#ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST - template inline - void put_head( BOOST_IO_STD ostream& , T& ) { - } - - template inline - void put_last( BOOST_IO_STD ostream& os, T& x ) { - os << x ; - } -#endif - - - - -template -void put( T x, - const format_item& specs, - std::string & res, - BOOST_IO_STD ostringstream& oss_ ) -{ - // does the actual conversion of x, with given params, into a string - // using the *supplied* strinstream. (the stream state is important) - - typedef std::string string_t; - typedef format_item format_item_t; - - stream_format_state prev_state(oss_); - - specs.state_.apply_on(oss_); - - // in case x is a group, apply the manip part of it, - // in order to find width - put_head( oss_, x ); - empty_buf( oss_); - - const std::streamsize w=oss_.width(); - const std::ios::fmtflags fl=oss_.flags(); - const bool internal = (fl & std::ios::internal) != 0; - const bool two_stepped_padding = internal - && ! ( specs.pad_scheme_ & format_item_t::spacepad ) - && specs.truncate_ < 0 ; - - - if(! two_stepped_padding) - { - if(w>0) // handle simple padding via do_pad, not natively in stream - oss_.width(0); - put_last( oss_, x); - res = oss_.str(); - - if (specs.truncate_ >= 0) - res.erase(specs.truncate_); - - // complex pads : - if(specs.pad_scheme_ & format_item_t::spacepad) - { - if( res.size()==0 || ( res[0]!='+' && res[0]!='-' )) - { - res.insert(res.begin(), 1, ' '); // insert 1 space at pos 0 - } - } - if(w > 0) // need do_pad - { - do_pad(res,w,oss_.fill(), fl, (specs.pad_scheme_ & format_item_t::centered) !=0 ); - } - } - else // 2-stepped padding - { - put_last( oss_, x); // oss_.width() may result in padding. - res = oss_.str(); - - if (specs.truncate_ >= 0) - res.erase(specs.truncate_); - - if( res.size() - w > 0) - { // length w exceeded - // either it was multi-output with first output padding up all width.. - // either it was one big arg and we are fine. - empty_buf( oss_); - oss_.width(0); - put_last(oss_, x ); - string_t tmp = oss_.str(); // minimal-length output - std::streamsize d; - if( (d=w - tmp.size()) <=0 ) - { - // minimal length is already >= w, so no padding (cool!) - res.swap(tmp); - } - else - { // hum.. we need to pad (it was necessarily multi-output) - typedef typename string_t::size_type size_type; - size_type i = 0; - while( i( d ), oss_.fill()); - res.swap( tmp ); - } - } - else - { // okay, only one thing was printed and padded, so res is fine. - } - } - - prev_state.apply_on(oss_); - empty_buf( oss_); - oss_.clear(); -} // end- put(..) - - -} // local namespace - - - - - -template -void distribute(basic_format& self, T x) - // call put(x, ..) on every occurence of the current argument : -{ - if(self.cur_arg_ >= self.num_args_) - { - if( self.exceptions() & too_many_args_bit ) - boost::throw_exception(too_many_args()); // too many variables have been supplied ! - else return; - } - for(unsigned long i=0; i < self.items_.size(); ++i) - { - if(self.items_[i].argN_ == self.cur_arg_) - { - put (x, self.items_[i], self.items_[i].res_, self.oss_ ); - } - } -} - -template -basic_format& feed(basic_format& self, T x) -{ - if(self.dumped_) self.clear(); - distribute (self, x); - ++self.cur_arg_; - if(self.bound_.size() != 0) - { - while( self.cur_arg_ < self.num_args_ && self.bound_[self.cur_arg_] ) - ++self.cur_arg_; - } - - // this arg is finished, reset the stream's format state - self.state0_.apply_on(self.oss_); - return self; -} - - -} // namespace detail -} // namespace io -} // namespace boost - - -#endif // BOOST_FORMAT_FEED_ARGS_HPP diff --git a/src/boost/format/format_class.hpp b/src/boost/format/format_class.hpp deleted file mode 100644 index 6875623acb4..00000000000 --- a/src/boost/format/format_class.hpp +++ /dev/null @@ -1,135 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rdiger Loos's format class -// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) - -// ------------------------------------------------------------------------------ -// format_class.hpp : class interface -// ------------------------------------------------------------------------------ - - -#ifndef BOOST_FORMAT_CLASS_HPP -#define BOOST_FORMAT_CLASS_HPP - -#include -#include - -#include -#include - -#include - -namespace boost { - -class basic_format -{ -public: - typedef std::string string_t; - typedef BOOST_IO_STD ostringstream internal_stream_t; -private: - typedef BOOST_IO_STD ostream stream_t; - typedef io::detail::stream_format_state stream_format_state; - typedef io::detail::format_item format_item_t; - -public: - basic_format(const char* str); - basic_format(const string_t& s); -#ifndef BOOST_NO_STD_LOCALE - basic_format(const char* str, const std::locale & loc); - basic_format(const string_t& s, const std::locale & loc); -#endif // no locale - basic_format(const basic_format& x); - basic_format& operator= (const basic_format& x); - - basic_format& clear(); // empty the string buffers (except bound arguments, see clear_binds() ) - - // pass arguments through those operators : - template basic_format& operator%(const T& x) - { - return io::detail::feed(*this,x); - } - -#ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST - template basic_format& operator%(T& x) - { - return io::detail::feed(*this,x); - } -#endif - - - // system for binding arguments : - template - basic_format& bind_arg(int argN, const T& val) - { - return io::detail::bind_arg_body(*this, argN, val); - } - basic_format& clear_bind(int argN); - basic_format& clear_binds(); - - // modify the params of a directive, by applying a manipulator : - template - basic_format& modify_item(int itemN, const T& manipulator) - { - return io::detail::modify_item_body(*this, itemN, manipulator) ; - } - - // Choosing which errors will throw exceptions : - unsigned char exceptions() const; - unsigned char exceptions(unsigned char newexcept); - - // final output - string_t str() const; - friend BOOST_IO_STD ostream& - operator<< ( BOOST_IO_STD ostream& , const basic_format& ); - - - template friend basic_format& - io::detail::feed(basic_format&, T); - - template friend - void io::detail::distribute(basic_format&, T); - - template friend - basic_format& io::detail::modify_item_body(basic_format&, int, const T&); - - template friend - basic_format& io::detail::bind_arg_body(basic_format&, int, const T&); - -// make the members private only if the friend templates are supported -private: - - // flag bits, used for style_ - enum style_values { ordered = 1, // set only if all directives are positional directives - special_needs = 4 }; - - // parse the format string : - void parse(const string_t&); - - int style_; // style of format-string : positional or not, etc - int cur_arg_; // keep track of wich argument will come - int num_args_; // number of expected arguments - mutable bool dumped_; // true only after call to str() or << - std::vector items_; // vector of directives (aka items) - string_t prefix_; // piece of string to insert before first item - - std::vector bound_; // stores which arguments were bound - // size = num_args OR zero - internal_stream_t oss_; // the internal stream. - stream_format_state state0_; // reference state for oss_ - unsigned char exceptions_; -}; // class basic_format - - -} // namespace boost - - -#endif // BOOST_FORMAT_CLASS_HPP diff --git a/src/boost/format/format_fwd.hpp b/src/boost/format/format_fwd.hpp deleted file mode 100644 index 97c55f6684c..00000000000 --- a/src/boost/format/format_fwd.hpp +++ /dev/null @@ -1,49 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rdiger Loos's format class -// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) - -// ------------------------------------------------------------------------------ -// format_fwd.hpp : forward declarations, for primary header format.hpp -// ------------------------------------------------------------------------------ - -#ifndef BOOST_FORMAT_FWD_HPP -#define BOOST_FORMAT_FWD_HPP - -#include -#include - -namespace boost { - -class basic_format; - -typedef basic_format format; - -namespace io { -enum format_error_bits { bad_format_string_bit = 1, - too_few_args_bit = 2, too_many_args_bit = 4, - out_of_range_bit = 8, - all_error_bits = 255, no_error_bits=0 }; - -// Convertion: format to string -std::string str(const basic_format& ) ; - -} // namespace io - - -BOOST_IO_STD ostream& -operator<<( BOOST_IO_STD ostream&, const basic_format&); - - -} // namespace boost - -#endif // BOOST_FORMAT_FWD_HPP diff --git a/src/boost/format/format_implementation.cc b/src/boost/format/format_implementation.cc deleted file mode 100644 index aa191afe113..00000000000 --- a/src/boost/format/format_implementation.cc +++ /dev/null @@ -1,256 +0,0 @@ -// -*- C++ -*- -// Boost general library format --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rdiger Loos's format class -// and Karl Nelson's ofstream - -// ---------------------------------------------------------------------------- -// format_implementation.hpp Implementation of the basic_format class -// ---------------------------------------------------------------------------- - - -#ifndef BOOST_FORMAT_IMPLEMENTATION_HPP -#define BOOST_FORMAT_IMPLEMENTATION_HPP - -#include -#include -#include - -namespace boost { - -// -------- format:: ------------------------------------------- -basic_format::basic_format(const char* str) - : style_(0), cur_arg_(0), num_args_(0), dumped_(false), - items_(), oss_(), exceptions_(io::all_error_bits) -{ - state0_.set_by_stream(oss_); - string_t emptyStr; - if( !str) str = emptyStr.c_str(); - parse( str ); -} - -#ifndef BOOST_NO_STD_LOCALE -basic_format::basic_format(const char* str, const std::locale & loc) - : style_(0), cur_arg_(0), num_args_(0), dumped_(false), - items_(), oss_(), exceptions_(io::all_error_bits) -{ - oss_.imbue( loc ); - state0_.set_by_stream(oss_); - string_t emptyStr; - if( !str) str = emptyStr.c_str(); - parse( str ); -} - -basic_format::basic_format(const string_t& s, const std::locale & loc) - : style_(0), cur_arg_(0), num_args_(0), dumped_(false), - items_(), oss_(), exceptions_(io::all_error_bits) -{ - oss_.imbue( loc ); - state0_.set_by_stream(oss_); - parse(s); -} -#endif //BOOST_NO_STD_LOCALE - -basic_format::basic_format(const string_t& s) - : style_(0), cur_arg_(0), num_args_(0), dumped_(false), - items_(), oss_(), exceptions_(io::all_error_bits) -{ - state0_.set_by_stream(oss_); - parse(s); -} - -basic_format:: basic_format(const basic_format& x) - : style_(x.style_), cur_arg_(x.cur_arg_), num_args_(x.num_args_), dumped_(false), - items_(x.items_), prefix_(x.prefix_), bound_(x.bound_), - oss_(), // <- we obviously can't copy x.oss_ - state0_(x.state0_), exceptions_(x.exceptions_) -{ - state0_.apply_on(oss_); -} - -basic_format& basic_format::operator= (const basic_format& x) -{ - if(this == &x) - return *this; - state0_ = x.state0_; - state0_.apply_on(oss_); - - // plus all the other (trivial) assignments : - exceptions_ = x.exceptions_; - items_ = x.items_; - prefix_ = x.prefix_; - bound_=x.bound_; - style_=x.style_; - cur_arg_=x.cur_arg_; - num_args_=x.num_args_; - dumped_=x.dumped_; - return *this; -} - - -unsigned char basic_format::exceptions() const -{ - return exceptions_; -} - -unsigned char basic_format::exceptions(unsigned char newexcept) -{ - unsigned char swp = exceptions_; - exceptions_ = newexcept; - return swp; -} - - -basic_format& basic_format ::clear() - // empty the string buffers (except bound arguments, see clear_binds() ) - // and make the format object ready for formatting a new set of arguments -{ - BOOST_ASSERT( bound_.size()==0 || num_args_ == static_cast(bound_.size()) ); - - for(unsigned long i=0; i num_args_ || bound_.size()==0 || !bound_[argN-1] ) - { - if( exceptions() & io::out_of_range_bit ) - boost::throw_exception(io::out_of_range()); // arg not in range. - else return *this; - } - bound_[argN-1]=false; - clear(); - return *this; -} - - - -std::string basic_format::str() const -{ - dumped_=true; - if(items_.size()==0) - return prefix_; - if( cur_arg_ < num_args_) - if( exceptions() & io::too_few_args_bit ) - boost::throw_exception(io::too_few_args()); // not enough variables have been supplied ! - - unsigned long sz = prefix_.size(); - unsigned long i; - for(i=0; i < items_.size(); ++i) - sz += items_[i].res_.size() + items_[i].appendix_.size(); - string_t res; - res.reserve(sz); - - res += prefix_; - for(i=0; i < items_.size(); ++i) - { - const format_item_t& item = items_[i]; - res += item.res_; - if( item.argN_ == format_item_t::argN_tabulation) - { - BOOST_ASSERT( item.pad_scheme_ & format_item_t::tabulation); - std::streamsize n = item.state_.width_ - res.size(); - if( n > 0 ) - res.append( n, item.state_.fill_ ); - } - res += item.appendix_; - } - return res; -} - -namespace io { -namespace detail { - -template -basic_format& bind_arg_body( basic_format& self, - int argN, - const T& val) - // bind one argument to a fixed value - // this is persistent over clear() calls, thus also over str() and << -{ - if(self.dumped_) self.clear(); // needed, because we will modify cur_arg_.. - if(argN<1 || argN > self.num_args_) - { - if( self.exceptions() & io::out_of_range_bit ) - boost::throw_exception(io::out_of_range()); // arg not in range. - else return self; - } - if(self.bound_.size()==0) - self.bound_.assign(self.num_args_,false); - else - BOOST_ASSERT( self.num_args_ == static_cast(self.bound_.size()) ); - int o_cur_arg = self.cur_arg_; - self.cur_arg_ = argN-1; // arrays begin at 0 - - self.bound_[self.cur_arg_]=false; // if already set, we unset and re-sets.. - self.operator%(val); // put val at the right place, because cur_arg is set - - - // Now re-position cur_arg before leaving : - self.cur_arg_ = o_cur_arg; - self.bound_[argN-1]=true; - if(self.cur_arg_ == argN-1 ) - // hum, now this arg is bound, so move to next free arg - { - while(self.cur_arg_ < self.num_args_ && self.bound_[self.cur_arg_]) ++self.cur_arg_; - } - // In any case, we either have all args, or are on a non-binded arg : - BOOST_ASSERT( self.cur_arg_ >= self.num_args_ || ! self.bound_[self.cur_arg_]); - return self; -} - -template -basic_format& modify_item_body( basic_format& self, - int itemN, - const T& manipulator) - // applies a manipulator to the format_item describing a given directive. - // this is a permanent change, clear or clear_binds won't cancel that. -{ - if(itemN<1 || itemN >= static_cast(self.items_.size() )) - { - if( self.exceptions() & io::out_of_range_bit ) - boost::throw_exception(io::out_of_range()); // item not in range. - else return self; - } - self.items_[itemN-1].ref_state_.apply_manip( manipulator ); - self.items_[itemN-1].state_ = self.items_[itemN-1].ref_state_; - return self; -} - -} // namespace detail - -} // namespace io - -} // namespace boost - - - -#endif // BOOST_FORMAT_IMPLEMENTATION_HPP diff --git a/src/boost/format/free_funcs.cc b/src/boost/format/free_funcs.cc deleted file mode 100644 index 151db37a0ac..00000000000 --- a/src/boost/format/free_funcs.cc +++ /dev/null @@ -1,71 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rdiger Loos's format class -// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) - -// ------------------------------------------------------------------------------ -// free_funcs.hpp : implementation of the free functions declared in namespace format -// ------------------------------------------------------------------------------ - -#ifndef BOOST_FORMAT_FUNCS_HPP -#define BOOST_FORMAT_FUNCS_HPP - -#include "boost/format.hpp" -#include "boost/throw_exception.hpp" - -namespace boost { - -namespace io { - inline - std::string str(const basic_format& f) - // adds up all pieces of strings and converted items, and return the formatted string - { - return f.str(); - } -} // - namespace io - -BOOST_IO_STD ostream& -operator<<( BOOST_IO_STD ostream& os, - const boost::basic_format& f) - // effect: "return os << str(f);" but we can try to do it faster -{ - typedef boost::basic_format format_t; - if(f.items_.size()==0) - os << f.prefix_; - else { - if(f.cur_arg_ < f.num_args_) - if( f.exceptions() & io::too_few_args_bit ) - boost::throw_exception(io::too_few_args()); // not enough variables have been supplied ! - if(f.style_ & format_t::special_needs) - os << f.str(); - else { - // else we dont have to count chars output, so we dump directly to os : - os << f.prefix_; - for(unsigned long i=0; i -inline -BOOST_IO_STD ostream& -operator << ( BOOST_IO_STD ostream& os, - const group0& ) -{ - return os; -} - -template -struct group1 -{ - T1 a1_; - group1(T1 a1) - : a1_(a1) - {} -}; - -template -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group1& x) -{ - os << x.a1_; - return os; -} - - - - -template -struct group2 -{ - T1 a1_; - T2 a2_; - group2(T1 a1,T2 a2) - : a1_(a1),a2_(a2) - {} -}; - -template -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group2& x) -{ - os << x.a1_<< x.a2_; - return os; -} - -template -struct group3 -{ - T1 a1_; - T2 a2_; - T3 a3_; - group3(T1 a1,T2 a2,T3 a3) - : a1_(a1),a2_(a2),a3_(a3) - {} -}; - -template -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group3& x) -{ - os << x.a1_<< x.a2_<< x.a3_; - return os; -} - -template -struct group4 -{ - T1 a1_; - T2 a2_; - T3 a3_; - T4 a4_; - group4(T1 a1,T2 a2,T3 a3,T4 a4) - : a1_(a1),a2_(a2),a3_(a3),a4_(a4) - {} -}; - -template -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group4& x) -{ - os << x.a1_<< x.a2_<< x.a3_<< x.a4_; - return os; -} - -template -struct group5 -{ - T1 a1_; - T2 a2_; - T3 a3_; - T4 a4_; - T5 a5_; - group5(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5) - : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5) - {} -}; - -template -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group5& x) -{ - os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_; - return os; -} - -template -struct group6 -{ - T1 a1_; - T2 a2_; - T3 a3_; - T4 a4_; - T5 a5_; - T6 a6_; - group6(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6) - : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6) - {} -}; - -template -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group6& x) -{ - os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_; - return os; -} - -template -struct group7 -{ - T1 a1_; - T2 a2_; - T3 a3_; - T4 a4_; - T5 a5_; - T6 a6_; - T7 a7_; - group7(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7) - : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7) - {} -}; - -template -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group7& x) -{ - os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_; - return os; -} - -template -struct group8 -{ - T1 a1_; - T2 a2_; - T3 a3_; - T4 a4_; - T5 a5_; - T6 a6_; - T7 a7_; - T8 a8_; - group8(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8) - : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7),a8_(a8) - {} -}; - -template -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group8& x) -{ - os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_<< x.a8_; - return os; -} - -template -struct group9 -{ - T1 a1_; - T2 a2_; - T3 a3_; - T4 a4_; - T5 a5_; - T6 a6_; - T7 a7_; - T8 a8_; - T9 a9_; - group9(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9) - : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7),a8_(a8),a9_(a9) - {} -}; - -template -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group9& x) -{ - os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_<< x.a8_<< x.a9_; - return os; -} - -template -struct group10 -{ - T1 a1_; - T2 a2_; - T3 a3_; - T4 a4_; - T5 a5_; - T6 a6_; - T7 a7_; - T8 a8_; - T9 a9_; - T10 a10_; - group10(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9,T10 a10) - : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7),a8_(a8),a9_(a9),a10_(a10) - {} -}; - -template -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group10& x) -{ - os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_<< x.a8_<< x.a9_<< x.a10_; - return os; -} - - - - -template -inline -group1 -group_head( group2 const& x) -{ - return group1 (x.a1_); -} - -template -inline -group1 -group_last( group2 const& x) -{ - return group1 (x.a2_); -} - - - -template -inline -group2 -group_head( group3 const& x) -{ - return group2 (x.a1_,x.a2_); -} - -template -inline -group1 -group_last( group3 const& x) -{ - return group1 (x.a3_); -} - - - -template -inline -group3 -group_head( group4 const& x) -{ - return group3 (x.a1_,x.a2_,x.a3_); -} - -template -inline -group1 -group_last( group4 const& x) -{ - return group1 (x.a4_); -} - - - -template -inline -group4 -group_head( group5 const& x) -{ - return group4 (x.a1_,x.a2_,x.a3_,x.a4_); -} - -template -inline -group1 -group_last( group5 const& x) -{ - return group1 (x.a5_); -} - - - -template -inline -group5 -group_head( group6 const& x) -{ - return group5 (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_); -} - -template -inline -group1 -group_last( group6 const& x) -{ - return group1 (x.a6_); -} - - - -template -inline -group6 -group_head( group7 const& x) -{ - return group6 (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_); -} - -template -inline -group1 -group_last( group7 const& x) -{ - return group1 (x.a7_); -} - - - -template -inline -group7 -group_head( group8 const& x) -{ - return group7 (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_,x.a7_); -} - -template -inline -group1 -group_last( group8 const& x) -{ - return group1 (x.a8_); -} - - - -template -inline -group8 -group_head( group9 const& x) -{ - return group8 (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_,x.a7_,x.a8_); -} - -template -inline -group1 -group_last( group9 const& x) -{ - return group1 (x.a9_); -} - - - -template -inline -group9 -group_head( group10 const& x) -{ - return group9 (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_,x.a7_,x.a8_,x.a9_); -} - -template -inline -group1 -group_last( group10 const& x) -{ - return group1 (x.a10_); -} - - - - - -} // namespace detail - - - -// helper functions - - -inline detail::group1< detail::group0 > -group() { return detail::group1< detail::group0 > ( detail::group0() ); } - -template -inline -detail::group1< detail::group2 > - group(T1 a1, Var const& var) -{ - return detail::group1< detail::group2 > - ( detail::group2 - (a1, var) - ); -} - -template -inline -detail::group1< detail::group3 > - group(T1 a1,T2 a2, Var const& var) -{ - return detail::group1< detail::group3 > - ( detail::group3 - (a1,a2, var) - ); -} - -template -inline -detail::group1< detail::group4 > - group(T1 a1,T2 a2,T3 a3, Var const& var) -{ - return detail::group1< detail::group4 > - ( detail::group4 - (a1,a2,a3, var) - ); -} - -template -inline -detail::group1< detail::group5 > - group(T1 a1,T2 a2,T3 a3,T4 a4, Var const& var) -{ - return detail::group1< detail::group5 > - ( detail::group5 - (a1,a2,a3,a4, var) - ); -} - -template -inline -detail::group1< detail::group6 > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5, Var const& var) -{ - return detail::group1< detail::group6 > - ( detail::group6 - (a1,a2,a3,a4,a5, var) - ); -} - -template -inline -detail::group1< detail::group7 > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6, Var const& var) -{ - return detail::group1< detail::group7 > - ( detail::group7 - (a1,a2,a3,a4,a5,a6, var) - ); -} - -template -inline -detail::group1< detail::group8 > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7, Var const& var) -{ - return detail::group1< detail::group8 > - ( detail::group8 - (a1,a2,a3,a4,a5,a6,a7, var) - ); -} - -template -inline -detail::group1< detail::group9 > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8, Var const& var) -{ - return detail::group1< detail::group9 > - ( detail::group9 - (a1,a2,a3,a4,a5,a6,a7,a8, var) - ); -} - -template -inline -detail::group1< detail::group10 > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9, Var const& var) -{ - return detail::group1< detail::group10 > - ( detail::group10 - (a1,a2,a3,a4,a5,a6,a7,a8,a9, var) - ); -} - - -#ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST - -template -inline -detail::group1< detail::group2 > - group(T1 a1, Var& var) -{ - return detail::group1< detail::group2 > - ( detail::group2 - (a1, var) - ); -} - -template -inline -detail::group1< detail::group3 > - group(T1 a1,T2 a2, Var& var) -{ - return detail::group1< detail::group3 > - ( detail::group3 - (a1,a2, var) - ); -} - -template -inline -detail::group1< detail::group4 > - group(T1 a1,T2 a2,T3 a3, Var& var) -{ - return detail::group1< detail::group4 > - ( detail::group4 - (a1,a2,a3, var) - ); -} - -template -inline -detail::group1< detail::group5 > - group(T1 a1,T2 a2,T3 a3,T4 a4, Var& var) -{ - return detail::group1< detail::group5 > - ( detail::group5 - (a1,a2,a3,a4, var) - ); -} - -template -inline -detail::group1< detail::group6 > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5, Var& var) -{ - return detail::group1< detail::group6 > - ( detail::group6 - (a1,a2,a3,a4,a5, var) - ); -} - -template -inline -detail::group1< detail::group7 > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6, Var& var) -{ - return detail::group1< detail::group7 > - ( detail::group7 - (a1,a2,a3,a4,a5,a6, var) - ); -} - -template -inline -detail::group1< detail::group8 > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7, Var& var) -{ - return detail::group1< detail::group8 > - ( detail::group8 - (a1,a2,a3,a4,a5,a6,a7, var) - ); -} - -template -inline -detail::group1< detail::group9 > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8, Var& var) -{ - return detail::group1< detail::group9 > - ( detail::group9 - (a1,a2,a3,a4,a5,a6,a7,a8, var) - ); -} - -template -inline -detail::group1< detail::group10 > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9, Var& var) -{ - return detail::group1< detail::group10 > - ( detail::group10 - (a1,a2,a3,a4,a5,a6,a7,a8,a9, var) - ); -} - - -#endif //end- #ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST - - -} // namespace io - -} // namespace boost - - -#endif // BOOST_FORMAT_GROUP_HPP diff --git a/src/boost/format/internals.hpp b/src/boost/format/internals.hpp deleted file mode 100644 index d25eb4c864c..00000000000 --- a/src/boost/format/internals.hpp +++ /dev/null @@ -1,167 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rdiger Loos's format class -// and Karl Nelson's ofstream - -// ---------------------------------------------------------------------------- -// internals.hpp : internal structs. included by format.hpp -// stream_format_state, and format_item -// ---------------------------------------------------------------------------- - - -#ifndef BOOST_FORMAT_INTERNALS_HPP -#define BOOST_FORMAT_INTERNALS_HPP - - -#include -#include - -namespace boost { -namespace io { -namespace detail { - - -// -------------- -// set of params that define the format state of a stream - -struct stream_format_state -{ - typedef std::ios basic_ios; - - std::streamsize width_; - std::streamsize precision_; - char fill_; - std::ios::fmtflags flags_; - - stream_format_state() : width_(-1), precision_(-1), fill_(0), flags_(std::ios::dec) {} - stream_format_state(basic_ios& os) {set_by_stream(os); } - - void apply_on(basic_ios & os) const; //- applies format_state to the stream - template void apply_manip(T manipulator) //- modifies state by applying manipulator. - { apply_manip_body( *this, manipulator) ; } - void reset(); //- sets to default state. - void set_by_stream(const basic_ios& os); //- sets to os's state. -}; - - - -// -------------- -// format_item : stores all parameters that can be defined by directives in the format-string - -struct format_item -{ - enum pad_values { zeropad = 1, spacepad =2, centered=4, tabulation = 8 }; - - enum arg_values { argN_no_posit = -1, // non-positional directive. argN will be set later. - argN_tabulation = -2, // tabulation directive. (no argument read) - argN_ignored = -3 // ignored directive. (no argument read) - }; - typedef BOOST_IO_STD ios basic_ios; - typedef detail::stream_format_state stream_format_state; - typedef std::string string_t; - typedef BOOST_IO_STD ostringstream internal_stream_t; - - - int argN_; //- argument number (starts at 0, eg : %1 => argN=0) - // negative values are used for items that don't process - // an argument - string_t res_; //- result of the formatting of this item - string_t appendix_; //- piece of string between this item and the next - - stream_format_state ref_state_;// set by parsing the format_string, is only affected by modify_item - stream_format_state state_; // always same as ref_state, _unless_ modified by manipulators 'group(..)' - - // non-stream format-state parameters - signed int truncate_; //- is >=0 for directives like %.5s (take 5 chars from the string) - unsigned int pad_scheme_; //- several possible padding schemes can mix. see pad_values - - format_item() : argN_(argN_no_posit), truncate_(-1), pad_scheme_(0) {} - - void compute_states(); // sets states according to truncate and pad_scheme. -}; - - - -// ----------------------------------------------------------- -// Definitions -// ----------------------------------------------------------- - -// --- stream_format_state:: ------------------------------------------- -inline -void stream_format_state::apply_on(basic_ios & os) const - // set the state of this stream according to our params -{ - if(width_ != -1) - os.width(width_); - if(precision_ != -1) - os.precision(precision_); - if(fill_ != 0) - os.fill(fill_); - os.flags(flags_); -} - -inline -void stream_format_state::set_by_stream(const basic_ios& os) - // set our params according to the state of this stream -{ - flags_ = os.flags(); - width_ = os.width(); - precision_ = os.precision(); - fill_ = os.fill(); -} - -template inline -void apply_manip_body( stream_format_state& self, - T manipulator) - // modify our params according to the manipulator -{ - BOOST_IO_STD stringstream ss; - self.apply_on( ss ); - ss << manipulator; - self.set_by_stream( ss ); -} - -inline -void stream_format_state::reset() - // set our params to standard's default state -{ - width_=-1; precision_=-1; fill_=0; - flags_ = std::ios::dec; -} - - -// --- format_items:: ------------------------------------------- -inline -void format_item::compute_states() - // reflect pad_scheme_ on state_ and ref_state_ - // because some pad_schemes has complex consequences on several state params. -{ - if(pad_scheme_ & zeropad) - { - if(ref_state_.flags_ & std::ios::left) - { - pad_scheme_ = pad_scheme_ & (~zeropad); // ignore zeropad in left alignment - } - else - { - ref_state_.fill_='0'; - ref_state_.flags_ |= std::ios::internal; - } - } - state_ = ref_state_; -} - - -} } } // namespaces boost :: io :: detail - - -#endif // BOOST_FORMAT_INTERNALS_HPP diff --git a/src/boost/format/internals_fwd.hpp b/src/boost/format/internals_fwd.hpp deleted file mode 100644 index a8ebf7c3abc..00000000000 --- a/src/boost/format/internals_fwd.hpp +++ /dev/null @@ -1,65 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rdiger Loos's format class -// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) - -// ------------------------------------------------------------------------------ -// internals_fwd.hpp : forward declarations, for internal headers -// ------------------------------------------------------------------------------ - -#ifndef BOOST_FORMAT_INTERNAL_FWD_HPP -#define BOOST_FORMAT_INTERNAL_FWD_HPP - -#include "boost/format/format_fwd.hpp" - - -namespace boost { -namespace io { - -namespace detail { - struct stream_format_state; - struct format_item; -} - - -namespace detail { - - // these functions were intended as methods, - // but MSVC have problems with template member functions : - - // defined in format_implementation.hpp : - template - basic_format& modify_item_body( basic_format& self, - int itemN, const T& manipulator); - - template - basic_format& bind_arg_body( basic_format& self, - int argN, const T& val); - - template - void apply_manip_body( stream_format_state& self, - T manipulator); - - // argument feeding (defined in feed_args.hpp ) : - template - void distribute(basic_format& self, T x); - - template - basic_format& feed(basic_format& self, T x); - -} // namespace detail - -} // namespace io -} // namespace boost - - -#endif // BOOST_FORMAT_INTERNAL_FWD_HPP diff --git a/src/boost/format/local.mk b/src/boost/format/local.mk deleted file mode 100644 index 3776eff382f..00000000000 --- a/src/boost/format/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -libraries += libformat - -libformat_NAME = libnixformat - -libformat_DIR := $(d) - -libformat_SOURCES := $(wildcard $(d)/*.cc) diff --git a/src/boost/format/macros_default.hpp b/src/boost/format/macros_default.hpp deleted file mode 100644 index 4fd84a163fb..00000000000 --- a/src/boost/format/macros_default.hpp +++ /dev/null @@ -1,48 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rdiger Loos's format class -// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) - -// ------------------------------------------------------------------------------ -// macros_default.hpp : configuration for the format library -// provides default values for the stl workaround macros -// ------------------------------------------------------------------------------ - -#ifndef BOOST_FORMAT_MACROS_DEFAULT_HPP -#define BOOST_FORMAT_MACROS_DEFAULT_HPP - -// *** This should go to "boost/config/suffix.hpp". - -#ifndef BOOST_IO_STD -# define BOOST_IO_STD std:: -#endif - -// **** Workaround for io streams, stlport and msvc. -#ifdef BOOST_IO_NEEDS_USING_DECLARATION -namespace boost { - using std::char_traits; - using std::basic_ostream; - using std::basic_ostringstream; - namespace io { - using std::basic_ostream; - namespace detail { - using std::basic_ios; - using std::basic_ostream; - using std::basic_ostringstream; - } - } -} -#endif - -// ------------------------------------------------------------------------------ - -#endif // BOOST_FORMAT_MACROS_DEFAULT_HPP diff --git a/src/boost/format/parsing.cc b/src/boost/format/parsing.cc deleted file mode 100644 index 34c36adeb73..00000000000 --- a/src/boost/format/parsing.cc +++ /dev/null @@ -1,454 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rudiger Loos's format class -// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) - -// ------------------------------------------------------------------------------ -// parsing.hpp : implementation of the parsing member functions -// ( parse, parse_printf_directive) -// ------------------------------------------------------------------------------ - - -#ifndef BOOST_FORMAT_PARSING_HPP -#define BOOST_FORMAT_PARSING_HPP - - -#include -#include -#include - - -namespace boost { -namespace io { -namespace detail { - - template inline - bool wrap_isdigit(char c, Stream &os) - { -#ifndef BOOST_NO_LOCALE_ISIDIGIT - return std::isdigit(c, os.rdbuf()->getloc() ); -# else - using namespace std; - return isdigit(c); -#endif - } //end- wrap_isdigit(..) - - template inline - Res str2int(const std::string& s, - std::string::size_type start, - BOOST_IO_STD ios &os, - const Res = Res(0) ) - // Input : char string, with starting index - // a basic_ios& merely to call its widen/narrow member function in the desired locale. - // Effects : reads s[start:] and converts digits into an integral n, of type Res - // Returns : n - { - Res n = 0; - while(start= buf.size() ) return; - if(buf[ *pos_p]=='*') { - ++ (*pos_p); - while (*pos_p < buf.size() && wrap_isdigit(buf[*pos_p],os)) ++(*pos_p); - if(buf[*pos_p]=='$') ++(*pos_p); - } - } - - - inline void maybe_throw_exception( unsigned char exceptions) - // auxiliary func called by parse_printf_directive - // for centralising error handling - // it either throws if user sets the corresponding flag, or does nothing. - { - if(exceptions & io::bad_format_string_bit) - boost::throw_exception(io::bad_format_string()); - } - - - - bool parse_printf_directive(const std::string & buf, - std::string::size_type * pos_p, - detail::format_item * fpar, - BOOST_IO_STD ios &os, - unsigned char exceptions) - // Input : a 'printf-directive' in the format-string, starting at buf[ *pos_p ] - // a basic_ios& merely to call its widen/narrow member function in the desired locale. - // a bitset'excpetions' telling whether to throw exceptions on errors. - // Returns : true if parse somehow succeeded (possibly ignoring errors if exceptions disabled) - // false if it failed so bad that the directive should be printed verbatim - // Effects : - *pos_p is incremented so that buf[*pos_p] is the first char after the directive - // - *fpar is set with the parameters read in the directive - { - typedef format_item format_item_t; - BOOST_ASSERT( pos_p != 0); - std::string::size_type &i1 = *pos_p, - i0; - fpar->argN_ = format_item_t::argN_no_posit; // if no positional-directive - - bool in_brackets=false; - if(buf[i1]=='|') - { - in_brackets=true; - if( ++i1 >= buf.size() ) { - maybe_throw_exception(exceptions); - return false; - } - } - - // the flag '0' would be picked as a digit for argument order, but here it's a flag : - if(buf[i1]=='0') - goto parse_flags; - - // handle argument order (%2$d) or possibly width specification: %2d - i0 = i1; // save position before digits - while (i1 < buf.size() && wrap_isdigit(buf[i1], os)) - ++i1; - if (i1!=i0) - { - if( i1 >= buf.size() ) { - maybe_throw_exception(exceptions); - return false; - } - int n=str2int(buf,i0, os, int(0) ); - - // %N% case : this is already the end of the directive - if( buf[i1] == '%' ) - { - fpar->argN_ = n-1; - ++i1; - if( in_brackets) - maybe_throw_exception(exceptions); - // but don't return. maybe "%" was used in lieu of '$', so we go on. - else return true; - } - - if ( buf[i1]=='$' ) - { - fpar->argN_ = n-1; - ++i1; - } - else - { - // non-positionnal directive - fpar->ref_state_.width_ = n; - fpar->argN_ = format_item_t::argN_no_posit; - goto parse_precision; - } - } - - parse_flags: - // handle flags - while ( i1 ref_state_.flags_ |= std::ios::left; - break; - case '=': - fpar->pad_scheme_ |= format_item_t::centered; - break; - case ' ': - fpar->pad_scheme_ |= format_item_t::spacepad; - break; - case '+': - fpar->ref_state_.flags_ |= std::ios::showpos; - break; - case '0': - fpar->pad_scheme_ |= format_item_t::zeropad; - // need to know alignment before really setting flags, - // so just add 'zeropad' flag for now, it will be processed later. - break; - case '#': - fpar->ref_state_.flags_ |= std::ios::showpoint | std::ios::showbase; - break; - default: - goto parse_width; - } - ++i1; - } // loop on flag. - if( i1>=buf.size()) { - maybe_throw_exception(exceptions); - return true; - } - - parse_width: - // handle width spec - skip_asterisk(buf, &i1, os); // skips 'asterisk fields' : *, or *N$ - i0 = i1; // save position before digits - while (i1ref_state_.width_ = str2int( buf,i0, os, std::streamsize(0) ); } - - parse_precision: - if( i1>=buf.size()) { - maybe_throw_exception(exceptions); - return true; - } - // handle precision spec - if (buf[i1]=='.') - { - ++i1; - skip_asterisk(buf, &i1, os); - i0 = i1; // save position before digits - while (i1ref_state_.precision_ = 0; - else - fpar->ref_state_.precision_ = str2int(buf,i0, os, std::streamsize(0) ); - } - - // handle formatting-type flags : - while( i1=buf.size()) { - maybe_throw_exception(exceptions); - return true; - } - - if( in_brackets && buf[i1]=='|' ) - { - ++i1; - return true; - } - switch (buf[i1]) - { - case 'X': - fpar->ref_state_.flags_ |= std::ios::uppercase; - case 'p': // pointer => set hex. - case 'x': - fpar->ref_state_.flags_ &= ~std::ios::basefield; - fpar->ref_state_.flags_ |= std::ios::hex; - break; - - case 'o': - fpar->ref_state_.flags_ &= ~std::ios::basefield; - fpar->ref_state_.flags_ |= std::ios::oct; - break; - - case 'E': - fpar->ref_state_.flags_ |= std::ios::uppercase; - case 'e': - fpar->ref_state_.flags_ &= ~std::ios::floatfield; - fpar->ref_state_.flags_ |= std::ios::scientific; - - fpar->ref_state_.flags_ &= ~std::ios::basefield; - fpar->ref_state_.flags_ |= std::ios::dec; - break; - - case 'f': - fpar->ref_state_.flags_ &= ~std::ios::floatfield; - fpar->ref_state_.flags_ |= std::ios::fixed; - case 'u': - case 'd': - case 'i': - fpar->ref_state_.flags_ &= ~std::ios::basefield; - fpar->ref_state_.flags_ |= std::ios::dec; - break; - - case 'T': - ++i1; - if( i1 >= buf.size()) - maybe_throw_exception(exceptions); - else - fpar->ref_state_.fill_ = buf[i1]; - fpar->pad_scheme_ |= format_item_t::tabulation; - fpar->argN_ = format_item_t::argN_tabulation; - break; - case 't': - fpar->ref_state_.fill_ = ' '; - fpar->pad_scheme_ |= format_item_t::tabulation; - fpar->argN_ = format_item_t::argN_tabulation; - break; - - case 'G': - fpar->ref_state_.flags_ |= std::ios::uppercase; - break; - case 'g': // 'g' conversion is default for floats. - fpar->ref_state_.flags_ &= ~std::ios::basefield; - fpar->ref_state_.flags_ |= std::ios::dec; - - // CLEAR all floatield flags, so stream will CHOOSE - fpar->ref_state_.flags_ &= ~std::ios::floatfield; - break; - - case 'C': - case 'c': - fpar->truncate_ = 1; - break; - case 'S': - case 's': - fpar->truncate_ = fpar->ref_state_.precision_; - fpar->ref_state_.precision_ = -1; - break; - case 'n' : - fpar->argN_ = format_item_t::argN_ignored; - break; - default: - maybe_throw_exception(exceptions); - } - ++i1; - - if( in_brackets ) - { - if( i1= buf.size() ) { - if(exceptions() & io::bad_format_string_bit) - boost::throw_exception(io::bad_format_string()); // must not end in "bla bla %" - else break; // stop there, ignore last '%' - } - if(buf[i1+1] == buf[i1] ) { i1+=2; continue; } // escaped "%%" / "##" - ++i1; - - // in case of %N% directives, dont count it double (wastes allocations..) : - while(i1 < buf.size() && io::detail::wrap_isdigit(buf[i1],oss_)) ++i1; - if( i1 < buf.size() && buf[i1] == arg_mark ) ++ i1; - - ++num_items; - } - items_.assign( num_items, format_item_t() ); - - // B: Now the real parsing of the format string : - num_items=0; - i1 = 0; - string_t::size_type i0 = i1; - bool special_things=false; - int cur_it=0; - while( (i1=buf.find(arg_mark,i1)) != string::npos ) - { - string_t & piece = (cur_it==0) ? prefix_ : items_[cur_it-1].appendix_; - - if( buf[i1+1] == buf[i1] ) // escaped mark, '%%' - { - piece += buf.substr(i0, i1-i0) + buf[i1]; - i1+=2; i0=i1; - continue; - } - BOOST_ASSERT( static_cast(cur_it) < items_.size() || cur_it==0); - - if(i1!=i0) piece += buf.substr(i0, i1-i0); - ++i1; - - bool parse_ok; - parse_ok = io::detail::parse_printf_directive(buf, &i1, &items_[cur_it], oss_, exceptions()); - if( ! parse_ok ) continue; // the directive will be printed verbatim - - i0=i1; - items_[cur_it].compute_states(); // process complex options, like zeropad, into stream params. - - int argN=items_[cur_it].argN_; - if(argN == format_item_t::argN_ignored) - continue; - if(argN ==format_item_t::argN_no_posit) - ordered_args=false; - else if(argN == format_item_t::argN_tabulation) special_things=true; - else if(argN > max_argN) max_argN = argN; - ++num_items; - ++cur_it; - } // loop on %'s - BOOST_ASSERT(cur_it == num_items); - - // store the final piece of string - string_t & piece = (cur_it==0) ? prefix_ : items_[cur_it-1].appendix_; - piece += buf.substr(i0); - - if( !ordered_args) - { - if(max_argN >= 0 ) // dont mix positional with non-positionnal directives - { - if(exceptions() & io::bad_format_string_bit) - boost::throw_exception(io::bad_format_string()); - // else do nothing. => positionnal arguments are processed as non-positionnal - } - // set things like it would have been with positional directives : - int non_ordered_items = 0; - for(int i=0; i< num_items; ++i) - if(items_[i].argN_ == format_item_t::argN_no_posit) - { - items_[i].argN_ = non_ordered_items; - ++non_ordered_items; - } - max_argN = non_ordered_items-1; - } - - // C: set some member data : - items_.resize(num_items); - - if(special_things) style_ |= special_needs; - num_args_ = max_argN + 1; - if(ordered_args) style_ |= ordered; - else style_ &= ~ordered; -} - -} // namespace boost - - -#endif // BOOST_FORMAT_PARSING_HPP diff --git a/src/boost/throw_exception.hpp b/src/boost/throw_exception.hpp deleted file mode 100644 index 07b4ae5ceae..00000000000 --- a/src/boost/throw_exception.hpp +++ /dev/null @@ -1,47 +0,0 @@ -#ifndef BOOST_THROW_EXCEPTION_HPP_INCLUDED -#define BOOST_THROW_EXCEPTION_HPP_INCLUDED - -// MS compatible compilers support #pragma once - -#if defined(_MSC_VER) && (_MSC_VER >= 1020) -# pragma once -#endif - -// -// boost/throw_exception.hpp -// -// Copyright (c) 2002 Peter Dimov and Multi Media Ltd. -// -// Permission to copy, use, modify, sell and distribute this software -// is granted provided this copyright notice appears in all copies. -// This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. -// -// http://www.boost.org/libs/utility/throw_exception.html -// - -//#include - -#ifdef BOOST_NO_EXCEPTIONS -# include -#endif - -namespace boost -{ - -#ifdef BOOST_NO_EXCEPTIONS - -void throw_exception(std::exception const & e); // user defined - -#else - -template void throw_exception(E const & e) -{ - throw e; -} - -#endif - -} // namespace boost - -#endif // #ifndef BOOST_THROW_EXCEPTION_HPP_INCLUDED diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 1daf0b80ba7..e071174960a 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -9,161 +9,104 @@ #include #endif +#include "machines.hh" #include "shared.hh" #include "pathlocks.hh" #include "globals.hh" -#include "serve-protocol.hh" #include "serialise.hh" #include "store-api.hh" #include "derivations.hh" +#include "local-store.hh" +#include "../nix/legacy.hh" using namespace nix; -using std::cerr; using std::cin; -static void handle_alarm(int sig) { +static void handleAlarm(int sig) { } -class machine { - const std::set supportedFeatures; - const std::set mandatoryFeatures; - -public: - const string hostName; - const std::vector systemTypes; - const string sshKey; - const unsigned long long maxJobs; - const unsigned long long speedFactor; - bool enabled; - - bool allSupported(const std::set & features) const { - return std::all_of(features.begin(), features.end(), - [&](const string & feature) { - return supportedFeatures.count(feature) || - mandatoryFeatures.count(feature); - }); - } - - bool mandatoryMet(const std::set & features) const { - return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(), - [&](const string & feature) { - return features.count(feature); - }); - } - - machine(decltype(hostName) hostName, - decltype(systemTypes) systemTypes, - decltype(sshKey) sshKey, - decltype(maxJobs) maxJobs, - decltype(speedFactor) speedFactor, - decltype(supportedFeatures) supportedFeatures, - decltype(mandatoryFeatures) mandatoryFeatures) : - supportedFeatures{std::move(supportedFeatures)}, - mandatoryFeatures{std::move(mandatoryFeatures)}, - hostName{std::move(hostName)}, - systemTypes{std::move(systemTypes)}, - sshKey{std::move(sshKey)}, - maxJobs{std::move(maxJobs)}, - speedFactor{speedFactor == 0 ? 1 : std::move(speedFactor)}, - enabled{true} {}; -};; - -static std::vector read_conf() +std::string escapeUri(std::string uri) { - auto conf = getEnv("NIX_REMOTE_SYSTEMS", SYSCONFDIR "/nix/machines"); - - auto machines = std::vector{}; - auto lines = std::vector{}; - try { - lines = tokenizeString>(readFile(conf), "\n"); - } catch (const SysError & e) { - if (e.errNo != ENOENT) - throw; - } - for (auto line : lines) { - chomp(line); - line.erase(std::find(line.begin(), line.end(), '#'), line.end()); - if (line.empty()) { - continue; - } - auto tokens = tokenizeString>(line); - auto sz = tokens.size(); - if (sz < 4) { - throw new FormatError(format("Bad machines.conf file %1%") - % conf); - } - machines.emplace_back(tokens[0], - tokenizeString>(tokens[1], ","), - tokens[2], - stoull(tokens[3]), - sz >= 5 ? stoull(tokens[4]) : 1LL, - sz >= 6 ? - tokenizeString>(tokens[5], ",") : - std::set{}, - sz >= 7 ? - tokenizeString>(tokens[6], ",") : - std::set{}); - } - return machines; + std::replace(uri.begin(), uri.end(), '/', '_'); + return uri; } static string currentLoad; -static int openSlotLock(const machine & m, unsigned long long slot) +static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot) { - std::ostringstream fn_stream(currentLoad, std::ios_base::ate | std::ios_base::out); - fn_stream << "/"; - for (auto t : m.systemTypes) { - fn_stream << t << "-"; - } - fn_stream << m.hostName << "-" << slot; - return openLockFile(fn_stream.str(), true); + return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); } -static char display_env[] = "DISPLAY="; -static char ssh_env[] = "SSH_ASKPASS="; +static bool allSupportedLocally(const std::set& requiredFeatures) { + for (auto & feature : requiredFeatures) + if (!settings.systemFeatures.get().count(feature)) return false; + return true; +} -int main (int argc, char * * argv) +static int _main(int argc, char * * argv) { - return handleExceptions(argv[0], [&]() { - initNix(); + { + logger = makeJSONLogger(*logger); + /* Ensure we don't get any SSH passphrase or host key popups. */ - if (putenv(display_env) == -1 || - putenv(ssh_env) == -1) { - throw SysError("Setting SSH env vars"); - } + unsetenv("DISPLAY"); + unsetenv("SSH_ASKPASS"); - if (argc != 4) { + if (argc != 2) throw UsageError("called without required arguments"); + + verbosity = (Verbosity) std::stoll(argv[1]); + + FdSource source(STDIN_FILENO); + + /* Read the parent's settings. */ + while (readInt(source)) { + auto name = readString(source); + auto value = readString(source); + settings.set(name, value); } - auto store = openStore(); + settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work - auto localSystem = argv[1]; - settings.maxSilentTime = stoull(string(argv[2])); - settings.buildTimeout = stoull(string(argv[3])); + initPlugins(); - currentLoad = getEnv("NIX_CURRENT_LOAD", "/run/nix/current-load"); + auto store = openStore().cast(); + + /* It would be more appropriate to use $XDG_RUNTIME_DIR, since + that gets cleared on reboot, but it wouldn't work on macOS. */ + currentLoad = store->stateDir + "/current-load"; std::shared_ptr sshStore; AutoCloseFD bestSlotLock; - auto machines = read_conf(); - string drvPath; - string hostName; - for (string line; getline(cin, line);) { - auto tokens = tokenizeString>(line); - auto sz = tokens.size(); - if (sz != 3 && sz != 4) { - throw Error(format("invalid build hook line %1%") % line); - } - auto amWilling = tokens[0] == "1"; - auto neededSystem = tokens[1]; - drvPath = tokens[2]; - auto requiredFeatures = sz == 3 ? - std::set{} : - tokenizeString>(tokens[3], ","); - auto canBuildLocally = amWilling && (neededSystem == localSystem); + auto machines = getMachines(); + debug("got %d remote builders", machines.size()); + + if (machines.empty()) { + std::cerr << "# decline-permanently\n"; + return 0; + } + + std::optional drvPath; + string storeUri; + + while (true) { + + try { + auto s = readString(source); + if (s != "try") return 0; + } catch (EndOfFile &) { return 0; } + + auto amWilling = readInt(source); + auto neededSystem = readString(source); + drvPath = store->parseStorePath(readString(source)); + auto requiredFeatures = readStrings>(source); + + auto canBuildLocally = amWilling + && ( neededSystem == settings.thisSystem + || settings.extraPlatforms.get().count(neededSystem) > 0) + && allSupportedLocally(requiredFeatures); /* Error ignored here, will be caught later */ mkdir(currentLoad.c_str(), 0777); @@ -175,9 +118,11 @@ int main (int argc, char * * argv) bool rightType = false; - machine * bestMachine = nullptr; + Machine * bestMachine = nullptr; unsigned long long bestLoad = 0; for (auto & m : machines) { + debug("considering building on remote machine '%s'", m.storeUri); + if (m.enabled && std::find(m.systemTypes.begin(), m.systemTypes.end(), neededSystem) != m.systemTypes.end() && @@ -187,7 +132,7 @@ int main (int argc, char * * argv) AutoCloseFD free; unsigned long long load = 0; for (unsigned long long slot = 0; slot < m.maxJobs; ++slot) { - AutoCloseFD slotLock = openSlotLock(m, slot); + auto slotLock = openSlotLock(m, slot); if (lockFile(slotLock.get(), ltWrite, false)) { if (!free) { free = std::move(slotLock); @@ -222,11 +167,10 @@ int main (int argc, char * * argv) } if (!bestSlotLock) { - if (rightType && !canBuildLocally) { - cerr << "# postpone\n"; - } else { - cerr << "# decline\n"; - } + if (rightType && !canBuildLocally) + std::cerr << "# postpone\n"; + else + std::cerr << "# decline\n"; break; } @@ -239,47 +183,88 @@ int main (int argc, char * * argv) lock = -1; try { - sshStore = openStore("ssh://" + bestMachine->hostName + "?key=" + bestMachine->sshKey); - hostName = bestMachine->hostName; + + Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri)); + + Store::Params storeParams; + if (hasPrefix(bestMachine->storeUri, "ssh://")) { + storeParams["max-connections"] = "1"; + storeParams["log-fd"] = "4"; + if (bestMachine->sshKey != "") + storeParams["ssh-key"] = bestMachine->sshKey; + } + + sshStore = openStore(bestMachine->storeUri, storeParams); + sshStore->connect(); + storeUri = bestMachine->storeUri; + } catch (std::exception & e) { - cerr << e.what() << '\n'; - cerr << "unable to open SSH connection to ‘" << bestMachine->hostName << "’, trying other available machines...\n"; + auto msg = chomp(drainFD(5, false)); + logError({ + .name = "Remote build", + .hint = hintfmt("cannot build on '%s': %s%s", + bestMachine->storeUri, e.what(), + (msg.empty() ? "" : ": " + msg)) + }); bestMachine->enabled = false; continue; } + goto connected; } } + connected: - cerr << "# accept\n"; - string line; - if (!getline(cin, line)) { - throw Error("hook caller didn't send inputs"); - } - auto inputs = tokenizeString>(line); - if (!getline(cin, line)) { - throw Error("hook caller didn't send outputs"); + close(5); + + std::cerr << "# accept\n" << storeUri << "\n"; + + auto inputs = readStrings(source); + auto outputs = readStrings(source); + + AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); + + { + Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri)); + + auto old = signal(SIGALRM, handleAlarm); + alarm(15 * 60); + if (!lockFile(uploadLock.get(), ltWrite, true)) + printError("somebody is hogging the upload lock for '%s', continuing..."); + alarm(0); + signal(SIGALRM, old); } - auto outputs = tokenizeString(line); - AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + hostName + ".upload-lock", true); - auto old = signal(SIGALRM, handle_alarm); - alarm(15 * 60); - if (!lockFile(uploadLock.get(), ltWrite, true)) { - cerr << "somebody is hogging the upload lock for " << hostName << ", continuing...\n"; + + auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute; + + { + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri)); + copyPaths(store, ref(sshStore), store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute); } - alarm(0); - signal(SIGALRM, old); - copyPaths(store, ref(sshStore), inputs); + uploadLock = -1; - cerr << "building ‘" << drvPath << "’ on ‘" << hostName << "’\n"; - sshStore->buildDerivation(drvPath, readDerivation(drvPath)); + auto drv = store->readDerivation(*drvPath); + drv.inputSrcs = store->parseStorePathSet(inputs); - std::remove_if(outputs.begin(), outputs.end(), [=](const Path & path) { return store->isValidPath(path); }); - if (!outputs.empty()) { - setenv("NIX_HELD_LOCKS", concatStringsSep(" ", outputs).c_str(), 1); /* FIXME: ugly */ - copyPaths(ref(sshStore), store, outputs); + auto result = sshStore->buildDerivation(*drvPath, drv); + + if (!result.success()) + throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); + + StorePathSet missing; + for (auto & path : outputs) + if (!store->isValidPath(store->parseStorePath(path))) missing.insert(store->parseStorePath(path)); + + if (!missing.empty()) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri)); + for (auto & i : missing) + store->locksHeld.insert(store->printStorePath(i)); /* FIXME: ugly */ + copyPaths(ref(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute); } - return; - }); + + return 0; + } } + +static RegisterLegacyCommand s1("build-remote", _main); diff --git a/src/build-remote/local.mk b/src/build-remote/local.mk deleted file mode 100644 index 05b8cb45143..00000000000 --- a/src/build-remote/local.mk +++ /dev/null @@ -1,11 +0,0 @@ -programs += build-remote - -build-remote_DIR := $(d) - -build-remote_INSTALL_DIR := $(libexecdir)/nix - -build-remote_LIBS = libmain libutil libformat libstore - -build-remote_SOURCES := $(d)/build-remote.cc - -build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\"" -Isrc/nix-store diff --git a/src/buildenv/buildenv.cc b/src/buildenv/buildenv.cc deleted file mode 100644 index f997096eddb..00000000000 --- a/src/buildenv/buildenv.cc +++ /dev/null @@ -1,186 +0,0 @@ -#include "shared.hh" -#include -#include -#include -#include - -using namespace nix; - -typedef std::map Priorities; - -static bool isDirectory (const Path & path) -{ - struct stat st; - if (stat(path.c_str(), &st) == -1) - throw SysError(format("getting status of ‘%1%’") % path); - return S_ISDIR(st.st_mode); -} - -static auto priorities = Priorities{}; - -static auto symlinks = 0; - -/* For each activated package, create symlinks */ -static void createLinks(const Path & srcDir, const Path & dstDir, int priority) -{ - auto srcFiles = readDirectory(srcDir); - for (const auto & ent : srcFiles) { - if (ent.name[0] == '.') - /* not matched by glob */ - continue; - const auto & srcFile = srcDir + "/" + ent.name; - auto dstFile = dstDir + "/" + ent.name; - - /* The files below are special-cased to that they don't show up - * in user profiles, either because they are useless, or - * because they would cauase pointless collisions (e.g., each - * Python package brings its own - * `$out/lib/pythonX.Y/site-packages/easy-install.pth'.) - */ - if (hasSuffix(srcFile, "/propagated-build-inputs") || - hasSuffix(srcFile, "/nix-support") || - hasSuffix(srcFile, "/perllocal.pod") || - hasSuffix(srcFile, "/info/dir") || - hasSuffix(srcFile, "/log")) { - continue; - } else if (isDirectory(srcFile)) { - struct stat dstSt; - auto res = lstat(dstFile.c_str(), &dstSt); - if (res == 0) { - if (S_ISDIR(dstSt.st_mode)) { - createLinks(srcFile, dstFile, priority); - continue; - } else if (S_ISLNK(dstSt.st_mode)) { - auto target = readLink(dstFile); - if (!isDirectory(target)) - throw Error(format("collision between ‘%1%’ and non-directory ‘%2%’") - % srcFile % target); - if (unlink(dstFile.c_str()) == -1) - throw SysError(format("unlinking ‘%1%’") % dstFile); - if (mkdir(dstFile.c_str(), 0755) == -1) - throw SysError(format("creating directory ‘%1%’")); - createLinks(target, dstFile, priorities[dstFile]); - createLinks(srcFile, dstFile, priority); - continue; - } - } else if (errno != ENOENT) - throw SysError(format("getting status of ‘%1%’") % dstFile); - } else { - struct stat dstSt; - auto res = lstat(dstFile.c_str(), &dstSt); - if (res == 0) { - if (S_ISLNK(dstSt.st_mode)) { - auto target = readLink(dstFile); - auto prevPriority = priorities[dstFile]; - if (prevPriority == priority) - throw Error(format( - "collision between ‘%1%’ and ‘%2%’; " - "use ‘nix-env --set-flag priority NUMBER PKGNAME’ " - "to change the priority of one of the conflicting packages" - ) % srcFile % target); - if (prevPriority < priority) - continue; - if (unlink(dstFile.c_str()) == -1) - throw SysError(format("unlinking ‘%1%’") % dstFile); - } - } else if (errno != ENOENT) - throw SysError(format("getting status of ‘%1%’") % dstFile); - } - createSymlink(srcFile, dstFile); - priorities[dstFile] = priority; - symlinks++; - } -} - -typedef std::set FileProp; - -static auto done = FileProp{}; -static auto postponed = FileProp{}; - -static auto out = string{}; - -static void addPkg(const Path & pkgDir, int priority) -{ - if (done.find(pkgDir) != done.end()) - return; - done.insert(pkgDir); - createLinks(pkgDir, out, priority); - auto propagatedFN = pkgDir + "/nix-support/propagated-user-env-packages"; - auto propagated = string{}; - { - AutoCloseFD fd = open(propagatedFN.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) { - if (errno == ENOENT) - return; - throw SysError(format("opening ‘%1%’") % propagatedFN); - } - propagated = readLine(fd.get()); - } - for (const auto & p : tokenizeString>(propagated, " ")) - if (done.find(p) == done.end()) - postponed.insert(p); -} - -struct Package { - Path path; - bool active; - int priority; - Package(Path path, bool active, int priority) : path{std::move(path)}, active{active}, priority{priority} {} -}; - -typedef std::vector Packages; - -int main(int argc, char ** argv) -{ - return handleExceptions(argv[0], [&]() { - initNix(); - out = getEnv("out"); - if (mkdir(out.c_str(), 0755) == -1) - throw SysError(format("creating %1%") % out); - - /* Convert the stuff we get from the environment back into a coherent - * data type. - */ - auto pkgs = Packages{}; - auto derivations = tokenizeString(getEnv("derivations")); - while (!derivations.empty()) { - /* !!! We're trusting the caller to structure derivations env var correctly */ - auto active = derivations.front(); derivations.pop_front(); - auto priority = stoi(derivations.front()); derivations.pop_front(); - auto outputs = stoi(derivations.front()); derivations.pop_front(); - for (auto n = 0; n < outputs; n++) { - auto path = derivations.front(); derivations.pop_front(); - pkgs.emplace_back(path, active != "false", priority); - } - } - - /* Symlink to the packages that have been installed explicitly by the - * user. Process in priority order to reduce unnecessary - * symlink/unlink steps. - */ - std::sort(pkgs.begin(), pkgs.end(), [](const Package & a, const Package & b) { - return a.priority < b.priority || (a.priority == b.priority && a.path < b.path); - }); - for (const auto & pkg : pkgs) - if (pkg.active) - addPkg(pkg.path, pkg.priority); - - /* Symlink to the packages that have been "propagated" by packages - * installed by the user (i.e., package X declares that it wants Y - * installed as well). We do these later because they have a lower - * priority in case of collisions. - */ - auto priorityCounter = 1000; - while (!postponed.empty()) { - auto pkgDirs = postponed; - postponed = FileProp{}; - for (const auto & pkgDir : pkgDirs) - addPkg(pkgDir, priorityCounter++); - } - - std::cerr << "created " << symlinks << " symlinks in user environment\n"; - - createSymlink(getEnv("manifest"), out + "/manifest.nix"); - }); -} - diff --git a/src/buildenv/local.mk b/src/buildenv/local.mk deleted file mode 100644 index 17ec13b235f..00000000000 --- a/src/buildenv/local.mk +++ /dev/null @@ -1,9 +0,0 @@ -programs += buildenv - -buildenv_DIR := $(d) - -buildenv_INSTALL_DIR := $(libexecdir)/nix - -buildenv_LIBS = libmain libstore libutil libformat - -buildenv_SOURCES := $(d)/buildenv.cc diff --git a/src/cpptoml/LICENSE b/src/cpptoml/LICENSE new file mode 100644 index 00000000000..8802c4fa5a3 --- /dev/null +++ b/src/cpptoml/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) 2014 Chase Geigle + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/src/cpptoml/cpptoml.h b/src/cpptoml/cpptoml.h new file mode 100644 index 00000000000..5a00da3b4cd --- /dev/null +++ b/src/cpptoml/cpptoml.h @@ -0,0 +1,3668 @@ +/** + * @file cpptoml.h + * @author Chase Geigle + * @date May 2013 + */ + +#ifndef CPPTOML_H +#define CPPTOML_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if __cplusplus > 201103L +#define CPPTOML_DEPRECATED(reason) [[deprecated(reason)]] +#elif defined(__clang__) +#define CPPTOML_DEPRECATED(reason) __attribute__((deprecated(reason))) +#elif defined(__GNUG__) +#define CPPTOML_DEPRECATED(reason) __attribute__((deprecated)) +#elif defined(_MSC_VER) +#if _MSC_VER < 1910 +#define CPPTOML_DEPRECATED(reason) __declspec(deprecated) +#else +#define CPPTOML_DEPRECATED(reason) [[deprecated(reason)]] +#endif +#endif + +namespace cpptoml +{ +class writer; // forward declaration +class base; // forward declaration +#if defined(CPPTOML_USE_MAP) +// a std::map will ensure that entries a sorted, albeit at a slight +// performance penalty relative to the (default) unordered_map +using string_to_base_map = std::map>; +#else +// by default an unordered_map is used for best performance as the +// toml specification does not require entries to be sorted +using string_to_base_map + = std::unordered_map>; +#endif + +// if defined, `base` will retain type information in form of an enum class +// such that static_cast can be used instead of dynamic_cast +// #define CPPTOML_NO_RTTI + +template +class option +{ + public: + option() : empty_{true} + { + // nothing + } + + option(T value) : empty_{false}, value_(std::move(value)) + { + // nothing + } + + explicit operator bool() const + { + return !empty_; + } + + const T& operator*() const + { + return value_; + } + + const T* operator->() const + { + return &value_; + } + + template + T value_or(U&& alternative) const + { + if (!empty_) + return value_; + return static_cast(std::forward(alternative)); + } + + private: + bool empty_; + T value_; +}; + +struct local_date +{ + int year = 0; + int month = 0; + int day = 0; +}; + +struct local_time +{ + int hour = 0; + int minute = 0; + int second = 0; + int microsecond = 0; +}; + +struct zone_offset +{ + int hour_offset = 0; + int minute_offset = 0; +}; + +struct local_datetime : local_date, local_time +{ +}; + +struct offset_datetime : local_datetime, zone_offset +{ + static inline struct offset_datetime from_zoned(const struct tm& t) + { + offset_datetime dt; + dt.year = t.tm_year + 1900; + dt.month = t.tm_mon + 1; + dt.day = t.tm_mday; + dt.hour = t.tm_hour; + dt.minute = t.tm_min; + dt.second = t.tm_sec; + + char buf[16]; + strftime(buf, 16, "%z", &t); + + int offset = std::stoi(buf); + dt.hour_offset = offset / 100; + dt.minute_offset = offset % 100; + return dt; + } + + CPPTOML_DEPRECATED("from_local has been renamed to from_zoned") + static inline struct offset_datetime from_local(const struct tm& t) + { + return from_zoned(t); + } + + static inline struct offset_datetime from_utc(const struct tm& t) + { + offset_datetime dt; + dt.year = t.tm_year + 1900; + dt.month = t.tm_mon + 1; + dt.day = t.tm_mday; + dt.hour = t.tm_hour; + dt.minute = t.tm_min; + dt.second = t.tm_sec; + return dt; + } +}; + +CPPTOML_DEPRECATED("datetime has been renamed to offset_datetime") +typedef offset_datetime datetime; + +class fill_guard +{ + public: + fill_guard(std::ostream& os) : os_(os), fill_{os.fill()} + { + // nothing + } + + ~fill_guard() + { + os_.fill(fill_); + } + + private: + std::ostream& os_; + std::ostream::char_type fill_; +}; + +inline std::ostream& operator<<(std::ostream& os, const local_date& dt) +{ + fill_guard g{os}; + os.fill('0'); + + using std::setw; + os << setw(4) << dt.year << "-" << setw(2) << dt.month << "-" << setw(2) + << dt.day; + + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const local_time& ltime) +{ + fill_guard g{os}; + os.fill('0'); + + using std::setw; + os << setw(2) << ltime.hour << ":" << setw(2) << ltime.minute << ":" + << setw(2) << ltime.second; + + if (ltime.microsecond > 0) + { + os << "."; + int power = 100000; + for (int curr_us = ltime.microsecond; curr_us; power /= 10) + { + auto num = curr_us / power; + os << num; + curr_us -= num * power; + } + } + + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const zone_offset& zo) +{ + fill_guard g{os}; + os.fill('0'); + + using std::setw; + + if (zo.hour_offset != 0 || zo.minute_offset != 0) + { + if (zo.hour_offset > 0) + { + os << "+"; + } + else + { + os << "-"; + } + os << setw(2) << std::abs(zo.hour_offset) << ":" << setw(2) + << std::abs(zo.minute_offset); + } + else + { + os << "Z"; + } + + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const local_datetime& dt) +{ + return os << static_cast(dt) << "T" + << static_cast(dt); +} + +inline std::ostream& operator<<(std::ostream& os, const offset_datetime& dt) +{ + return os << static_cast(dt) + << static_cast(dt); +} + +template +struct is_one_of; + +template +struct is_one_of : std::is_same +{ +}; + +template +struct is_one_of +{ + const static bool value + = std::is_same::value || is_one_of::value; +}; + +template +class value; + +template +struct valid_value + : is_one_of +{ +}; + +template +struct value_traits; + +template +struct valid_value_or_string_convertible +{ + + const static bool value = valid_value::type>::value + || std::is_convertible::value; +}; + +template +struct value_traits::value>::type> +{ + using value_type = typename std::conditional< + valid_value::type>::value, + typename std::decay::type, std::string>::type; + + using type = value; + + static value_type construct(T&& val) + { + return value_type(val); + } +}; + +template +struct value_traits< + T, + typename std::enable_if< + !valid_value_or_string_convertible::value + && std::is_floating_point::type>::value>::type> +{ + using value_type = typename std::decay::type; + + using type = value; + + static value_type construct(T&& val) + { + return value_type(val); + } +}; + +template +struct value_traits< + T, typename std::enable_if< + !valid_value_or_string_convertible::value + && !std::is_floating_point::type>::value + && std::is_signed::type>::value>::type> +{ + using value_type = int64_t; + + using type = value; + + static value_type construct(T&& val) + { + if (val < (std::numeric_limits::min)()) + throw std::underflow_error{"constructed value cannot be " + "represented by a 64-bit signed " + "integer"}; + + if (val > (std::numeric_limits::max)()) + throw std::overflow_error{"constructed value cannot be represented " + "by a 64-bit signed integer"}; + + return static_cast(val); + } +}; + +template +struct value_traits< + T, typename std::enable_if< + !valid_value_or_string_convertible::value + && std::is_unsigned::type>::value>::type> +{ + using value_type = int64_t; + + using type = value; + + static value_type construct(T&& val) + { + if (val > static_cast((std::numeric_limits::max)())) + throw std::overflow_error{"constructed value cannot be represented " + "by a 64-bit signed integer"}; + + return static_cast(val); + } +}; + +class array; +class table; +class table_array; + +template +struct array_of_trait +{ + using return_type = option>; +}; + +template <> +struct array_of_trait +{ + using return_type = option>>; +}; + +template +inline std::shared_ptr::type> make_value(T&& val); +inline std::shared_ptr make_array(); + +namespace detail +{ +template +inline std::shared_ptr make_element(); +} + +inline std::shared_ptr make_table(); +inline std::shared_ptr make_table_array(bool is_inline = false); + +#if defined(CPPTOML_NO_RTTI) +/// Base type used to store underlying data type explicitly if RTTI is disabled +enum class base_type +{ + NONE, + STRING, + LOCAL_TIME, + LOCAL_DATE, + LOCAL_DATETIME, + OFFSET_DATETIME, + INT, + FLOAT, + BOOL, + TABLE, + ARRAY, + TABLE_ARRAY +}; + +/// Type traits class to convert C++ types to enum member +template +struct base_type_traits; + +template <> +struct base_type_traits +{ + static const base_type type = base_type::STRING; +}; + +template <> +struct base_type_traits +{ + static const base_type type = base_type::LOCAL_TIME; +}; + +template <> +struct base_type_traits +{ + static const base_type type = base_type::LOCAL_DATE; +}; + +template <> +struct base_type_traits +{ + static const base_type type = base_type::LOCAL_DATETIME; +}; + +template <> +struct base_type_traits +{ + static const base_type type = base_type::OFFSET_DATETIME; +}; + +template <> +struct base_type_traits +{ + static const base_type type = base_type::INT; +}; + +template <> +struct base_type_traits +{ + static const base_type type = base_type::FLOAT; +}; + +template <> +struct base_type_traits +{ + static const base_type type = base_type::BOOL; +}; + +template <> +struct base_type_traits
+{ + static const base_type type = base_type::TABLE; +}; + +template <> +struct base_type_traits +{ + static const base_type type = base_type::ARRAY; +}; + +template <> +struct base_type_traits +{ + static const base_type type = base_type::TABLE_ARRAY; +}; +#endif + +/** + * A generic base TOML value used for type erasure. + */ +class base : public std::enable_shared_from_this +{ + public: + virtual ~base() = default; + + virtual std::shared_ptr clone() const = 0; + + /** + * Determines if the given TOML element is a value. + */ + virtual bool is_value() const + { + return false; + } + + /** + * Determines if the given TOML element is a table. + */ + virtual bool is_table() const + { + return false; + } + + /** + * Converts the TOML element into a table. + */ + std::shared_ptr
as_table() + { + if (is_table()) + return std::static_pointer_cast
(shared_from_this()); + return nullptr; + } + /** + * Determines if the TOML element is an array of "leaf" elements. + */ + virtual bool is_array() const + { + return false; + } + + /** + * Converts the TOML element to an array. + */ + std::shared_ptr as_array() + { + if (is_array()) + return std::static_pointer_cast(shared_from_this()); + return nullptr; + } + + /** + * Determines if the given TOML element is an array of tables. + */ + virtual bool is_table_array() const + { + return false; + } + + /** + * Converts the TOML element into a table array. + */ + std::shared_ptr as_table_array() + { + if (is_table_array()) + return std::static_pointer_cast(shared_from_this()); + return nullptr; + } + + /** + * Attempts to coerce the TOML element into a concrete TOML value + * of type T. + */ + template + std::shared_ptr> as(); + + template + std::shared_ptr> as() const; + + template + void accept(Visitor&& visitor, Args&&... args) const; + +#if defined(CPPTOML_NO_RTTI) + base_type type() const + { + return type_; + } + + protected: + base(const base_type t) : type_(t) + { + // nothing + } + + private: + const base_type type_ = base_type::NONE; + +#else + protected: + base() + { + // nothing + } +#endif +}; + +/** + * A concrete TOML value representing the "leaves" of the "tree". + */ +template +class value : public base +{ + struct make_shared_enabler + { + // nothing; this is a private key accessible only to friends + }; + + template + friend std::shared_ptr::type> + cpptoml::make_value(U&& val); + + public: + static_assert(valid_value::value, "invalid value type"); + + std::shared_ptr clone() const override; + + value(const make_shared_enabler&, const T& val) : value(val) + { + // nothing; note that users cannot actually invoke this function + // because they lack access to the make_shared_enabler. + } + + bool is_value() const override + { + return true; + } + + /** + * Gets the data associated with this value. + */ + T& get() + { + return data_; + } + + /** + * Gets the data associated with this value. Const version. + */ + const T& get() const + { + return data_; + } + + private: + T data_; + + /** + * Constructs a value from the given data. + */ +#if defined(CPPTOML_NO_RTTI) + value(const T& val) : base(base_type_traits::type), data_(val) + { + } +#else + value(const T& val) : data_(val) + { + } +#endif + + value(const value& val) = delete; + value& operator=(const value& val) = delete; +}; + +template +std::shared_ptr::type> make_value(T&& val) +{ + using value_type = typename value_traits::type; + using enabler = typename value_type::make_shared_enabler; + return std::make_shared( + enabler{}, value_traits::construct(std::forward(val))); +} + +template +inline std::shared_ptr> base::as() +{ +#if defined(CPPTOML_NO_RTTI) + if (type() == base_type_traits::type) + return std::static_pointer_cast>(shared_from_this()); + else + return nullptr; +#else + return std::dynamic_pointer_cast>(shared_from_this()); +#endif +} + +// special case value to allow getting an integer parameter as a +// double value +template <> +inline std::shared_ptr> base::as() +{ +#if defined(CPPTOML_NO_RTTI) + if (type() == base_type::FLOAT) + return std::static_pointer_cast>(shared_from_this()); + + if (type() == base_type::INT) + { + auto v = std::static_pointer_cast>(shared_from_this()); + return make_value(static_cast(v->get())); + } +#else + if (auto v = std::dynamic_pointer_cast>(shared_from_this())) + return v; + + if (auto v = std::dynamic_pointer_cast>(shared_from_this())) + return make_value(static_cast(v->get())); +#endif + + return nullptr; +} + +template +inline std::shared_ptr> base::as() const +{ +#if defined(CPPTOML_NO_RTTI) + if (type() == base_type_traits::type) + return std::static_pointer_cast>(shared_from_this()); + else + return nullptr; +#else + return std::dynamic_pointer_cast>(shared_from_this()); +#endif +} + +// special case value to allow getting an integer parameter as a +// double value +template <> +inline std::shared_ptr> base::as() const +{ +#if defined(CPPTOML_NO_RTTI) + if (type() == base_type::FLOAT) + return std::static_pointer_cast>( + shared_from_this()); + + if (type() == base_type::INT) + { + auto v = as(); + // the below has to be a non-const value due to a bug in + // libc++: https://llvm.org/bugs/show_bug.cgi?id=18843 + return make_value(static_cast(v->get())); + } +#else + if (auto v + = std::dynamic_pointer_cast>(shared_from_this())) + return v; + + if (auto v = as()) + { + // the below has to be a non-const value due to a bug in + // libc++: https://llvm.org/bugs/show_bug.cgi?id=18843 + return make_value(static_cast(v->get())); + } +#endif + + return nullptr; +} + +/** + * Exception class for array insertion errors. + */ +class array_exception : public std::runtime_error +{ + public: + array_exception(const std::string& err) : std::runtime_error{err} + { + } +}; + +class array : public base +{ + public: + friend std::shared_ptr make_array(); + + std::shared_ptr clone() const override; + + virtual bool is_array() const override + { + return true; + } + + using size_type = std::size_t; + + /** + * arrays can be iterated over + */ + using iterator = std::vector>::iterator; + + /** + * arrays can be iterated over. Const version. + */ + using const_iterator = std::vector>::const_iterator; + + iterator begin() + { + return values_.begin(); + } + + const_iterator begin() const + { + return values_.begin(); + } + + iterator end() + { + return values_.end(); + } + + const_iterator end() const + { + return values_.end(); + } + + /** + * Obtains the array (vector) of base values. + */ + std::vector>& get() + { + return values_; + } + + /** + * Obtains the array (vector) of base values. Const version. + */ + const std::vector>& get() const + { + return values_; + } + + std::shared_ptr at(size_t idx) const + { + return values_.at(idx); + } + + /** + * Obtains an array of values. Note that elements may be + * nullptr if they cannot be converted to a value. + */ + template + std::vector>> array_of() const + { + std::vector>> result(values_.size()); + + std::transform(values_.begin(), values_.end(), result.begin(), + [&](std::shared_ptr v) { return v->as(); }); + + return result; + } + + /** + * Obtains a option>. The option will be empty if the array + * contains values that are not of type T. + */ + template + inline typename array_of_trait::return_type get_array_of() const + { + std::vector result; + result.reserve(values_.size()); + + for (const auto& val : values_) + { + if (auto v = val->as()) + result.push_back(v->get()); + else + return {}; + } + + return {std::move(result)}; + } + + /** + * Obtains an array of arrays. Note that elements may be nullptr + * if they cannot be converted to a array. + */ + std::vector> nested_array() const + { + std::vector> result(values_.size()); + + std::transform(values_.begin(), values_.end(), result.begin(), + [&](std::shared_ptr v) -> std::shared_ptr { + if (v->is_array()) + return std::static_pointer_cast(v); + return std::shared_ptr{}; + }); + + return result; + } + + /** + * Add a value to the end of the array + */ + template + void push_back(const std::shared_ptr>& val) + { + if (values_.empty() || values_[0]->as()) + { + values_.push_back(val); + } + else + { + throw array_exception{"Arrays must be homogenous."}; + } + } + + /** + * Add an array to the end of the array + */ + void push_back(const std::shared_ptr& val) + { + if (values_.empty() || values_[0]->is_array()) + { + values_.push_back(val); + } + else + { + throw array_exception{"Arrays must be homogenous."}; + } + } + + /** + * Convenience function for adding a simple element to the end + * of the array. + */ + template + void push_back(T&& val, typename value_traits::type* = 0) + { + push_back(make_value(std::forward(val))); + } + + /** + * Insert a value into the array + */ + template + iterator insert(iterator position, const std::shared_ptr>& value) + { + if (values_.empty() || values_[0]->as()) + { + return values_.insert(position, value); + } + else + { + throw array_exception{"Arrays must be homogenous."}; + } + } + + /** + * Insert an array into the array + */ + iterator insert(iterator position, const std::shared_ptr& value) + { + if (values_.empty() || values_[0]->is_array()) + { + return values_.insert(position, value); + } + else + { + throw array_exception{"Arrays must be homogenous."}; + } + } + + /** + * Convenience function for inserting a simple element in the array + */ + template + iterator insert(iterator position, T&& val, + typename value_traits::type* = 0) + { + return insert(position, make_value(std::forward(val))); + } + + /** + * Erase an element from the array + */ + iterator erase(iterator position) + { + return values_.erase(position); + } + + /** + * Clear the array + */ + void clear() + { + values_.clear(); + } + + /** + * Reserve space for n values. + */ + void reserve(size_type n) + { + values_.reserve(n); + } + + private: +#if defined(CPPTOML_NO_RTTI) + array() : base(base_type::ARRAY) + { + // empty + } +#else + array() = default; +#endif + + template + array(InputIterator begin, InputIterator end) : values_{begin, end} + { + // nothing + } + + array(const array& obj) = delete; + array& operator=(const array& obj) = delete; + + std::vector> values_; +}; + +inline std::shared_ptr make_array() +{ + struct make_shared_enabler : public array + { + make_shared_enabler() + { + // nothing + } + }; + + return std::make_shared(); +} + +namespace detail +{ +template <> +inline std::shared_ptr make_element() +{ + return make_array(); +} +} // namespace detail + +/** + * Obtains a option>. The option will be empty if the array + * contains values that are not of type T. + */ +template <> +inline typename array_of_trait::return_type +array::get_array_of() const +{ + std::vector> result; + result.reserve(values_.size()); + + for (const auto& val : values_) + { + if (auto v = val->as_array()) + result.push_back(v); + else + return {}; + } + + return {std::move(result)}; +} + +class table; + +class table_array : public base +{ + friend class table; + friend std::shared_ptr make_table_array(bool); + + public: + std::shared_ptr clone() const override; + + using size_type = std::size_t; + + /** + * arrays can be iterated over + */ + using iterator = std::vector>::iterator; + + /** + * arrays can be iterated over. Const version. + */ + using const_iterator = std::vector>::const_iterator; + + iterator begin() + { + return array_.begin(); + } + + const_iterator begin() const + { + return array_.begin(); + } + + iterator end() + { + return array_.end(); + } + + const_iterator end() const + { + return array_.end(); + } + + virtual bool is_table_array() const override + { + return true; + } + + std::vector>& get() + { + return array_; + } + + const std::vector>& get() const + { + return array_; + } + + /** + * Add a table to the end of the array + */ + void push_back(const std::shared_ptr
& val) + { + array_.push_back(val); + } + + /** + * Insert a table into the array + */ + iterator insert(iterator position, const std::shared_ptr
& value) + { + return array_.insert(position, value); + } + + /** + * Erase an element from the array + */ + iterator erase(iterator position) + { + return array_.erase(position); + } + + /** + * Clear the array + */ + void clear() + { + array_.clear(); + } + + /** + * Reserve space for n tables. + */ + void reserve(size_type n) + { + array_.reserve(n); + } + + /** + * Whether or not the table array is declared inline. This mostly + * matters for parsing, where statically defined arrays cannot be + * appended to using the array-of-table syntax. + */ + bool is_inline() const + { + return is_inline_; + } + + private: +#if defined(CPPTOML_NO_RTTI) + table_array(bool is_inline = false) + : base(base_type::TABLE_ARRAY), is_inline_(is_inline) + { + // nothing + } +#else + table_array(bool is_inline = false) : is_inline_(is_inline) + { + // nothing + } +#endif + + table_array(const table_array& obj) = delete; + table_array& operator=(const table_array& rhs) = delete; + + std::vector> array_; + const bool is_inline_ = false; +}; + +inline std::shared_ptr make_table_array(bool is_inline) +{ + struct make_shared_enabler : public table_array + { + make_shared_enabler(bool mse_is_inline) : table_array(mse_is_inline) + { + // nothing + } + }; + + return std::make_shared(is_inline); +} + +namespace detail +{ +template <> +inline std::shared_ptr make_element() +{ + return make_table_array(true); +} +} // namespace detail + +// The below are overloads for fetching specific value types out of a value +// where special casting behavior (like bounds checking) is desired + +template +typename std::enable_if::value + && std::is_signed::value, + option>::type +get_impl(const std::shared_ptr& elem) +{ + if (auto v = elem->as()) + { + if (v->get() < (std::numeric_limits::min)()) + throw std::underflow_error{ + "T cannot represent the value requested in get"}; + + if (v->get() > (std::numeric_limits::max)()) + throw std::overflow_error{ + "T cannot represent the value requested in get"}; + + return {static_cast(v->get())}; + } + else + { + return {}; + } +} + +template +typename std::enable_if::value + && std::is_unsigned::value, + option>::type +get_impl(const std::shared_ptr& elem) +{ + if (auto v = elem->as()) + { + if (v->get() < 0) + throw std::underflow_error{"T cannot store negative value in get"}; + + if (static_cast(v->get()) > (std::numeric_limits::max)()) + throw std::overflow_error{ + "T cannot represent the value requested in get"}; + + return {static_cast(v->get())}; + } + else + { + return {}; + } +} + +template +typename std::enable_if::value + || std::is_same::value, + option>::type +get_impl(const std::shared_ptr& elem) +{ + if (auto v = elem->as()) + { + return {v->get()}; + } + else + { + return {}; + } +} + +/** + * Represents a TOML keytable. + */ +class table : public base +{ + public: + friend class table_array; + friend std::shared_ptr
make_table(); + + std::shared_ptr clone() const override; + + /** + * tables can be iterated over. + */ + using iterator = string_to_base_map::iterator; + + /** + * tables can be iterated over. Const version. + */ + using const_iterator = string_to_base_map::const_iterator; + + iterator begin() + { + return map_.begin(); + } + + const_iterator begin() const + { + return map_.begin(); + } + + iterator end() + { + return map_.end(); + } + + const_iterator end() const + { + return map_.end(); + } + + bool is_table() const override + { + return true; + } + + bool empty() const + { + return map_.empty(); + } + + /** + * Determines if this key table contains the given key. + */ + bool contains(const std::string& key) const + { + return map_.find(key) != map_.end(); + } + + /** + * Determines if this key table contains the given key. Will + * resolve "qualified keys". Qualified keys are the full access + * path separated with dots like "grandparent.parent.child". + */ + bool contains_qualified(const std::string& key) const + { + return resolve_qualified(key); + } + + /** + * Obtains the base for a given key. + * @throw std::out_of_range if the key does not exist + */ + std::shared_ptr get(const std::string& key) const + { + return map_.at(key); + } + + /** + * Obtains the base for a given key. Will resolve "qualified + * keys". Qualified keys are the full access path separated with + * dots like "grandparent.parent.child". + * + * @throw std::out_of_range if the key does not exist + */ + std::shared_ptr get_qualified(const std::string& key) const + { + std::shared_ptr p; + resolve_qualified(key, &p); + return p; + } + + /** + * Obtains a table for a given key, if possible. + */ + std::shared_ptr
get_table(const std::string& key) const + { + if (contains(key) && get(key)->is_table()) + return std::static_pointer_cast
(get(key)); + return nullptr; + } + + /** + * Obtains a table for a given key, if possible. Will resolve + * "qualified keys". + */ + std::shared_ptr
get_table_qualified(const std::string& key) const + { + if (contains_qualified(key) && get_qualified(key)->is_table()) + return std::static_pointer_cast
(get_qualified(key)); + return nullptr; + } + + /** + * Obtains an array for a given key. + */ + std::shared_ptr get_array(const std::string& key) const + { + if (!contains(key)) + return nullptr; + return get(key)->as_array(); + } + + /** + * Obtains an array for a given key. Will resolve "qualified keys". + */ + std::shared_ptr get_array_qualified(const std::string& key) const + { + if (!contains_qualified(key)) + return nullptr; + return get_qualified(key)->as_array(); + } + + /** + * Obtains a table_array for a given key, if possible. + */ + std::shared_ptr get_table_array(const std::string& key) const + { + if (!contains(key)) + return nullptr; + return get(key)->as_table_array(); + } + + /** + * Obtains a table_array for a given key, if possible. Will resolve + * "qualified keys". + */ + std::shared_ptr + get_table_array_qualified(const std::string& key) const + { + if (!contains_qualified(key)) + return nullptr; + return get_qualified(key)->as_table_array(); + } + + /** + * Helper function that attempts to get a value corresponding + * to the template parameter from a given key. + */ + template + option get_as(const std::string& key) const + { + try + { + return get_impl(get(key)); + } + catch (const std::out_of_range&) + { + return {}; + } + } + + /** + * Helper function that attempts to get a value corresponding + * to the template parameter from a given key. Will resolve "qualified + * keys". + */ + template + option get_qualified_as(const std::string& key) const + { + try + { + return get_impl(get_qualified(key)); + } + catch (const std::out_of_range&) + { + return {}; + } + } + + /** + * Helper function that attempts to get an array of values of a given + * type corresponding to the template parameter for a given key. + * + * If the key doesn't exist, doesn't exist as an array type, or one or + * more keys inside the array type are not of type T, an empty option + * is returned. Otherwise, an option containing a vector of the values + * is returned. + */ + template + inline typename array_of_trait::return_type + get_array_of(const std::string& key) const + { + if (auto v = get_array(key)) + { + std::vector result; + result.reserve(v->get().size()); + + for (const auto& b : v->get()) + { + if (auto val = b->as()) + result.push_back(val->get()); + else + return {}; + } + return {std::move(result)}; + } + + return {}; + } + + /** + * Helper function that attempts to get an array of values of a given + * type corresponding to the template parameter for a given key. Will + * resolve "qualified keys". + * + * If the key doesn't exist, doesn't exist as an array type, or one or + * more keys inside the array type are not of type T, an empty option + * is returned. Otherwise, an option containing a vector of the values + * is returned. + */ + template + inline typename array_of_trait::return_type + get_qualified_array_of(const std::string& key) const + { + if (auto v = get_array_qualified(key)) + { + std::vector result; + result.reserve(v->get().size()); + + for (const auto& b : v->get()) + { + if (auto val = b->as()) + result.push_back(val->get()); + else + return {}; + } + return {std::move(result)}; + } + + return {}; + } + + /** + * Adds an element to the keytable. + */ + void insert(const std::string& key, const std::shared_ptr& value) + { + map_[key] = value; + } + + /** + * Convenience shorthand for adding a simple element to the + * keytable. + */ + template + void insert(const std::string& key, T&& val, + typename value_traits::type* = 0) + { + insert(key, make_value(std::forward(val))); + } + + /** + * Removes an element from the table. + */ + void erase(const std::string& key) + { + map_.erase(key); + } + + private: +#if defined(CPPTOML_NO_RTTI) + table() : base(base_type::TABLE) + { + // nothing + } +#else + table() + { + // nothing + } +#endif + + table(const table& obj) = delete; + table& operator=(const table& rhs) = delete; + + std::vector split(const std::string& value, + char separator) const + { + std::vector result; + std::string::size_type p = 0; + std::string::size_type q; + while ((q = value.find(separator, p)) != std::string::npos) + { + result.emplace_back(value, p, q - p); + p = q + 1; + } + result.emplace_back(value, p); + return result; + } + + // If output parameter p is specified, fill it with the pointer to the + // specified entry and throw std::out_of_range if it couldn't be found. + // + // Otherwise, just return true if the entry could be found or false + // otherwise and do not throw. + bool resolve_qualified(const std::string& key, + std::shared_ptr* p = nullptr) const + { + auto parts = split(key, '.'); + auto last_key = parts.back(); + parts.pop_back(); + + auto cur_table = this; + for (const auto& part : parts) + { + cur_table = cur_table->get_table(part).get(); + if (!cur_table) + { + if (!p) + return false; + + throw std::out_of_range{key + " is not a valid key"}; + } + } + + if (!p) + return cur_table->map_.count(last_key) != 0; + + *p = cur_table->map_.at(last_key); + return true; + } + + string_to_base_map map_; +}; + +/** + * Helper function that attempts to get an array of arrays for a given + * key. + * + * If the key doesn't exist, doesn't exist as an array type, or one or + * more keys inside the array type are not of type T, an empty option + * is returned. Otherwise, an option containing a vector of the values + * is returned. + */ +template <> +inline typename array_of_trait::return_type +table::get_array_of(const std::string& key) const +{ + if (auto v = get_array(key)) + { + std::vector> result; + result.reserve(v->get().size()); + + for (const auto& b : v->get()) + { + if (auto val = b->as_array()) + result.push_back(val); + else + return {}; + } + + return {std::move(result)}; + } + + return {}; +} + +/** + * Helper function that attempts to get an array of arrays for a given + * key. Will resolve "qualified keys". + * + * If the key doesn't exist, doesn't exist as an array type, or one or + * more keys inside the array type are not of type T, an empty option + * is returned. Otherwise, an option containing a vector of the values + * is returned. + */ +template <> +inline typename array_of_trait::return_type +table::get_qualified_array_of(const std::string& key) const +{ + if (auto v = get_array_qualified(key)) + { + std::vector> result; + result.reserve(v->get().size()); + + for (const auto& b : v->get()) + { + if (auto val = b->as_array()) + result.push_back(val); + else + return {}; + } + + return {std::move(result)}; + } + + return {}; +} + +std::shared_ptr
make_table() +{ + struct make_shared_enabler : public table + { + make_shared_enabler() + { + // nothing + } + }; + + return std::make_shared(); +} + +namespace detail +{ +template <> +inline std::shared_ptr
make_element
() +{ + return make_table(); +} +} // namespace detail + +template +std::shared_ptr value::clone() const +{ + return make_value(data_); +} + +inline std::shared_ptr array::clone() const +{ + auto result = make_array(); + result->reserve(values_.size()); + for (const auto& ptr : values_) + result->values_.push_back(ptr->clone()); + return result; +} + +inline std::shared_ptr table_array::clone() const +{ + auto result = make_table_array(is_inline()); + result->reserve(array_.size()); + for (const auto& ptr : array_) + result->array_.push_back(ptr->clone()->as_table()); + return result; +} + +inline std::shared_ptr table::clone() const +{ + auto result = make_table(); + for (const auto& pr : map_) + result->insert(pr.first, pr.second->clone()); + return result; +} + +/** + * Exception class for all TOML parsing errors. + */ +class parse_exception : public std::runtime_error +{ + public: + parse_exception(const std::string& err) : std::runtime_error{err} + { + } + + parse_exception(const std::string& err, std::size_t line_number) + : std::runtime_error{err + " at line " + std::to_string(line_number)} + { + } +}; + +inline bool is_number(char c) +{ + return c >= '0' && c <= '9'; +} + +inline bool is_hex(char c) +{ + return is_number(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); +} + +/** + * Helper object for consuming expected characters. + */ +template +class consumer +{ + public: + consumer(std::string::iterator& it, const std::string::iterator& end, + OnError&& on_error) + : it_(it), end_(end), on_error_(std::forward(on_error)) + { + // nothing + } + + void operator()(char c) + { + if (it_ == end_ || *it_ != c) + on_error_(); + ++it_; + } + + template + void operator()(const char (&str)[N]) + { + std::for_each(std::begin(str), std::end(str) - 1, + [&](char c) { (*this)(c); }); + } + + void eat_or(char a, char b) + { + if (it_ == end_ || (*it_ != a && *it_ != b)) + on_error_(); + ++it_; + } + + int eat_digits(int len) + { + int val = 0; + for (int i = 0; i < len; ++i) + { + if (!is_number(*it_) || it_ == end_) + on_error_(); + val = 10 * val + (*it_++ - '0'); + } + return val; + } + + void error() + { + on_error_(); + } + + private: + std::string::iterator& it_; + const std::string::iterator& end_; + OnError on_error_; +}; + +template +consumer make_consumer(std::string::iterator& it, + const std::string::iterator& end, + OnError&& on_error) +{ + return consumer(it, end, std::forward(on_error)); +} + +// replacement for std::getline to handle incorrectly line-ended files +// https://stackoverflow.com/questions/6089231/getting-std-ifstream-to-handle-lf-cr-and-crlf +namespace detail +{ +inline std::istream& getline(std::istream& input, std::string& line) +{ + line.clear(); + + std::istream::sentry sentry{input, true}; + auto sb = input.rdbuf(); + + while (true) + { + auto c = sb->sbumpc(); + if (c == '\r') + { + if (sb->sgetc() == '\n') + c = sb->sbumpc(); + } + + if (c == '\n') + return input; + + if (c == std::istream::traits_type::eof()) + { + if (line.empty()) + input.setstate(std::ios::eofbit); + return input; + } + + line.push_back(static_cast(c)); + } +} +} // namespace detail + +/** + * The parser class. + */ +class parser +{ + public: + /** + * Parsers are constructed from streams. + */ + parser(std::istream& stream) : input_(stream) + { + // nothing + } + + parser& operator=(const parser& parser) = delete; + + /** + * Parses the stream this parser was created on until EOF. + * @throw parse_exception if there are errors in parsing + */ + std::shared_ptr
parse() + { + std::shared_ptr
root = make_table(); + + table* curr_table = root.get(); + + while (detail::getline(input_, line_)) + { + line_number_++; + auto it = line_.begin(); + auto end = line_.end(); + consume_whitespace(it, end); + if (it == end || *it == '#') + continue; + if (*it == '[') + { + curr_table = root.get(); + parse_table(it, end, curr_table); + } + else + { + parse_key_value(it, end, curr_table); + consume_whitespace(it, end); + eol_or_comment(it, end); + } + } + return root; + } + + private: +#if defined _MSC_VER + __declspec(noreturn) +#elif defined __GNUC__ + __attribute__((noreturn)) +#endif + void throw_parse_exception(const std::string& err) + { + throw parse_exception{err, line_number_}; + } + + void parse_table(std::string::iterator& it, + const std::string::iterator& end, table*& curr_table) + { + // remove the beginning keytable marker + ++it; + if (it == end) + throw_parse_exception("Unexpected end of table"); + if (*it == '[') + parse_table_array(it, end, curr_table); + else + parse_single_table(it, end, curr_table); + } + + void parse_single_table(std::string::iterator& it, + const std::string::iterator& end, + table*& curr_table) + { + if (it == end || *it == ']') + throw_parse_exception("Table name cannot be empty"); + + std::string full_table_name; + bool inserted = false; + + auto key_end = [](char c) { return c == ']'; }; + + auto key_part_handler = [&](const std::string& part) { + if (part.empty()) + throw_parse_exception("Empty component of table name"); + + if (!full_table_name.empty()) + full_table_name += '.'; + full_table_name += part; + + if (curr_table->contains(part)) + { +#if !defined(__PGI) + auto b = curr_table->get(part); +#else + // Workaround for PGI compiler + std::shared_ptr b = curr_table->get(part); +#endif + if (b->is_table()) + curr_table = static_cast(b.get()); + else if (b->is_table_array()) + curr_table = std::static_pointer_cast(b) + ->get() + .back() + .get(); + else + throw_parse_exception("Key " + full_table_name + + "already exists as a value"); + } + else + { + inserted = true; + curr_table->insert(part, make_table()); + curr_table = static_cast(curr_table->get(part).get()); + } + }; + + key_part_handler(parse_key(it, end, key_end, key_part_handler)); + + if (it == end) + throw_parse_exception( + "Unterminated table declaration; did you forget a ']'?"); + + if (*it != ']') + { + std::string errmsg{"Unexpected character in table definition: "}; + errmsg += '"'; + errmsg += *it; + errmsg += '"'; + throw_parse_exception(errmsg); + } + + // table already existed + if (!inserted) + { + auto is_value + = [](const std::pair&>& p) { + return p.second->is_value(); + }; + + // if there are any values, we can't add values to this table + // since it has already been defined. If there aren't any + // values, then it was implicitly created by something like + // [a.b] + if (curr_table->empty() + || std::any_of(curr_table->begin(), curr_table->end(), + is_value)) + { + throw_parse_exception("Redefinition of table " + + full_table_name); + } + } + + ++it; + consume_whitespace(it, end); + eol_or_comment(it, end); + } + + void parse_table_array(std::string::iterator& it, + const std::string::iterator& end, table*& curr_table) + { + ++it; + if (it == end || *it == ']') + throw_parse_exception("Table array name cannot be empty"); + + auto key_end = [](char c) { return c == ']'; }; + + std::string full_ta_name; + auto key_part_handler = [&](const std::string& part) { + if (part.empty()) + throw_parse_exception("Empty component of table array name"); + + if (!full_ta_name.empty()) + full_ta_name += '.'; + full_ta_name += part; + + if (curr_table->contains(part)) + { +#if !defined(__PGI) + auto b = curr_table->get(part); +#else + // Workaround for PGI compiler + std::shared_ptr b = curr_table->get(part); +#endif + + // if this is the end of the table array name, add an + // element to the table array that we just looked up, + // provided it was not declared inline + if (it != end && *it == ']') + { + if (!b->is_table_array()) + { + throw_parse_exception("Key " + full_ta_name + + " is not a table array"); + } + + auto v = b->as_table_array(); + + if (v->is_inline()) + { + throw_parse_exception("Static array " + full_ta_name + + " cannot be appended to"); + } + + v->get().push_back(make_table()); + curr_table = v->get().back().get(); + } + // otherwise, just keep traversing down the key name + else + { + if (b->is_table()) + curr_table = static_cast(b.get()); + else if (b->is_table_array()) + curr_table = std::static_pointer_cast(b) + ->get() + .back() + .get(); + else + throw_parse_exception("Key " + full_ta_name + + " already exists as a value"); + } + } + else + { + // if this is the end of the table array name, add a new + // table array and a new table inside that array for us to + // add keys to next + if (it != end && *it == ']') + { + curr_table->insert(part, make_table_array()); + auto arr = std::static_pointer_cast( + curr_table->get(part)); + arr->get().push_back(make_table()); + curr_table = arr->get().back().get(); + } + // otherwise, create the implicitly defined table and move + // down to it + else + { + curr_table->insert(part, make_table()); + curr_table + = static_cast(curr_table->get(part).get()); + } + } + }; + + key_part_handler(parse_key(it, end, key_end, key_part_handler)); + + // consume the last "]]" + auto eat = make_consumer(it, end, [this]() { + throw_parse_exception("Unterminated table array name"); + }); + eat(']'); + eat(']'); + + consume_whitespace(it, end); + eol_or_comment(it, end); + } + + void parse_key_value(std::string::iterator& it, std::string::iterator& end, + table* curr_table) + { + auto key_end = [](char c) { return c == '='; }; + + auto key_part_handler = [&](const std::string& part) { + // two cases: this key part exists already, in which case it must + // be a table, or it doesn't exist in which case we must create + // an implicitly defined table + if (curr_table->contains(part)) + { + auto val = curr_table->get(part); + if (val->is_table()) + { + curr_table = static_cast(val.get()); + } + else + { + throw_parse_exception("Key " + part + + " already exists as a value"); + } + } + else + { + auto newtable = make_table(); + curr_table->insert(part, newtable); + curr_table = newtable.get(); + } + }; + + auto key = parse_key(it, end, key_end, key_part_handler); + + if (curr_table->contains(key)) + throw_parse_exception("Key " + key + " already present"); + if (it == end || *it != '=') + throw_parse_exception("Value must follow after a '='"); + ++it; + consume_whitespace(it, end); + curr_table->insert(key, parse_value(it, end)); + consume_whitespace(it, end); + } + + template + std::string + parse_key(std::string::iterator& it, const std::string::iterator& end, + KeyEndFinder&& key_end, KeyPartHandler&& key_part_handler) + { + // parse the key as a series of one or more simple-keys joined with '.' + while (it != end && !key_end(*it)) + { + auto part = parse_simple_key(it, end); + consume_whitespace(it, end); + + if (it == end || key_end(*it)) + { + return part; + } + + if (*it != '.') + { + std::string errmsg{"Unexpected character in key: "}; + errmsg += '"'; + errmsg += *it; + errmsg += '"'; + throw_parse_exception(errmsg); + } + + key_part_handler(part); + + // consume the dot + ++it; + } + + throw_parse_exception("Unexpected end of key"); + } + + std::string parse_simple_key(std::string::iterator& it, + const std::string::iterator& end) + { + consume_whitespace(it, end); + + if (it == end) + throw_parse_exception("Unexpected end of key (blank key?)"); + + if (*it == '"' || *it == '\'') + { + return string_literal(it, end, *it); + } + else + { + auto bke = std::find_if(it, end, [](char c) { + return c == '.' || c == '=' || c == ']'; + }); + return parse_bare_key(it, bke); + } + } + + std::string parse_bare_key(std::string::iterator& it, + const std::string::iterator& end) + { + if (it == end) + { + throw_parse_exception("Bare key missing name"); + } + + auto key_end = end; + --key_end; + consume_backwards_whitespace(key_end, it); + ++key_end; + std::string key{it, key_end}; + + if (std::find(it, key_end, '#') != key_end) + { + throw_parse_exception("Bare key " + key + " cannot contain #"); + } + + if (std::find_if(it, key_end, + [](char c) { return c == ' ' || c == '\t'; }) + != key_end) + { + throw_parse_exception("Bare key " + key + + " cannot contain whitespace"); + } + + if (std::find_if(it, key_end, + [](char c) { return c == '[' || c == ']'; }) + != key_end) + { + throw_parse_exception("Bare key " + key + + " cannot contain '[' or ']'"); + } + + it = end; + return key; + } + + enum class parse_type + { + STRING = 1, + LOCAL_TIME, + LOCAL_DATE, + LOCAL_DATETIME, + OFFSET_DATETIME, + INT, + FLOAT, + BOOL, + ARRAY, + INLINE_TABLE + }; + + std::shared_ptr parse_value(std::string::iterator& it, + std::string::iterator& end) + { + parse_type type = determine_value_type(it, end); + switch (type) + { + case parse_type::STRING: + return parse_string(it, end); + case parse_type::LOCAL_TIME: + return parse_time(it, end); + case parse_type::LOCAL_DATE: + case parse_type::LOCAL_DATETIME: + case parse_type::OFFSET_DATETIME: + return parse_date(it, end); + case parse_type::INT: + case parse_type::FLOAT: + return parse_number(it, end); + case parse_type::BOOL: + return parse_bool(it, end); + case parse_type::ARRAY: + return parse_array(it, end); + case parse_type::INLINE_TABLE: + return parse_inline_table(it, end); + default: + throw_parse_exception("Failed to parse value"); + } + } + + parse_type determine_value_type(const std::string::iterator& it, + const std::string::iterator& end) + { + if (it == end) + { + throw_parse_exception("Failed to parse value type"); + } + if (*it == '"' || *it == '\'') + { + return parse_type::STRING; + } + else if (is_time(it, end)) + { + return parse_type::LOCAL_TIME; + } + else if (auto dtype = date_type(it, end)) + { + return *dtype; + } + else if (is_number(*it) || *it == '-' || *it == '+' + || (*it == 'i' && it + 1 != end && it[1] == 'n' + && it + 2 != end && it[2] == 'f') + || (*it == 'n' && it + 1 != end && it[1] == 'a' + && it + 2 != end && it[2] == 'n')) + { + return determine_number_type(it, end); + } + else if (*it == 't' || *it == 'f') + { + return parse_type::BOOL; + } + else if (*it == '[') + { + return parse_type::ARRAY; + } + else if (*it == '{') + { + return parse_type::INLINE_TABLE; + } + throw_parse_exception("Failed to parse value type"); + } + + parse_type determine_number_type(const std::string::iterator& it, + const std::string::iterator& end) + { + // determine if we are an integer or a float + auto check_it = it; + if (*check_it == '-' || *check_it == '+') + ++check_it; + + if (check_it == end) + throw_parse_exception("Malformed number"); + + if (*check_it == 'i' || *check_it == 'n') + return parse_type::FLOAT; + + while (check_it != end && is_number(*check_it)) + ++check_it; + if (check_it != end && *check_it == '.') + { + ++check_it; + while (check_it != end && is_number(*check_it)) + ++check_it; + return parse_type::FLOAT; + } + else + { + return parse_type::INT; + } + } + + std::shared_ptr> parse_string(std::string::iterator& it, + std::string::iterator& end) + { + auto delim = *it; + assert(delim == '"' || delim == '\''); + + // end is non-const here because we have to be able to potentially + // parse multiple lines in a string, not just one + auto check_it = it; + ++check_it; + if (check_it != end && *check_it == delim) + { + ++check_it; + if (check_it != end && *check_it == delim) + { + it = ++check_it; + return parse_multiline_string(it, end, delim); + } + } + return make_value(string_literal(it, end, delim)); + } + + std::shared_ptr> + parse_multiline_string(std::string::iterator& it, + std::string::iterator& end, char delim) + { + std::stringstream ss; + + auto is_ws = [](char c) { return c == ' ' || c == '\t'; }; + + bool consuming = false; + std::shared_ptr> ret; + + auto handle_line = [&](std::string::iterator& local_it, + std::string::iterator& local_end) { + if (consuming) + { + local_it = std::find_if_not(local_it, local_end, is_ws); + + // whole line is whitespace + if (local_it == local_end) + return; + } + + consuming = false; + + while (local_it != local_end) + { + // handle escaped characters + if (delim == '"' && *local_it == '\\') + { + auto check = local_it; + // check if this is an actual escape sequence or a + // whitespace escaping backslash + ++check; + consume_whitespace(check, local_end); + if (check == local_end) + { + consuming = true; + break; + } + + ss << parse_escape_code(local_it, local_end); + continue; + } + + // if we can end the string + if (std::distance(local_it, local_end) >= 3) + { + auto check = local_it; + // check for """ + if (*check++ == delim && *check++ == delim + && *check++ == delim) + { + local_it = check; + ret = make_value(ss.str()); + break; + } + } + + ss << *local_it++; + } + }; + + // handle the remainder of the current line + handle_line(it, end); + if (ret) + return ret; + + // start eating lines + while (detail::getline(input_, line_)) + { + ++line_number_; + + it = line_.begin(); + end = line_.end(); + + handle_line(it, end); + + if (ret) + return ret; + + if (!consuming) + ss << std::endl; + } + + throw_parse_exception("Unterminated multi-line basic string"); + } + + std::string string_literal(std::string::iterator& it, + const std::string::iterator& end, char delim) + { + ++it; + std::string val; + while (it != end) + { + // handle escaped characters + if (delim == '"' && *it == '\\') + { + val += parse_escape_code(it, end); + } + else if (*it == delim) + { + ++it; + consume_whitespace(it, end); + return val; + } + else + { + val += *it++; + } + } + throw_parse_exception("Unterminated string literal"); + } + + std::string parse_escape_code(std::string::iterator& it, + const std::string::iterator& end) + { + ++it; + if (it == end) + throw_parse_exception("Invalid escape sequence"); + char value; + if (*it == 'b') + { + value = '\b'; + } + else if (*it == 't') + { + value = '\t'; + } + else if (*it == 'n') + { + value = '\n'; + } + else if (*it == 'f') + { + value = '\f'; + } + else if (*it == 'r') + { + value = '\r'; + } + else if (*it == '"') + { + value = '"'; + } + else if (*it == '\\') + { + value = '\\'; + } + else if (*it == 'u' || *it == 'U') + { + return parse_unicode(it, end); + } + else + { + throw_parse_exception("Invalid escape sequence"); + } + ++it; + return std::string(1, value); + } + + std::string parse_unicode(std::string::iterator& it, + const std::string::iterator& end) + { + bool large = *it++ == 'U'; + auto codepoint = parse_hex(it, end, large ? 0x10000000 : 0x1000); + + if ((codepoint > 0xd7ff && codepoint < 0xe000) || codepoint > 0x10ffff) + { + throw_parse_exception( + "Unicode escape sequence is not a Unicode scalar value"); + } + + std::string result; + // See Table 3-6 of the Unicode standard + if (codepoint <= 0x7f) + { + // 1-byte codepoints: 00000000 0xxxxxxx + // repr: 0xxxxxxx + result += static_cast(codepoint & 0x7f); + } + else if (codepoint <= 0x7ff) + { + // 2-byte codepoints: 00000yyy yyxxxxxx + // repr: 110yyyyy 10xxxxxx + // + // 0x1f = 00011111 + // 0xc0 = 11000000 + // + result += static_cast(0xc0 | ((codepoint >> 6) & 0x1f)); + // + // 0x80 = 10000000 + // 0x3f = 00111111 + // + result += static_cast(0x80 | (codepoint & 0x3f)); + } + else if (codepoint <= 0xffff) + { + // 3-byte codepoints: zzzzyyyy yyxxxxxx + // repr: 1110zzzz 10yyyyyy 10xxxxxx + // + // 0xe0 = 11100000 + // 0x0f = 00001111 + // + result += static_cast(0xe0 | ((codepoint >> 12) & 0x0f)); + result += static_cast(0x80 | ((codepoint >> 6) & 0x1f)); + result += static_cast(0x80 | (codepoint & 0x3f)); + } + else + { + // 4-byte codepoints: 000uuuuu zzzzyyyy yyxxxxxx + // repr: 11110uuu 10uuzzzz 10yyyyyy 10xxxxxx + // + // 0xf0 = 11110000 + // 0x07 = 00000111 + // + result += static_cast(0xf0 | ((codepoint >> 18) & 0x07)); + result += static_cast(0x80 | ((codepoint >> 12) & 0x3f)); + result += static_cast(0x80 | ((codepoint >> 6) & 0x3f)); + result += static_cast(0x80 | (codepoint & 0x3f)); + } + return result; + } + + uint32_t parse_hex(std::string::iterator& it, + const std::string::iterator& end, uint32_t place) + { + uint32_t value = 0; + while (place > 0) + { + if (it == end) + throw_parse_exception("Unexpected end of unicode sequence"); + + if (!is_hex(*it)) + throw_parse_exception("Invalid unicode escape sequence"); + + value += place * hex_to_digit(*it++); + place /= 16; + } + return value; + } + + uint32_t hex_to_digit(char c) + { + if (is_number(c)) + return static_cast(c - '0'); + return 10 + + static_cast(c + - ((c >= 'a' && c <= 'f') ? 'a' : 'A')); + } + + std::shared_ptr parse_number(std::string::iterator& it, + const std::string::iterator& end) + { + auto check_it = it; + auto check_end = find_end_of_number(it, end); + + auto eat_sign = [&]() { + if (check_it != end && (*check_it == '-' || *check_it == '+')) + ++check_it; + }; + + auto check_no_leading_zero = [&]() { + if (check_it != end && *check_it == '0' && check_it + 1 != check_end + && check_it[1] != '.') + { + throw_parse_exception("Numbers may not have leading zeros"); + } + }; + + auto eat_digits = [&](bool (*check_char)(char)) { + auto beg = check_it; + while (check_it != end && check_char(*check_it)) + { + ++check_it; + if (check_it != end && *check_it == '_') + { + ++check_it; + if (check_it == end || !check_char(*check_it)) + throw_parse_exception("Malformed number"); + } + } + + if (check_it == beg) + throw_parse_exception("Malformed number"); + }; + + auto eat_hex = [&]() { eat_digits(&is_hex); }; + + auto eat_numbers = [&]() { eat_digits(&is_number); }; + + if (check_it != end && *check_it == '0' && check_it + 1 != check_end + && (check_it[1] == 'x' || check_it[1] == 'o' || check_it[1] == 'b')) + { + ++check_it; + char base = *check_it; + ++check_it; + if (base == 'x') + { + eat_hex(); + return parse_int(it, check_it, 16); + } + else if (base == 'o') + { + auto start = check_it; + eat_numbers(); + auto val = parse_int(start, check_it, 8, "0"); + it = start; + return val; + } + else // if (base == 'b') + { + auto start = check_it; + eat_numbers(); + auto val = parse_int(start, check_it, 2); + it = start; + return val; + } + } + + eat_sign(); + check_no_leading_zero(); + + if (check_it != end && check_it + 1 != end && check_it + 2 != end) + { + if (check_it[0] == 'i' && check_it[1] == 'n' && check_it[2] == 'f') + { + auto val = std::numeric_limits::infinity(); + if (*it == '-') + val = -val; + it = check_it + 3; + return make_value(val); + } + else if (check_it[0] == 'n' && check_it[1] == 'a' + && check_it[2] == 'n') + { + auto val = std::numeric_limits::quiet_NaN(); + if (*it == '-') + val = -val; + it = check_it + 3; + return make_value(val); + } + } + + eat_numbers(); + + if (check_it != end + && (*check_it == '.' || *check_it == 'e' || *check_it == 'E')) + { + bool is_exp = *check_it == 'e' || *check_it == 'E'; + + ++check_it; + if (check_it == end) + throw_parse_exception("Floats must have trailing digits"); + + auto eat_exp = [&]() { + eat_sign(); + check_no_leading_zero(); + eat_numbers(); + }; + + if (is_exp) + eat_exp(); + else + eat_numbers(); + + if (!is_exp && check_it != end + && (*check_it == 'e' || *check_it == 'E')) + { + ++check_it; + eat_exp(); + } + + return parse_float(it, check_it); + } + else + { + return parse_int(it, check_it); + } + } + + std::shared_ptr> parse_int(std::string::iterator& it, + const std::string::iterator& end, + int base = 10, + const char* prefix = "") + { + std::string v{it, end}; + v = prefix + v; + v.erase(std::remove(v.begin(), v.end(), '_'), v.end()); + it = end; + try + { + return make_value(std::stoll(v, nullptr, base)); + } + catch (const std::invalid_argument& ex) + { + throw_parse_exception("Malformed number (invalid argument: " + + std::string{ex.what()} + ")"); + } + catch (const std::out_of_range& ex) + { + throw_parse_exception("Malformed number (out of range: " + + std::string{ex.what()} + ")"); + } + } + + std::shared_ptr> parse_float(std::string::iterator& it, + const std::string::iterator& end) + { + std::string v{it, end}; + v.erase(std::remove(v.begin(), v.end(), '_'), v.end()); + it = end; + char decimal_point = std::localeconv()->decimal_point[0]; + std::replace(v.begin(), v.end(), '.', decimal_point); + try + { + return make_value(std::stod(v)); + } + catch (const std::invalid_argument& ex) + { + throw_parse_exception("Malformed number (invalid argument: " + + std::string{ex.what()} + ")"); + } + catch (const std::out_of_range& ex) + { + throw_parse_exception("Malformed number (out of range: " + + std::string{ex.what()} + ")"); + } + } + + std::shared_ptr> parse_bool(std::string::iterator& it, + const std::string::iterator& end) + { + auto eat = make_consumer(it, end, [this]() { + throw_parse_exception("Attempted to parse invalid boolean value"); + }); + + if (*it == 't') + { + eat("true"); + return make_value(true); + } + else if (*it == 'f') + { + eat("false"); + return make_value(false); + } + + eat.error(); + return nullptr; + } + + std::string::iterator find_end_of_number(std::string::iterator it, + std::string::iterator end) + { + auto ret = std::find_if(it, end, [](char c) { + return !is_number(c) && c != '_' && c != '.' && c != 'e' && c != 'E' + && c != '-' && c != '+' && c != 'x' && c != 'o' && c != 'b'; + }); + if (ret != end && ret + 1 != end && ret + 2 != end) + { + if ((ret[0] == 'i' && ret[1] == 'n' && ret[2] == 'f') + || (ret[0] == 'n' && ret[1] == 'a' && ret[2] == 'n')) + { + ret = ret + 3; + } + } + return ret; + } + + std::string::iterator find_end_of_date(std::string::iterator it, + std::string::iterator end) + { + auto end_of_date = std::find_if(it, end, [](char c) { + return !is_number(c) && c != '-'; + }); + if (end_of_date != end && *end_of_date == ' ' && end_of_date + 1 != end + && is_number(end_of_date[1])) + end_of_date++; + return std::find_if(end_of_date, end, [](char c) { + return !is_number(c) && c != 'T' && c != 'Z' && c != ':' + && c != '-' && c != '+' && c != '.'; + }); + } + + std::string::iterator find_end_of_time(std::string::iterator it, + std::string::iterator end) + { + return std::find_if(it, end, [](char c) { + return !is_number(c) && c != ':' && c != '.'; + }); + } + + local_time read_time(std::string::iterator& it, + const std::string::iterator& end) + { + auto time_end = find_end_of_time(it, end); + + auto eat = make_consumer( + it, time_end, [&]() { throw_parse_exception("Malformed time"); }); + + local_time ltime; + + ltime.hour = eat.eat_digits(2); + eat(':'); + ltime.minute = eat.eat_digits(2); + eat(':'); + ltime.second = eat.eat_digits(2); + + int power = 100000; + if (it != time_end && *it == '.') + { + ++it; + while (it != time_end && is_number(*it)) + { + ltime.microsecond += power * (*it++ - '0'); + power /= 10; + } + } + + if (it != time_end) + throw_parse_exception("Malformed time"); + + return ltime; + } + + std::shared_ptr> + parse_time(std::string::iterator& it, const std::string::iterator& end) + { + return make_value(read_time(it, end)); + } + + std::shared_ptr parse_date(std::string::iterator& it, + const std::string::iterator& end) + { + auto date_end = find_end_of_date(it, end); + + auto eat = make_consumer( + it, date_end, [&]() { throw_parse_exception("Malformed date"); }); + + local_date ldate; + ldate.year = eat.eat_digits(4); + eat('-'); + ldate.month = eat.eat_digits(2); + eat('-'); + ldate.day = eat.eat_digits(2); + + if (it == date_end) + return make_value(ldate); + + eat.eat_or('T', ' '); + + local_datetime ldt; + static_cast(ldt) = ldate; + static_cast(ldt) = read_time(it, date_end); + + if (it == date_end) + return make_value(ldt); + + offset_datetime dt; + static_cast(dt) = ldt; + + int hoff = 0; + int moff = 0; + if (*it == '+' || *it == '-') + { + auto plus = *it == '+'; + ++it; + + hoff = eat.eat_digits(2); + dt.hour_offset = (plus) ? hoff : -hoff; + eat(':'); + moff = eat.eat_digits(2); + dt.minute_offset = (plus) ? moff : -moff; + } + else if (*it == 'Z') + { + ++it; + } + + if (it != date_end) + throw_parse_exception("Malformed date"); + + return make_value(dt); + } + + std::shared_ptr parse_array(std::string::iterator& it, + std::string::iterator& end) + { + // this gets ugly because of the "homogeneity" restriction: + // arrays can either be of only one type, or contain arrays + // (each of those arrays could be of different types, though) + // + // because of the latter portion, we don't really have a choice + // but to represent them as arrays of base values... + ++it; + + // ugh---have to read the first value to determine array type... + skip_whitespace_and_comments(it, end); + + // edge case---empty array + if (*it == ']') + { + ++it; + return make_array(); + } + + auto val_end = std::find_if( + it, end, [](char c) { return c == ',' || c == ']' || c == '#'; }); + parse_type type = determine_value_type(it, val_end); + switch (type) + { + case parse_type::STRING: + return parse_value_array(it, end); + case parse_type::LOCAL_TIME: + return parse_value_array(it, end); + case parse_type::LOCAL_DATE: + return parse_value_array(it, end); + case parse_type::LOCAL_DATETIME: + return parse_value_array(it, end); + case parse_type::OFFSET_DATETIME: + return parse_value_array(it, end); + case parse_type::INT: + return parse_value_array(it, end); + case parse_type::FLOAT: + return parse_value_array(it, end); + case parse_type::BOOL: + return parse_value_array(it, end); + case parse_type::ARRAY: + return parse_object_array(&parser::parse_array, '[', it, + end); + case parse_type::INLINE_TABLE: + return parse_object_array( + &parser::parse_inline_table, '{', it, end); + default: + throw_parse_exception("Unable to parse array"); + } + } + + template + std::shared_ptr parse_value_array(std::string::iterator& it, + std::string::iterator& end) + { + auto arr = make_array(); + while (it != end && *it != ']') + { + auto val = parse_value(it, end); + if (auto v = val->as()) + arr->get().push_back(val); + else + throw_parse_exception("Arrays must be homogeneous"); + skip_whitespace_and_comments(it, end); + if (*it != ',') + break; + ++it; + skip_whitespace_and_comments(it, end); + } + if (it != end) + ++it; + return arr; + } + + template + std::shared_ptr parse_object_array(Function&& fun, char delim, + std::string::iterator& it, + std::string::iterator& end) + { + auto arr = detail::make_element(); + + while (it != end && *it != ']') + { + if (*it != delim) + throw_parse_exception("Unexpected character in array"); + + arr->get().push_back(((*this).*fun)(it, end)); + skip_whitespace_and_comments(it, end); + + if (it == end || *it != ',') + break; + + ++it; + skip_whitespace_and_comments(it, end); + } + + if (it == end || *it != ']') + throw_parse_exception("Unterminated array"); + + ++it; + return arr; + } + + std::shared_ptr
parse_inline_table(std::string::iterator& it, + std::string::iterator& end) + { + auto tbl = make_table(); + do + { + ++it; + if (it == end) + throw_parse_exception("Unterminated inline table"); + + consume_whitespace(it, end); + if (it != end && *it != '}') + { + parse_key_value(it, end, tbl.get()); + consume_whitespace(it, end); + } + } while (*it == ','); + + if (it == end || *it != '}') + throw_parse_exception("Unterminated inline table"); + + ++it; + consume_whitespace(it, end); + + return tbl; + } + + void skip_whitespace_and_comments(std::string::iterator& start, + std::string::iterator& end) + { + consume_whitespace(start, end); + while (start == end || *start == '#') + { + if (!detail::getline(input_, line_)) + throw_parse_exception("Unclosed array"); + line_number_++; + start = line_.begin(); + end = line_.end(); + consume_whitespace(start, end); + } + } + + void consume_whitespace(std::string::iterator& it, + const std::string::iterator& end) + { + while (it != end && (*it == ' ' || *it == '\t')) + ++it; + } + + void consume_backwards_whitespace(std::string::iterator& back, + const std::string::iterator& front) + { + while (back != front && (*back == ' ' || *back == '\t')) + --back; + } + + void eol_or_comment(const std::string::iterator& it, + const std::string::iterator& end) + { + if (it != end && *it != '#') + throw_parse_exception("Unidentified trailing character '" + + std::string{*it} + + "'---did you forget a '#'?"); + } + + bool is_time(const std::string::iterator& it, + const std::string::iterator& end) + { + auto time_end = find_end_of_time(it, end); + auto len = std::distance(it, time_end); + + if (len < 8) + return false; + + if (it[2] != ':' || it[5] != ':') + return false; + + if (len > 8) + return it[8] == '.' && len > 9; + + return true; + } + + option date_type(const std::string::iterator& it, + const std::string::iterator& end) + { + auto date_end = find_end_of_date(it, end); + auto len = std::distance(it, date_end); + + if (len < 10) + return {}; + + if (it[4] != '-' || it[7] != '-') + return {}; + + if (len >= 19 && (it[10] == 'T' || it[10] == ' ') + && is_time(it + 11, date_end)) + { + // datetime type + auto time_end = find_end_of_time(it + 11, date_end); + if (time_end == date_end) + return {parse_type::LOCAL_DATETIME}; + else + return {parse_type::OFFSET_DATETIME}; + } + else if (len == 10) + { + // just a regular date + return {parse_type::LOCAL_DATE}; + } + + return {}; + } + + std::istream& input_; + std::string line_; + std::size_t line_number_ = 0; +}; + +/** + * Utility function to parse a file as a TOML file. Returns the root table. + * Throws a parse_exception if the file cannot be opened. + */ +inline std::shared_ptr
parse_file(const std::string& filename) +{ +#if defined(BOOST_NOWIDE_FSTREAM_INCLUDED_HPP) + boost::nowide::ifstream file{filename.c_str()}; +#elif defined(NOWIDE_FSTREAM_INCLUDED_HPP) + nowide::ifstream file{filename.c_str()}; +#else + std::ifstream file{filename}; +#endif + if (!file.is_open()) + throw parse_exception{filename + " could not be opened for parsing"}; + parser p{file}; + return p.parse(); +} + +template +struct value_accept; + +template <> +struct value_accept<> +{ + template + static void accept(const base&, Visitor&&, Args&&...) + { + // nothing + } +}; + +template +struct value_accept +{ + template + static void accept(const base& b, Visitor&& visitor, Args&&... args) + { + if (auto v = b.as()) + { + visitor.visit(*v, std::forward(args)...); + } + else + { + value_accept::accept(b, std::forward(visitor), + std::forward(args)...); + } + } +}; + +/** + * base implementation of accept() that calls visitor.visit() on the concrete + * class. + */ +template +void base::accept(Visitor&& visitor, Args&&... args) const +{ + if (is_value()) + { + using value_acceptor + = value_accept; + value_acceptor::accept(*this, std::forward(visitor), + std::forward(args)...); + } + else if (is_table()) + { + visitor.visit(static_cast(*this), + std::forward(args)...); + } + else if (is_array()) + { + visitor.visit(static_cast(*this), + std::forward(args)...); + } + else if (is_table_array()) + { + visitor.visit(static_cast(*this), + std::forward(args)...); + } +} + +/** + * Writer that can be passed to accept() functions of cpptoml objects and + * will output valid TOML to a stream. + */ +class toml_writer +{ + public: + /** + * Construct a toml_writer that will write to the given stream + */ + toml_writer(std::ostream& s, const std::string& indent_space = "\t") + : stream_(s), indent_(indent_space), has_naked_endline_(false) + { + // nothing + } + + public: + /** + * Output a base value of the TOML tree. + */ + template + void visit(const value& v, bool = false) + { + write(v); + } + + /** + * Output a table element of the TOML tree + */ + void visit(const table& t, bool in_array = false) + { + write_table_header(in_array); + std::vector values; + std::vector tables; + + for (const auto& i : t) + { + if (i.second->is_table() || i.second->is_table_array()) + { + tables.push_back(i.first); + } + else + { + values.push_back(i.first); + } + } + + for (unsigned int i = 0; i < values.size(); ++i) + { + path_.push_back(values[i]); + + if (i > 0) + endline(); + + write_table_item_header(*t.get(values[i])); + t.get(values[i])->accept(*this, false); + path_.pop_back(); + } + + for (unsigned int i = 0; i < tables.size(); ++i) + { + path_.push_back(tables[i]); + + if (values.size() > 0 || i > 0) + endline(); + + write_table_item_header(*t.get(tables[i])); + t.get(tables[i])->accept(*this, false); + path_.pop_back(); + } + + endline(); + } + + /** + * Output an array element of the TOML tree + */ + void visit(const array& a, bool = false) + { + write("["); + + for (unsigned int i = 0; i < a.get().size(); ++i) + { + if (i > 0) + write(", "); + + if (a.get()[i]->is_array()) + { + a.get()[i]->as_array()->accept(*this, true); + } + else + { + a.get()[i]->accept(*this, true); + } + } + + write("]"); + } + + /** + * Output a table_array element of the TOML tree + */ + void visit(const table_array& t, bool = false) + { + for (unsigned int j = 0; j < t.get().size(); ++j) + { + if (j > 0) + endline(); + + t.get()[j]->accept(*this, true); + } + + endline(); + } + + /** + * Escape a string for output. + */ + static std::string escape_string(const std::string& str) + { + std::string res; + for (auto it = str.begin(); it != str.end(); ++it) + { + if (*it == '\b') + { + res += "\\b"; + } + else if (*it == '\t') + { + res += "\\t"; + } + else if (*it == '\n') + { + res += "\\n"; + } + else if (*it == '\f') + { + res += "\\f"; + } + else if (*it == '\r') + { + res += "\\r"; + } + else if (*it == '"') + { + res += "\\\""; + } + else if (*it == '\\') + { + res += "\\\\"; + } + else if (static_cast(*it) <= UINT32_C(0x001f)) + { + res += "\\u"; + std::stringstream ss; + ss << std::hex << static_cast(*it); + res += ss.str(); + } + else + { + res += *it; + } + } + return res; + } + + protected: + /** + * Write out a string. + */ + void write(const value& v) + { + write("\""); + write(escape_string(v.get())); + write("\""); + } + + /** + * Write out a double. + */ + void write(const value& v) + { + std::stringstream ss; + ss << std::showpoint + << std::setprecision(std::numeric_limits::max_digits10) + << v.get(); + + auto double_str = ss.str(); + auto pos = double_str.find("e0"); + if (pos != std::string::npos) + double_str.replace(pos, 2, "e"); + pos = double_str.find("e-0"); + if (pos != std::string::npos) + double_str.replace(pos, 3, "e-"); + + stream_ << double_str; + has_naked_endline_ = false; + } + + /** + * Write out an integer, local_date, local_time, local_datetime, or + * offset_datetime. + */ + template + typename std::enable_if< + is_one_of::value>::type + write(const value& v) + { + write(v.get()); + } + + /** + * Write out a boolean. + */ + void write(const value& v) + { + write((v.get() ? "true" : "false")); + } + + /** + * Write out the header of a table. + */ + void write_table_header(bool in_array = false) + { + if (!path_.empty()) + { + indent(); + + write("["); + + if (in_array) + { + write("["); + } + + for (unsigned int i = 0; i < path_.size(); ++i) + { + if (i > 0) + { + write("."); + } + + if (path_[i].find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcde" + "fghijklmnopqrstuvwxyz0123456789" + "_-") + == std::string::npos) + { + write(path_[i]); + } + else + { + write("\""); + write(escape_string(path_[i])); + write("\""); + } + } + + if (in_array) + { + write("]"); + } + + write("]"); + endline(); + } + } + + /** + * Write out the identifier for an item in a table. + */ + void write_table_item_header(const base& b) + { + if (!b.is_table() && !b.is_table_array()) + { + indent(); + + if (path_.back().find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcde" + "fghijklmnopqrstuvwxyz0123456789" + "_-") + == std::string::npos) + { + write(path_.back()); + } + else + { + write("\""); + write(escape_string(path_.back())); + write("\""); + } + + write(" = "); + } + } + + private: + /** + * Indent the proper number of tabs given the size of + * the path. + */ + void indent() + { + for (std::size_t i = 1; i < path_.size(); ++i) + write(indent_); + } + + /** + * Write a value out to the stream. + */ + template + void write(const T& v) + { + stream_ << v; + has_naked_endline_ = false; + } + + /** + * Write an endline out to the stream + */ + void endline() + { + if (!has_naked_endline_) + { + stream_ << "\n"; + has_naked_endline_ = true; + } + } + + private: + std::ostream& stream_; + const std::string indent_; + std::vector path_; + bool has_naked_endline_; +}; + +inline std::ostream& operator<<(std::ostream& stream, const base& b) +{ + toml_writer writer{stream}; + b.accept(writer); + return stream; +} + +template +std::ostream& operator<<(std::ostream& stream, const value& v) +{ + toml_writer writer{stream}; + v.accept(writer); + return stream; +} + +inline std::ostream& operator<<(std::ostream& stream, const table& t) +{ + toml_writer writer{stream}; + t.accept(writer); + return stream; +} + +inline std::ostream& operator<<(std::ostream& stream, const table_array& t) +{ + toml_writer writer{stream}; + t.accept(writer); + return stream; +} + +inline std::ostream& operator<<(std::ostream& stream, const array& a) +{ + toml_writer writer{stream}; + a.accept(writer); + return stream; +} +} // namespace cpptoml +#endif // CPPTOML_H diff --git a/src/download-via-ssh/download-via-ssh.cc b/src/download-via-ssh/download-via-ssh.cc deleted file mode 100644 index ff28a60ff4b..00000000000 --- a/src/download-via-ssh/download-via-ssh.cc +++ /dev/null @@ -1,141 +0,0 @@ -#include "shared.hh" -#include "util.hh" -#include "serialise.hh" -#include "archive.hh" -#include "affinity.hh" -#include "globals.hh" -#include "serve-protocol.hh" -#include "worker-protocol.hh" -#include "store-api.hh" - -#include -#include -#include - -using namespace nix; - -// !!! TODO: -// * Respect more than the first host -// * use a database -// * show progress - - -static std::pair connect(const string & conn) -{ - Pipe to, from; - to.create(); - from.create(); - startProcess([&]() { - if (dup2(to.readSide, STDIN_FILENO) == -1) - throw SysError("dupping stdin"); - if (dup2(from.writeSide, STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - execlp("ssh", "ssh", "-x", "-T", conn.c_str(), "nix-store --serve", NULL); - throw SysError("executing ssh"); - }); - // If child exits unexpectedly, we'll EPIPE or EOF early. - // If we exit unexpectedly, child will EPIPE or EOF early. - // So no need to keep track of it. - - return std::pair(to.writeSide.borrow(), from.readSide.borrow()); -} - - -static void substitute(std::pair & pipes, Path storePath, Path destPath) -{ - pipes.first << cmdDumpStorePath << storePath; - pipes.first.flush(); - restorePath(destPath, pipes.second); - std::cout << std::endl; -} - - -static void query(std::pair & pipes) -{ - for (string line; getline(std::cin, line);) { - Strings tokenized = tokenizeString(line); - string cmd = tokenized.front(); - tokenized.pop_front(); - if (cmd == "have") { - pipes.first - << cmdQueryValidPaths - << 0 // don't lock - << 0 // don't substitute - << tokenized; - pipes.first.flush(); - PathSet paths = readStrings(pipes.second); - for (auto & i : paths) - std::cout << i << std::endl; - } else if (cmd == "info") { - pipes.first << cmdQueryPathInfos << tokenized; - pipes.first.flush(); - while (1) { - Path path = readString(pipes.second); - if (path.empty()) break; - assertStorePath(path); - std::cout << path << std::endl; - string deriver = readString(pipes.second); - if (!deriver.empty()) assertStorePath(deriver); - std::cout << deriver << std::endl; - PathSet references = readStorePaths(pipes.second); - std::cout << references.size() << std::endl; - for (auto & i : references) - std::cout << i << std::endl; - std::cout << readLongLong(pipes.second) << std::endl; - std::cout << readLongLong(pipes.second) << std::endl; - } - } else - throw Error(format("unknown substituter query ‘%1%’") % cmd); - std::cout << std::endl; - } -} - - -int main(int argc, char * * argv) -{ - return handleExceptions(argv[0], [&]() { - if (argc < 2) - throw UsageError("download-via-ssh requires an argument"); - - initNix(); - - settings.update(); - - if (settings.sshSubstituterHosts.empty()) - return; - - std::cout << std::endl; - - /* Pass on the location of the daemon client's SSH - authentication socket. */ - string sshAuthSock = settings.get("ssh-auth-sock", string("")); - if (sshAuthSock != "") setenv("SSH_AUTH_SOCK", sshAuthSock.c_str(), 1); - - string host = settings.sshSubstituterHosts.front(); - std::pair pipes = connect(host); - - /* Exchange the greeting */ - pipes.first << SERVE_MAGIC_1; - pipes.first.flush(); - unsigned int magic = readInt(pipes.second); - if (magic != SERVE_MAGIC_2) - throw Error("protocol mismatch"); - readInt(pipes.second); // Server version, unused for now - pipes.first << SERVE_PROTOCOL_VERSION; - pipes.first.flush(); - - string arg = argv[1]; - if (arg == "--query") - query(pipes); - else if (arg == "--substitute") { - if (argc != 4) - throw UsageError("download-via-ssh: --substitute takes exactly two arguments"); - Path storePath = argv[2]; - Path destPath = argv[3]; - printError(format("downloading ‘%1%’ via SSH from ‘%2%’...") % storePath % host); - substitute(pipes, storePath, destPath); - } - else - throw UsageError(format("download-via-ssh: unknown command ‘%1%’") % arg); - }); -} diff --git a/src/download-via-ssh/local.mk b/src/download-via-ssh/local.mk deleted file mode 100644 index 80f4c385acb..00000000000 --- a/src/download-via-ssh/local.mk +++ /dev/null @@ -1,11 +0,0 @@ -programs += download-via-ssh - -download-via-ssh_DIR := $(d) - -download-via-ssh_SOURCES := $(d)/download-via-ssh.cc - -download-via-ssh_INSTALL_DIR := $(libexecdir)/nix/substituters - -download-via-ssh_CXXFLAGS = -Isrc/nix-store - -download-via-ssh_LIBS = libmain libstore libutil libformat diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 55379f94b18..8980bc09d55 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -19,7 +19,7 @@ static Strings parseAttrPath(const string & s) ++i; while (1) { if (i == s.end()) - throw Error(format("missing closing quote in selection path ‘%1%’") % s); + throw Error("missing closing quote in selection path '%1%'", s); if (*i == '"') break; cur.push_back(*i++); } @@ -32,15 +32,13 @@ static Strings parseAttrPath(const string & s) } -Value * findAlongAttrPath(EvalState & state, const string & attrPath, +std::pair findAlongAttrPath(EvalState & state, const string & attrPath, Bindings & autoArgs, Value & vIn) { Strings tokens = parseAttrPath(attrPath); - Error attrError = - Error(format("attribute selection path ‘%1%’ does not match expression") % attrPath); - Value * v = &vIn; + Pos pos = noPos; for (auto & attr : tokens) { @@ -62,34 +60,68 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath, if (v->type != tAttrs) throw TypeError( - format("the expression selected by the selection path ‘%1%’ should be a set but is %2%") - % attrPath % showType(*v)); - + "the expression selected by the selection path '%1%' should be a set but is %2%", + attrPath, + showType(*v)); if (attr.empty()) - throw Error(format("empty attribute name in selection path ‘%1%’") % attrPath); + throw Error("empty attribute name in selection path '%1%'", attrPath); Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); if (a == v->attrs->end()) - throw Error(format("attribute ‘%1%’ in selection path ‘%2%’ not found") % attr % attrPath); + throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath); v = &*a->value; + pos = *a->pos; } else if (apType == apIndex) { if (!v->isList()) throw TypeError( - format("the expression selected by the selection path ‘%1%’ should be a list but is %2%") - % attrPath % showType(*v)); - + "the expression selected by the selection path '%1%' should be a list but is %2%", + attrPath, + showType(*v)); if (attrIndex >= v->listSize()) - throw Error(format("list index %1% in selection path ‘%2%’ is out of range") % attrIndex % attrPath); + throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath); v = v->listElems()[attrIndex]; + pos = noPos; } } - return v; + return {v, pos}; +} + + +Pos findDerivationFilename(EvalState & state, Value & v, std::string what) +{ + Value * v2; + try { + auto dummyArgs = state.allocBindings(0); + v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v).first; + } catch (Error &) { + throw NoPositionInfo("package '%s' has no source location information", what); + } + + // FIXME: is it possible to extract the Pos object instead of doing this + // toString + parsing? + auto pos = state.forceString(*v2); + + auto colon = pos.rfind(':'); + if (colon == std::string::npos) + throw Error("cannot parse meta.position attribute '%s'", pos); + + std::string filename(pos, 0, colon); + unsigned int lineno; + try { + lineno = std::stoi(std::string(pos, colon + 1)); + } catch (std::invalid_argument & e) { + throw Error("cannot parse line number '%s'", pos); + } + + Symbol file = state.symbols.create(filename); + + return { file, lineno, 0 }; } diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index 46a34195093..fce160da727 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -7,7 +7,13 @@ namespace nix { -Value * findAlongAttrPath(EvalState & state, const string & attrPath, +MakeError(AttrPathNotFound, Error); +MakeError(NoPositionInfo, Error); + +std::pair findAlongAttrPath(EvalState & state, const string & attrPath, Bindings & autoArgs, Value & vIn); +/* Heuristic to find the filename and lineno or a nix value. */ +Pos findDerivationFilename(EvalState & state, Value & v, std::string what); + } diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc index 910428c0268..b1d61a28519 100644 --- a/src/libexpr/attr-set.cc +++ b/src/libexpr/attr-set.cc @@ -1,5 +1,5 @@ #include "attr-set.hh" -#include "eval.hh" +#include "eval-inline.hh" #include @@ -7,29 +7,18 @@ namespace nix { -static void * allocBytes(size_t n) -{ - void * p; -#if HAVE_BOEHMGC - p = GC_malloc(n); -#else - p = malloc(n); -#endif - if (!p) throw std::bad_alloc(); - return p; -} - - /* Allocate a new array of attributes for an attribute set with a specific capacity. The space is implicitly reserved after the Bindings structure. */ -Bindings * EvalState::allocBindings(Bindings::size_t capacity) +Bindings * EvalState::allocBindings(size_t capacity) { - return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings(capacity); + if (capacity > std::numeric_limits::max()) + throw Error("attribute set of size %d is too big", capacity); + return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity); } -void EvalState::mkAttrs(Value & v, unsigned int capacity) +void EvalState::mkAttrs(Value & v, size_t capacity) { if (capacity == 0) { v = vEmptySet; @@ -54,6 +43,12 @@ Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name) } +Value * EvalState::allocAttr(Value & vAttrs, const std::string & name) +{ + return allocAttr(vAttrs, symbols.create(name)); +} + + void Bindings::sort() { std::sort(begin(), end()); diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index 7cf6a9c5808..c601d09c2ad 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -4,6 +4,7 @@ #include "symbol-table.hh" #include +#include namespace nix { @@ -63,6 +64,26 @@ public: return end(); } + Attr * get(const Symbol & name) + { + Attr key(name, 0); + iterator i = std::lower_bound(begin(), end(), key); + if (i != end() && i->name == name) return &*i; + return nullptr; + } + + Attr & need(const Symbol & name, const Pos & pos = noPos) + { + auto a = get(name); + if (!a) + throw Error({ + .hint = hintfmt("attribute '%s' missing", name), + .nixCode = NixCode { .errPos = pos } + }); + + return *a; + } + iterator begin() { return &attrs[0]; } iterator end() { return &attrs[size_]; } @@ -75,6 +96,19 @@ public: size_t capacity() { return capacity_; } + /* Returns the attributes in lexicographically sorted order. */ + std::vector lexicographicOrder() const + { + std::vector res; + res.reserve(size_); + for (size_t n = 0; n < size_; n++) + res.emplace_back(&attrs[n]); + std::sort(res.begin(), res.end(), [](const Attr * a, const Attr * b) { + return (const string &) a->name < (const string &) b->name; + }); + return res; + } + friend class EvalState; }; diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc new file mode 100644 index 00000000000..44baadd53b6 --- /dev/null +++ b/src/libexpr/common-eval-args.cc @@ -0,0 +1,64 @@ +#include "common-eval-args.hh" +#include "shared.hh" +#include "filetransfer.hh" +#include "util.hh" +#include "eval.hh" +#include "fetchers.hh" +#include "store-api.hh" + +namespace nix { + +MixEvalArgs::MixEvalArgs() +{ + addFlag({ + .longName = "arg", + .description = "argument to be passed to Nix functions", + .labels = {"name", "expr"}, + .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }} + }); + + addFlag({ + .longName = "argstr", + .description = "string-valued argument to be passed to Nix functions", + .labels = {"name", "string"}, + .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }}, + }); + + addFlag({ + .longName = "include", + .shortName = 'I', + .description = "add a path to the list of locations used to look up <...> file names", + .labels = {"path"}, + .handler = {[&](std::string s) { searchPath.push_back(s); }} + }); +} + +Bindings * MixEvalArgs::getAutoArgs(EvalState & state) +{ + Bindings * res = state.allocBindings(autoArgs.size()); + for (auto & i : autoArgs) { + Value * v = state.allocValue(); + if (i.second[0] == 'E') + state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath("."))); + else + mkString(*v, string(i.second, 1)); + res->push_back(Attr(state.symbols.create(i.first), v)); + } + res->sort(); + return res; +} + +Path lookupFileArg(EvalState & state, string s) +{ + if (isUri(s)) { + return state.store->toRealPath( + fetchers::downloadTarball( + state.store, resolveUri(s), "source", false).storePath); + } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { + Path p = s.substr(1, s.size() - 2); + return state.findFile(p); + } else + return absPath(s); +} + +} diff --git a/src/libexpr/common-eval-args.hh b/src/libexpr/common-eval-args.hh new file mode 100644 index 00000000000..be7fda78378 --- /dev/null +++ b/src/libexpr/common-eval-args.hh @@ -0,0 +1,26 @@ +#pragma once + +#include "args.hh" + +namespace nix { + +class Store; +class EvalState; +class Bindings; + +struct MixEvalArgs : virtual Args +{ + MixEvalArgs(); + + Bindings * getAutoArgs(EvalState & state); + + Strings searchPath; + +private: + + std::map autoArgs; +}; + +Path lookupFileArg(EvalState & state, string s); + +} diff --git a/src/libexpr/common-opts.cc b/src/libexpr/common-opts.cc deleted file mode 100644 index 06d6ed87df9..00000000000 --- a/src/libexpr/common-opts.cc +++ /dev/null @@ -1,67 +0,0 @@ -#include "common-opts.hh" -#include "shared.hh" -#include "download.hh" -#include "util.hh" - - -namespace nix { - - -bool parseAutoArgs(Strings::iterator & i, - const Strings::iterator & argsEnd, std::map & res) -{ - string arg = *i; - if (arg != "--arg" && arg != "--argstr") return false; - - UsageError error(format("‘%1%’ requires two arguments") % arg); - - if (++i == argsEnd) throw error; - string name = *i; - if (++i == argsEnd) throw error; - string value = *i; - - res[name] = (arg == "--arg" ? 'E' : 'S') + value; - - return true; -} - - -Bindings * evalAutoArgs(EvalState & state, std::map & in) -{ - Bindings * res = state.allocBindings(in.size()); - for (auto & i : in) { - Value * v = state.allocValue(); - if (i.second[0] == 'E') - state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath("."))); - else - mkString(*v, string(i.second, 1)); - res->push_back(Attr(state.symbols.create(i.first), v)); - } - res->sort(); - return res; -} - - -bool parseSearchPathArg(Strings::iterator & i, - const Strings::iterator & argsEnd, Strings & searchPath) -{ - if (*i != "-I") return false; - if (++i == argsEnd) throw UsageError("‘-I’ requires an argument"); - searchPath.push_back(*i); - return true; -} - - -Path lookupFileArg(EvalState & state, string s) -{ - if (isUri(s)) - return getDownloader()->downloadCached(state.store, s, true); - else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { - Path p = s.substr(1, s.size() - 2); - return state.findFile(p); - } else - return absPath(s); -} - - -} diff --git a/src/libexpr/common-opts.hh b/src/libexpr/common-opts.hh deleted file mode 100644 index cb2732d6fe7..00000000000 --- a/src/libexpr/common-opts.hh +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include "eval.hh" - -namespace nix { - -class Store; - -/* Some common option parsing between nix-env and nix-instantiate. */ -bool parseAutoArgs(Strings::iterator & i, - const Strings::iterator & argsEnd, std::map & res); - -Bindings * evalAutoArgs(EvalState & state, std::map & in); - -bool parseSearchPathArg(Strings::iterator & i, - const Strings::iterator & argsEnd, Strings & searchPath); - -Path lookupFileArg(EvalState & state, string s); - -} diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 0748fbd3f3e..3d544c903fb 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -7,20 +7,26 @@ namespace nix { -LocalNoInlineNoReturn(void throwEvalError(const char * s, const Pos & pos)) +LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s)) { - throw EvalError(format(s) % pos); + throw EvalError({ + .hint = hintfmt(s), + .nixCode = NixCode { .errPos = pos } + }); } LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v)) { - throw TypeError(format(s) % showType(v)); + throw TypeError(s, showType(v)); } -LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const Pos & pos)) +LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const Value & v)) { - throw TypeError(format(s) % showType(v) % pos); + throw TypeError({ + .hint = hintfmt(s, showType(v)), + .nixCode = NixCode { .errPos = pos } + }); } @@ -33,7 +39,7 @@ void EvalState::forceValue(Value & v, const Pos & pos) v.type = tBlackhole; //checkInterrupt(); expr->eval(*this, *env, v); - } catch (Error & e) { + } catch (...) { v.type = tThunk; v.thunk.env = env; v.thunk.expr = expr; @@ -43,7 +49,7 @@ void EvalState::forceValue(Value & v, const Pos & pos) else if (v.type == tApp) callFunction(*v.app.left, *v.app.right, v, noPos); else if (v.type == tBlackhole) - throwEvalError("infinite recursion encountered, at %1%", pos); + throwEvalError(pos, "infinite recursion encountered"); } @@ -57,9 +63,9 @@ inline void EvalState::forceAttrs(Value & v) inline void EvalState::forceAttrs(Value & v, const Pos & pos) { - forceValue(v); + forceValue(v, pos); if (v.type != tAttrs) - throwTypeError("value is %1% while a set was expected, at %2%", v, pos); + throwTypeError(pos, "value is %1% while a set was expected", v); } @@ -73,9 +79,23 @@ inline void EvalState::forceList(Value & v) inline void EvalState::forceList(Value & v, const Pos & pos) { - forceValue(v); + forceValue(v, pos); if (!v.isList()) - throwTypeError("value is %1% while a list was expected, at %2%", v, pos); + throwTypeError(pos, "value is %1% while a list was expected", v); +} + +/* Note: Various places expect the allocated memory to be zeroed. */ +inline void * allocBytes(size_t n) +{ + void * p; +#if HAVE_BOEHMGC + p = GC_MALLOC(n); +#else + p = calloc(n, 1); +#endif + if (!p) throw std::bad_alloc(); + return p; } + } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 64f3874db61..8e71db2b872 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -5,28 +5,30 @@ #include "derivations.hh" #include "globals.hh" #include "eval-inline.hh" -#include "download.hh" +#include "filetransfer.hh" +#include "json.hh" +#include "function-trace.hh" #include +#include #include #include #include +#include +#include +#include + #include #if HAVE_BOEHMGC +#define GC_INCLUDE_NEW + #include #include -#define NEW new (UseGC) - -#else - -#define NEW new - #endif - namespace nix { @@ -34,7 +36,7 @@ static char * dupString(const char * s) { char * t; #if HAVE_BOEHMGC - t = GC_strdup(s); + t = GC_STRDUP(s); #else t = strdup(s); #endif @@ -43,16 +45,22 @@ static char * dupString(const char * s) } -static void * allocBytes(size_t n) +static char * dupStringWithLen(const char * s, size_t size) { - void * p; + char * t; #if HAVE_BOEHMGC - p = GC_malloc(n); + t = GC_STRNDUP(s, size); #else - p = malloc(n); + t = strndup(s, size); #endif - if (!p) throw std::bad_alloc(); - return p; + if (!t) throw std::bad_alloc(); + return t; +} + + +RootValue allocRootValue(Value * v) +{ + return std::allocate_shared(traceable_allocator(), v); } @@ -60,11 +68,10 @@ static void printValue(std::ostream & str, std::set & active, con { checkInterrupt(); - if (active.find(&v) != active.end()) { + if (!active.insert(&v).second) { str << ""; return; } - active.insert(&v); switch (v.type) { case tInt: @@ -91,13 +98,9 @@ static void printValue(std::ostream & str, std::set & active, con break; case tAttrs: { str << "{ "; - typedef std::map Sorted; - Sorted sorted; - for (auto & i : *v.attrs) - sorted[i.name] = i.value; - for (auto & i : sorted) { - str << i.first << " = "; - printValue(str, active, *i.second); + for (auto & i : v.attrs->lexicographicOrder()) { + str << i->name << " = "; + printValue(str, active, *i->value); str << "; "; } str << "}"; @@ -148,12 +151,22 @@ std::ostream & operator << (std::ostream & str, const Value & v) } +const Value *getPrimOp(const Value &v) { + const Value * primOp = &v; + while (primOp->type == tPrimOpApp) { + primOp = primOp->primOpApp.left; + } + assert(primOp->type == tPrimOp); + return primOp; +} + + string showType(const Value & v) { switch (v.type) { case tInt: return "an integer"; case tBool: return "a boolean"; - case tString: return "a string"; + case tString: return v.string.context ? "a string with context" : "a string"; case tPath: return "a path"; case tNull: return "null"; case tAttrs: return "a set"; @@ -162,8 +175,10 @@ string showType(const Value & v) case tApp: return "a function application"; case tLambda: return "a function"; case tBlackhole: return "a black hole"; - case tPrimOp: return "a built-in function"; - case tPrimOpApp: return "a partially applied built-in function"; + case tPrimOp: + return fmt("the built-in function '%s'", string(v.primOp->name)); + case tPrimOpApp: + return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name)); case tExternal: return v.external->showType(); case tFloat: return "a float"; } @@ -202,11 +217,18 @@ void initGC() #if HAVE_BOEHMGC /* Initialise the Boehm garbage collector. */ + + /* Don't look for interior pointers. This reduces the odds of + misdetection a bit. */ GC_set_all_interior_pointers(0); + /* We don't have any roots in data segments, so don't scan from + there. */ + GC_set_no_dls(1); + GC_INIT(); - GC_oom_fn = oomHandler; + GC_set_oom_fn(oomHandler); /* Set the initial heap size to something fairly big (25% of physical RAM, up to a maximum of 384 MiB) so that in most cases @@ -217,7 +239,7 @@ void initGC() that GC_expand_hp() causes a lot of virtual, but not physical (resident) memory to be allocated. This might be a problem on systems that don't overcommit. */ - if (!getenv("GC_INITIAL_HEAP_SIZE")) { + if (!getEnv("GC_INITIAL_HEAP_SIZE")) { size_t size = 32 * 1024 * 1024; #if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES) size_t maxSize = 384 * 1024 * 1024; @@ -295,21 +317,48 @@ EvalState::EvalState(const Strings & _searchPath, ref store) , sToString(symbols.create("__toString")) , sRight(symbols.create("right")) , sWrong(symbols.create("wrong")) + , sStructuredAttrs(symbols.create("__structuredAttrs")) + , sBuilder(symbols.create("builder")) + , sArgs(symbols.create("args")) + , sOutputHash(symbols.create("outputHash")) + , sOutputHashAlgo(symbols.create("outputHashAlgo")) + , sOutputHashMode(symbols.create("outputHashMode")) + , repair(NoRepair) , store(store) , baseEnv(allocEnv(128)) , staticBaseEnv(false, 0) { - countCalls = getEnv("NIX_COUNT_CALLS", "0") != "0"; - - restricted = settings.get("restrict-eval", false); + countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; assert(gcInitialised); + static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes"); + /* Initialise the Nix expression search path. */ - Strings paths = parseNixPath(getEnv("NIX_PATH", "")); - for (auto & i : _searchPath) addToSearchPath(i); - for (auto & i : paths) addToSearchPath(i); - addToSearchPath("nix=" + settings.nixDataDir + "/nix/corepkgs"); + if (!evalSettings.pureEval) { + for (auto & i : _searchPath) addToSearchPath(i); + for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i); + } + addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true)); + + if (evalSettings.restrictEval || evalSettings.pureEval) { + allowedPaths = PathSet(); + + for (auto & i : searchPath) { + auto r = resolveSearchPathElem(i); + if (!r.first) continue; + + auto path = r.second; + + if (store->isInStore(r.second)) { + StorePathSet closure; + store->computeFSClosure(store->parseStorePath(store->toStorePath(r.second)), closure); + for (auto & path : closure) + allowedPaths->insert(store->printStorePath(path)); + } else + allowedPaths->insert(r.second); + } + } clearValue(vEmptySet); vEmptySet.type = tAttrs; @@ -321,42 +370,93 @@ EvalState::EvalState(const Strings & _searchPath, ref store) EvalState::~EvalState() { - fileEvalCache.clear(); } Path EvalState::checkSourcePath(const Path & path_) { - if (!restricted) return path_; + if (!allowedPaths) return path_; + + auto i = resolvedPaths.find(path_); + if (i != resolvedPaths.end()) + return i->second; + + bool found = false; + + /* First canonicalize the path without symlinks, so we make sure an + * attacker can't append ../../... to a path that would be in allowedPaths + * and thus leak symlink targets. + */ + Path abspath = canonPath(path_); + + for (auto & i : *allowedPaths) { + if (isDirOrInDir(abspath, i)) { + found = true; + break; + } + } + + if (!found) + throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", abspath); /* Resolve symlinks. */ - debug(format("checking access to ‘%s’") % path_); - Path path = canonPath(path_, true); + debug(format("checking access to '%s'") % abspath); + Path path = canonPath(abspath, true); - for (auto & i : searchPath) { - auto r = resolveSearchPathElem(i); - if (!r.first) continue; - if (path == r.second || isInDir(path, r.second)) + for (auto & i : *allowedPaths) { + if (isDirOrInDir(path, i)) { + resolvedPaths[path_] = path; return path; + } } - /* To support import-from-derivation, allow access to anything in - the store. FIXME: only allow access to paths that have been - constructed by this evaluation. */ - if (store->isInStore(path)) return path; + throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path); +} -#if 0 - /* Hack to support the chroot dependencies of corepkgs (see - corepkgs/config.nix.in). */ - if (path == settings.nixPrefix && isStorePath(settings.nixPrefix)) - return path; -#endif - throw RestrictedPathError(format("access to path ‘%1%’ is forbidden in restricted mode") % path_); +void EvalState::checkURI(const std::string & uri) +{ + if (!evalSettings.restrictEval) return; + + /* 'uri' should be equal to a prefix, or in a subdirectory of a + prefix. Thus, the prefix https://github.co does not permit + access to https://github.com. Note: this allows 'http://' and + 'https://' as prefixes for any http/https URI. */ + for (auto & prefix : evalSettings.allowedUris.get()) + if (uri == prefix || + (uri.size() > prefix.size() + && prefix.size() > 0 + && hasPrefix(uri, prefix) + && (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/'))) + return; + + /* If the URI is a path, then check it against allowedPaths as + well. */ + if (hasPrefix(uri, "/")) { + checkSourcePath(uri); + return; + } + + if (hasPrefix(uri, "file://")) { + checkSourcePath(std::string(uri, 7)); + return; + } + + throw RestrictedPathError("access to URI '%s' is forbidden in restricted mode", uri); +} + + +Path EvalState::toRealPath(const Path & path, const PathSet & context) +{ + // FIXME: check whether 'path' is in 'context'. + return + !context.empty() && store->isInStore(path) + ? store->toRealPath(path) + : path; } -void EvalState::addConstant(const string & name, Value & v) +Value * EvalState::addConstant(const string & name, Value & v) { Value * v2 = allocValue(); *v2 = v; @@ -364,20 +464,27 @@ void EvalState::addConstant(const string & name, Value & v) baseEnv.values[baseEnvDispl++] = v2; string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v2)); + return v2; } -void EvalState::addPrimOp(const string & name, - unsigned int arity, PrimOpFun primOp) +Value * EvalState::addPrimOp(const string & name, + size_t arity, PrimOpFun primOp) { + if (arity == 0) { + Value v; + primOp(*this, noPos, nullptr, v); + return addConstant(name, v); + } Value * v = allocValue(); string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; Symbol sym = symbols.create(name2); v->type = tPrimOp; - v->primOp = NEW PrimOp(primOp, arity, sym); + v->primOp = new PrimOp(primOp, arity, sym); staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; baseEnv.values[baseEnvDispl++] = v; baseEnv.values[0]->attrs->push_back(Attr(sym, v)); + return v; } @@ -394,52 +501,74 @@ Value & EvalState::getBuiltin(const string & name) LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2)) { - throw EvalError(format(s) % s2); + throw EvalError(s, s2); } -LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const Pos & pos)) +LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2)) { - throw EvalError(format(s) % s2 % pos); + throw EvalError({ + .hint = hintfmt(s, s2), + .nixCode = NixCode { .errPos = pos } + }); } LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3)) { - throw EvalError(format(s) % s2 % s3); + throw EvalError(s, s2, s3); } -LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3, const Pos & pos)) +LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2, const string & s3)) { - throw EvalError(format(s) % s2 % s3 % pos); + throw EvalError({ + .hint = hintfmt(s, s2, s3), + .nixCode = NixCode { .errPos = pos } + }); } -LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, const Pos & p1, const Pos & p2)) +LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const Symbol & sym, const Pos & p2)) { - throw EvalError(format(s) % sym % p1 % p2); + // p1 is where the error occurred; p2 is a position mentioned in the message. + throw EvalError({ + .hint = hintfmt(s, sym, p2), + .nixCode = NixCode { .errPos = p1 } + }); } -LocalNoInlineNoReturn(void throwTypeError(const char * s, const Pos & pos)) +LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s)) { - throw TypeError(format(s) % pos); + throw TypeError({ + .hint = hintfmt(s), + .nixCode = NixCode { .errPos = pos } + }); } LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1)) { - throw TypeError(format(s) % s1); + throw TypeError(s, s1); } -LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun, const Symbol & s2, const Pos & pos)) +LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2)) { - throw TypeError(format(s) % fun.showNamePos() % s2 % pos); + throw TypeError({ + .hint = hintfmt(s, fun.showNamePos(), s2), + .nixCode = NixCode { .errPos = pos } + }); } -LocalNoInlineNoReturn(void throwAssertionError(const char * s, const Pos & pos)) +LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, const string & s1)) { - throw AssertionError(format(s) % pos); + throw AssertionError({ + .hint = hintfmt(s, s1), + .nixCode = NixCode { .errPos = pos } + }); } -LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos)) +LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * s, const string & s1)) { - throw UndefinedVarError(format(s) % s1 % pos); + throw UndefinedVarError({ + .hint = hintfmt(s, s1), + .nixCode = NixCode { .errPos = pos } + }); } LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2)) @@ -464,11 +593,13 @@ void mkString(Value & v, const char * s) } -Value & mkString(Value & v, const string & s, const PathSet & context) +Value & mkString(Value & v, std::string_view s, const PathSet & context) { - mkString(v, s.c_str()); + v.type = tString; + v.string.s = dupStringWithLen(s.data(), s.size()); + v.string.context = 0; if (!context.empty()) { - unsigned int n = 0; + size_t n = 0; v.string.context = (const char * *) allocBytes((context.size() + 1) * sizeof(char *)); for (auto & i : context) @@ -487,17 +618,17 @@ void mkPath(Value & v, const char * s) inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) { - for (unsigned int l = var.level; l; --l, env = env->up) ; + for (size_t l = var.level; l; --l, env = env->up) ; if (!var.fromWith) return env->values[var.displ]; while (1) { - if (!env->haveWithAttrs) { + if (env->type == Env::HasWithExpr) { if (noEval) return 0; Value * v = allocValue(); evalAttrs(*env->up, (Expr *) env->values[0], *v); env->values[0] = v; - env->haveWithAttrs = true; + env->type = Env::HasWithAttrs; } Bindings::iterator j = env->values[0]->attrs->find(var.name); if (j != env->values[0]->attrs->end()) { @@ -505,37 +636,42 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) return j->value; } if (!env->prevWith) - throwUndefinedVarError("undefined variable ‘%1%’ at %2%", var.name, var.pos); - for (unsigned int l = env->prevWith; l; --l, env = env->up) ; + throwUndefinedVarError(var.pos, "undefined variable '%1%'", var.name); + for (size_t l = env->prevWith; l; --l, env = env->up) ; } } +std::atomic nrValuesFreed{0}; + +void finalizeValue(void * obj, void * data) +{ + nrValuesFreed++; +} + Value * EvalState::allocValue() { nrValues++; - return (Value *) allocBytes(sizeof(Value)); + auto v = (Value *) allocBytes(sizeof(Value)); + //GC_register_finalizer_no_order(v, finalizeValue, nullptr, nullptr, nullptr); + return v; } -Env & EvalState::allocEnv(unsigned int size) +Env & EvalState::allocEnv(size_t size) { - assert(size <= std::numeric_limits::max()); - nrEnvs++; nrValuesInEnvs += size; Env * env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *)); - env->size = size; + env->type = Env::Plain; - /* Clear the values because maybeThunk() and lookupVar fromWith expect this. */ - for (unsigned i = 0; i < size; ++i) - env->values[i] = 0; + /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */ return *env; } -void EvalState::mkList(Value & v, unsigned int size) +void EvalState::mkList(Value & v, size_t size) { clearValue(v); if (size == 1) @@ -570,7 +706,7 @@ void EvalState::mkThunk_(Value & v, Expr * expr) void EvalState::mkPos(Value & v, Pos * pos) { - if (pos) { + if (pos && pos->file.set()) { mkAttrs(v, 3); mkString(*allocAttr(v, sFile), pos->file); mkInt(*allocAttr(v, sLine), pos->line); @@ -630,8 +766,10 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env) } -void EvalState::evalFile(const Path & path, Value & v) +void EvalState::evalFile(const Path & path_, Value & v) { + auto path = checkSourcePath(path_); + FileEvalCache::iterator i; if ((i = fileEvalCache.find(path)) != fileEvalCache.end()) { v = i->second; @@ -644,12 +782,22 @@ void EvalState::evalFile(const Path & path, Value & v) return; } - Activity act(*logger, lvlTalkative, format("evaluating file ‘%1%’") % path2); - Expr * e = parseExprFromFile(checkSourcePath(path2)); + printTalkative("evaluating file '%1%'", path2); + Expr * e = nullptr; + + auto j = fileParseCache.find(path2); + if (j != fileParseCache.end()) + e = j->second; + + if (!e) + e = parseExprFromFile(checkSourcePath(path2)); + + fileParseCache[path2] = e; + try { eval(e, v); } catch (Error & e) { - addErrorPrefix(e, "while evaluating the file ‘%1%’:\n", path2); + addErrorPrefix(e, "while evaluating the file '%1%':\n", path2); throw; } @@ -661,6 +809,7 @@ void EvalState::evalFile(const Path & path, Value & v) void EvalState::resetFileCache() { fileEvalCache.clear(); + fileParseCache.clear(); } @@ -685,7 +834,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos) Value v; e->eval(*this, env, v); if (v.type != tBool) - throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos); + throwTypeError(pos, "value is %1% while a Boolean was expected", v); return v.boolean; } @@ -745,7 +894,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) /* The recursive attributes are evaluated in the new environment, while the inherited attributes are evaluated in the original environment. */ - unsigned int displ = 0; + size_t displ = 0; for (auto & i : attrs) { Value * vAttr; if (hasOverrides && !i.second.inherited) { @@ -768,7 +917,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) if (hasOverrides) { Value * vOverrides = (*v.attrs)[overrides->second.displ].value; state.forceAttrs(*vOverrides); - Bindings * newBnds = state.allocBindings(v.attrs->size() + vOverrides->attrs->size()); + Bindings * newBnds = state.allocBindings(v.attrs->capacity() + vOverrides->attrs->size()); for (auto & i : *v.attrs) newBnds->push_back(i); for (auto & i : *vOverrides->attrs) { @@ -799,7 +948,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) Symbol nameSym = state.symbols.create(nameVal.string.s); Bindings::iterator j = v.attrs->find(nameSym); if (j != v.attrs->end()) - throwEvalError("dynamic attribute ‘%1%’ at %2% already defined at %3%", nameSym, i.pos, *j->pos); + throwEvalError(i.pos, "dynamic attribute '%1%' already defined at %2%", nameSym, *j->pos); i.valueExpr->setName(nameSym); /* Keep sorted order so find can catch duplicates */ @@ -819,7 +968,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v) /* The recursive attributes are evaluated in the new environment, while the inherited attributes are evaluated in the original environment. */ - unsigned int displ = 0; + size_t displ = 0; for (auto & i : attrs->attrs) env2.values[displ++] = i.second.e->maybeThunk(state, i.second.inherited ? env : env2); @@ -830,7 +979,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v) void ExprList::eval(EvalState & state, Env & env, Value & v) { state.mkList(v, elems.size()); - for (unsigned int n = 0; n < elems.size(); ++n) + for (size_t n = 0; n < elems.size(); ++n) v.listElems()[n] = elems[n]->maybeThunk(state, env); } @@ -887,7 +1036,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) } else { state.forceAttrs(*vAttrs, pos); if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) - throwEvalError("attribute ‘%1%’ missing, at %2%", name, pos); + throwEvalError(pos, "attribute '%1%' missing", name); } vAttrs = j->value; pos2 = j->pos; @@ -898,7 +1047,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) } catch (Error & e) { if (pos2 && pos2->file != state.sDerivationNix) - addErrorPrefix(e, "while evaluating the attribute ‘%1%’ at %2%:\n", + addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n", showAttrPath(state, env, attrPath), *pos2); throw; } @@ -952,22 +1101,22 @@ void ExprApp::eval(EvalState & state, Env & env, Value & v) void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) { /* Figure out the number of arguments still needed. */ - unsigned int argsDone = 0; + size_t argsDone = 0; Value * primOp = &fun; while (primOp->type == tPrimOpApp) { argsDone++; primOp = primOp->primOpApp.left; } assert(primOp->type == tPrimOp); - unsigned int arity = primOp->primOp->arity; - unsigned int argsLeft = arity - argsDone; + auto arity = primOp->primOp->arity; + auto argsLeft = arity - argsDone; if (argsLeft == 1) { /* We have all the arguments, so call the primop. */ /* Put all the arguments in an array. */ Value * vArgs[arity]; - unsigned int n = arity - 1; + auto n = arity - 1; vArgs[n--] = &arg; for (Value * arg = &fun; arg->type == tPrimOpApp; arg = arg->primOpApp.left) vArgs[n--] = arg->primOpApp.right; @@ -985,9 +1134,12 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) } } - void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos) { + auto trace = evalSettings.traceFunctionCalls ? std::make_unique(pos) : nullptr; + + forceValue(fun, pos); + if (fun.type == tPrimOp || fun.type == tPrimOpApp) { callPrimOp(fun, arg, v, pos); return; @@ -1003,26 +1155,24 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po auto & fun2 = *allocValue(); fun2 = fun; /* !!! Should we use the attr pos here? */ - forceValue(*found->value, pos); Value v2; callFunction(*found->value, fun2, v2, pos); - forceValue(v2, pos); return callFunction(v2, arg, v, pos); } } if (fun.type != tLambda) - throwTypeError("attempt to call something which is not a function but %1%, at %2%", fun, pos); + throwTypeError(pos, "attempt to call something which is not a function but %1%", fun); ExprLambda & lambda(*fun.lambda.fun); - unsigned int size = + auto size = (lambda.arg.empty() ? 0 : 1) + (lambda.matchAttrs ? lambda.formals->formals.size() : 0); Env & env2(allocEnv(size)); env2.up = fun.lambda.env; - unsigned int displ = 0; + size_t displ = 0; if (!lambda.matchAttrs) env2.values[displ++] = &arg; @@ -1036,12 +1186,12 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po /* For each formal argument, get the actual argument. If there is no matching actual argument but the formal argument has a default, use the default. */ - unsigned int attrsUsed = 0; + size_t attrsUsed = 0; for (auto & i : lambda.formals->formals) { Bindings::iterator j = arg.attrs->find(i.name); if (j == arg.attrs->end()) { - if (!i.def) throwTypeError("%1% called without required argument ‘%2%’, at %3%", - lambda, i.name, pos); + if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'", + lambda, i.name); env2.values[displ++] = i.def->maybeThunk(*this, env2); } else { attrsUsed++; @@ -1056,7 +1206,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po user. */ for (auto & i : *arg.attrs) if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end()) - throwTypeError("%1% called with unexpected argument ‘%2%’, at %3%", lambda, i.name, pos); + throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name); abort(); // can't happen } } @@ -1093,7 +1243,6 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) if (fun.type == tAttrs) { auto found = fun.attrs->find(sFunctor); if (found != fun.attrs->end()) { - forceValue(*found->value); Value * v = allocValue(); callFunction(*found->value, fun, *v, noPos); forceValue(*v); @@ -1114,7 +1263,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) if (j != args.end()) actualArgs->attrs->push_back(*j); else if (!i.def) - throwTypeError("cannot auto-call a function that has an argument without a default value (‘%1%’)", i.name); + throwTypeError("cannot auto-call a function that has an argument without a default value ('%1%')", i.name); } actualArgs->attrs->sort(); @@ -1128,7 +1277,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v) Env & env2(state.allocEnv(1)); env2.up = &env; env2.prevWith = prevWith; - env2.haveWithAttrs = false; + env2.type = Env::HasWithExpr; env2.values[0] = (Value *) attrs; body->eval(state, env2, v); @@ -1137,14 +1286,17 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v) void ExprIf::eval(EvalState & state, Env & env, Value & v) { - (state.evalBool(env, cond) ? then : else_)->eval(state, env, v); + (state.evalBool(env, cond, pos) ? then : else_)->eval(state, env, v); } void ExprAssert::eval(EvalState & state, Env & env, Value & v) { - if (!state.evalBool(env, cond, pos)) - throwAssertionError("assertion failed at %1%", pos); + if (!state.evalBool(env, cond, pos)) { + std::ostringstream out; + cond->show(out); + throwAssertionError(pos, "assertion '%1%' failed at %2%", out.str()); + } body->eval(state, env, v); } @@ -1234,15 +1386,15 @@ void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v) } -void EvalState::concatLists(Value & v, unsigned int nrLists, Value * * lists, const Pos & pos) +void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos) { nrListConcats++; Value * nonEmpty = 0; - unsigned int len = 0; - for (unsigned int n = 0; n < nrLists; ++n) { + size_t len = 0; + for (size_t n = 0; n < nrLists; ++n) { forceList(*lists[n], pos); - unsigned int l = lists[n]->listSize(); + auto l = lists[n]->listSize(); len += l; if (l) nonEmpty = lists[n]; } @@ -1254,9 +1406,10 @@ void EvalState::concatLists(Value & v, unsigned int nrLists, Value * * lists, co mkList(v, len); auto out = v.listElems(); - for (unsigned int n = 0, pos = 0; n < nrLists; ++n) { - unsigned int l = lists[n]->listSize(); - memcpy(out + pos, lists[n]->listElems(), l * sizeof(Value *)); + for (size_t n = 0, pos = 0; n < nrLists; ++n) { + auto l = lists[n]->listSize(); + if (l) + memcpy(out + pos, lists[n]->listElems(), l * sizeof(Value *)); pos += l; } } @@ -1294,14 +1447,14 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) nf = n; nf += vTmp.fpoint; } else - throwEvalError("cannot add %1% to an integer, at %2%", showType(vTmp), pos); + throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp)); } else if (firstType == tFloat) { if (vTmp.type == tInt) { nf += vTmp.integer; } else if (vTmp.type == tFloat) { nf += vTmp.fpoint; } else - throwEvalError("cannot add %1% to a float, at %2%", showType(vTmp), pos); + throwEvalError(pos, "cannot add %1% to a float", showType(vTmp)); } else s << state.coerceToString(pos, vTmp, context, false, firstType == tString); } @@ -1312,7 +1465,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) mkFloat(v, nf); else if (firstType == tPath) { if (!context.empty()) - throwEvalError("a string that refers to a store path cannot be appended to a path, at %1%", pos); + throwEvalError(pos, "a string that refers to a store path cannot be appended to a path"); auto path = canonPath(s.str()); mkPath(v, path.c_str()); } else @@ -1333,8 +1486,7 @@ void EvalState::forceValueDeep(Value & v) std::function recurse; recurse = [&](Value & v) { - if (seen.find(&v) != seen.end()) return; - seen.insert(&v); + if (!seen.insert(&v).second) return; forceValue(v); @@ -1343,13 +1495,13 @@ void EvalState::forceValueDeep(Value & v) try { recurse(*i.value); } catch (Error & e) { - addErrorPrefix(e, "while evaluating the attribute ‘%1%’ at %2%:\n", i.name, *i.pos); + addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n", i.name, *i.pos); throw; } } else if (v.isList()) { - for (unsigned int n = 0; n < v.listSize(); ++n) + for (size_t n = 0; n < v.listSize(); ++n) recurse(*v.listElems()[n]); } }; @@ -1362,7 +1514,7 @@ NixInt EvalState::forceInt(Value & v, const Pos & pos) { forceValue(v, pos); if (v.type != tInt) - throwTypeError("value is %1% while an integer was expected, at %2%", v, pos); + throwTypeError(pos, "value is %1% while an integer was expected", v); return v.integer; } @@ -1373,16 +1525,16 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos) if (v.type == tInt) return v.integer; else if (v.type != tFloat) - throwTypeError("value is %1% while a float was expected, at %2%", v, pos); + throwTypeError(pos, "value is %1% while a float was expected", v); return v.fpoint; } bool EvalState::forceBool(Value & v, const Pos & pos) { - forceValue(v); + forceValue(v, pos); if (v.type != tBool) - throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos); + throwTypeError(pos, "value is %1% while a Boolean was expected", v); return v.boolean; } @@ -1395,9 +1547,9 @@ bool EvalState::isFunctor(Value & fun) void EvalState::forceFunction(Value & v, const Pos & pos) { - forceValue(v); + forceValue(v, pos); if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v)) - throwTypeError("value is %1% while a function was expected, at %2%", v, pos); + throwTypeError(pos, "value is %1% while a function was expected", v); } @@ -1406,7 +1558,7 @@ string EvalState::forceString(Value & v, const Pos & pos) forceValue(v, pos); if (v.type != tString) { if (pos) - throwTypeError("value is %1% while a string was expected, at %2%", v, pos); + throwTypeError(pos, "value is %1% while a string was expected", v); else throwTypeError("value is %1% while a string was expected", v); } @@ -1435,10 +1587,10 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos) string s = forceString(v, pos); if (v.string.context) { if (pos) - throwEvalError("the string ‘%1%’ is not allowed to refer to a store path (such as ‘%2%’), at %3%", - v.string.s, v.string.context[0], pos); + throwEvalError(pos, "the string '%1%' is not allowed to refer to a store path (such as '%2%')", + v.string.s, v.string.context[0]); else - throwEvalError("the string ‘%1%’ is not allowed to refer to a store path (such as ‘%2%’)", + throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string.s, v.string.context[0]); } return s; @@ -1456,10 +1608,23 @@ bool EvalState::isDerivation(Value & v) } +std::optional EvalState::tryAttrsToString(const Pos & pos, Value & v, + PathSet & context, bool coerceMore, bool copyToStore) +{ + auto i = v.attrs->find(sToString); + if (i != v.attrs->end()) { + Value v1; + callFunction(*i->value, v, v1, pos); + return coerceToString(pos, v1, context, coerceMore, copyToStore); + } + + return {}; +} + string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, bool coerceMore, bool copyToStore) { - forceValue(v); + forceValue(v, pos); string s; @@ -1474,15 +1639,12 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, } if (v.type == tAttrs) { - auto i = v.attrs->find(sToString); - if (i != v.attrs->end()) { - forceValue(*i->value, pos); - Value v1; - callFunction(*i->value, v, v1, pos); - return coerceToString(pos, v1, context, coerceMore, copyToStore); + auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore); + if (maybeString) { + return *maybeString; } - i = v.attrs->find(sOutPath); - if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string, at %1%", pos); + auto i = v.attrs->find(sOutPath); + if (i == v.attrs->end()) throwTypeError(pos, "cannot coerce a set to a string"); return coerceToString(pos, *i->value, context, coerceMore, copyToStore); } @@ -1501,7 +1663,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, if (v.isList()) { string result; - for (unsigned int n = 0; n < v.listSize(); ++n) { + for (size_t n = 0; n < v.listSize(); ++n) { result += coerceToString(pos, *v.listElems()[n], context, coerceMore, copyToStore); if (n < v.listSize() - 1 @@ -1513,25 +1675,26 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, } } - throwTypeError("cannot coerce %1% to a string, at %2%", v, pos); + throwTypeError(pos, "cannot coerce %1% to a string", v); } string EvalState::copyPathToStore(PathSet & context, const Path & path) { if (nix::isDerivation(path)) - throwEvalError("file names are not allowed to end in ‘%1%’", drvExtension); + throwEvalError("file names are not allowed to end in '%1%'", drvExtension); Path dstPath; - if (srcToStore[path] != "") - dstPath = srcToStore[path]; + auto i = srcToStore.find(path); + if (i != srcToStore.end()) + dstPath = store->printStorePath(i->second); else { - dstPath = settings.readOnlyMode - ? store->computeStorePathForPath(checkSourcePath(path)).first - : store->addToStore(baseNameOf(path), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair); - srcToStore[path] = dstPath; - printMsg(lvlChatty, format("copied source ‘%1%’ -> ‘%2%’") - % path % dstPath); + auto p = settings.readOnlyMode + ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first + : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair); + dstPath = store->printStorePath(p); + srcToStore.insert_or_assign(path, std::move(p)); + printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath); } context.insert(dstPath); @@ -1543,7 +1706,7 @@ Path EvalState::coerceToPath(const Pos & pos, Value & v, PathSet & context) { string path = coerceToString(pos, v, context, false, false); if (path == "" || path[0] != '/') - throwEvalError("string ‘%1%’ doesn't represent an absolute path, at %2%", path, pos); + throwEvalError(pos, "string '%1%' doesn't represent an absolute path", path); return path; } @@ -1588,7 +1751,7 @@ bool EvalState::eqValues(Value & v1, Value & v2) case tList2: case tListN: if (v1.listSize() != v2.listSize()) return false; - for (unsigned int n = 0; n < v1.listSize(); ++n) + for (size_t n = 0; n < v1.listSize(); ++n) if (!eqValues(*v1.listElems()[n], *v2.listElems()[n])) return false; return true; @@ -1630,12 +1793,9 @@ bool EvalState::eqValues(Value & v1, Value & v2) } } - void EvalState::printStats() { - bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0"; - Verbosity v = showStats ? lvlInfo : lvlDebug; - printMsg(v, "evaluation statistics:"); + bool showStats = getEnv("NIX_SHOW_STATS").value_or("0") != "0"; struct rusage buf; getrusage(RUSAGE_SELF, &buf); @@ -1646,159 +1806,117 @@ void EvalState::printStats() uint64_t bValues = nrValues * sizeof(Value); uint64_t bAttrsets = nrAttrsets * sizeof(Bindings) + nrAttrsInAttrsets * sizeof(Attr); - printMsg(v, format(" time elapsed: %1%") % cpuTime); - printMsg(v, format(" size of a value: %1%") % sizeof(Value)); - printMsg(v, format(" size of an attr: %1%") % sizeof(Attr)); - printMsg(v, format(" environments allocated: %1% (%2% bytes)") % nrEnvs % bEnvs); - printMsg(v, format(" list elements: %1% (%2% bytes)") % nrListElems % bLists); - printMsg(v, format(" list concatenations: %1%") % nrListConcats); - printMsg(v, format(" values allocated: %1% (%2% bytes)") % nrValues % bValues); - printMsg(v, format(" sets allocated: %1% (%2% bytes)") % nrAttrsets % bAttrsets); - printMsg(v, format(" right-biased unions: %1%") % nrOpUpdates); - printMsg(v, format(" values copied in right-biased unions: %1%") % nrOpUpdateValuesCopied); - printMsg(v, format(" symbols in symbol table: %1%") % symbols.size()); - printMsg(v, format(" size of symbol table: %1%") % symbols.totalSize()); - printMsg(v, format(" number of thunks: %1%") % nrThunks); - printMsg(v, format(" number of thunks avoided: %1%") % nrAvoided); - printMsg(v, format(" number of attr lookups: %1%") % nrLookups); - printMsg(v, format(" number of primop calls: %1%") % nrPrimOpCalls); - printMsg(v, format(" number of function calls: %1%") % nrFunctionCalls); - printMsg(v, format(" total allocations: %1% bytes") % (bEnvs + bLists + bValues + bAttrsets)); - #if HAVE_BOEHMGC GC_word heapSize, totalBytes; GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes); - printMsg(v, format(" current Boehm heap size: %1% bytes") % heapSize); - printMsg(v, format(" total Boehm heap allocations: %1% bytes") % totalBytes); +#endif + if (showStats) { + auto outPath = getEnv("NIX_SHOW_STATS_PATH").value_or("-"); + std::fstream fs; + if (outPath != "-") + fs.open(outPath, std::fstream::out); + JSONObject topObj(outPath == "-" ? std::cerr : fs, true); + topObj.attr("cpuTime",cpuTime); + { + auto envs = topObj.object("envs"); + envs.attr("number", nrEnvs); + envs.attr("elements", nrValuesInEnvs); + envs.attr("bytes", bEnvs); + } + { + auto lists = topObj.object("list"); + lists.attr("elements", nrListElems); + lists.attr("bytes", bLists); + lists.attr("concats", nrListConcats); + } + { + auto values = topObj.object("values"); + values.attr("number", nrValues); + values.attr("bytes", bValues); + } + { + auto syms = topObj.object("symbols"); + syms.attr("number", symbols.size()); + syms.attr("bytes", symbols.totalSize()); + } + { + auto sets = topObj.object("sets"); + sets.attr("number", nrAttrsets); + sets.attr("bytes", bAttrsets); + sets.attr("elements", nrAttrsInAttrsets); + } + { + auto sizes = topObj.object("sizes"); + sizes.attr("Env", sizeof(Env)); + sizes.attr("Value", sizeof(Value)); + sizes.attr("Bindings", sizeof(Bindings)); + sizes.attr("Attr", sizeof(Attr)); + } + topObj.attr("nrOpUpdates", nrOpUpdates); + topObj.attr("nrOpUpdateValuesCopied", nrOpUpdateValuesCopied); + topObj.attr("nrThunks", nrThunks); + topObj.attr("nrAvoided", nrAvoided); + topObj.attr("nrLookups", nrLookups); + topObj.attr("nrPrimOpCalls", nrPrimOpCalls); + topObj.attr("nrFunctionCalls", nrFunctionCalls); +#if HAVE_BOEHMGC + { + auto gc = topObj.object("gc"); + gc.attr("heapSize", heapSize); + gc.attr("totalBytes", totalBytes); + } #endif - if (countCalls) { - v = lvlInfo; - - printMsg(v, format("calls to %1% primops:") % primOpCalls.size()); - typedef std::multimap PrimOpCalls_; - PrimOpCalls_ primOpCalls_; - for (auto & i : primOpCalls) - primOpCalls_.insert(std::pair(i.second, i.first)); - for (auto i = primOpCalls_.rbegin(); i != primOpCalls_.rend(); ++i) - printMsg(v, format("%1$10d %2%") % i->first % i->second); - - printMsg(v, format("calls to %1% functions:") % functionCalls.size()); - typedef std::multimap FunctionCalls_; - FunctionCalls_ functionCalls_; - for (auto & i : functionCalls) - functionCalls_.insert(std::pair(i.second, i.first)); - for (auto i = functionCalls_.rbegin(); i != functionCalls_.rend(); ++i) - printMsg(v, format("%1$10d %2%") % i->first % i->second->showNamePos()); - - printMsg(v, format("evaluations of %1% attributes:") % attrSelects.size()); - typedef std::multimap AttrSelects_; - AttrSelects_ attrSelects_; - for (auto & i : attrSelects) - attrSelects_.insert(std::pair(i.second, i.first)); - for (auto i = attrSelects_.rbegin(); i != attrSelects_.rend(); ++i) - printMsg(v, format("%1$10d %2%") % i->first % i->second); - - } -} - - -size_t valueSize(Value & v) -{ - std::set seen; - - auto doString = [&](const char * s) -> size_t { - if (seen.find(s) != seen.end()) return 0; - seen.insert(s); - return strlen(s) + 1; - }; - - std::function doValue; - std::function doEnv; - - doValue = [&](Value & v) -> size_t { - if (seen.find(&v) != seen.end()) return 0; - seen.insert(&v); - - size_t sz = sizeof(Value); - - switch (v.type) { - case tString: - sz += doString(v.string.s); - if (v.string.context) - for (const char * * p = v.string.context; *p; ++p) - sz += doString(*p); - break; - case tPath: - sz += doString(v.path); - break; - case tAttrs: - if (seen.find(v.attrs) == seen.end()) { - seen.insert(v.attrs); - sz += sizeof(Bindings) + sizeof(Attr) * v.attrs->capacity(); - for (auto & i : *v.attrs) - sz += doValue(*i.value); + if (countCalls) { + { + auto obj = topObj.object("primops"); + for (auto & i : primOpCalls) + obj.attr(i.first, i.second); } - break; - case tList1: - case tList2: - case tListN: - if (seen.find(v.listElems()) == seen.end()) { - seen.insert(v.listElems()); - sz += v.listSize() * sizeof(Value *); - for (unsigned int n = 0; n < v.listSize(); ++n) - sz += doValue(*v.listElems()[n]); + { + auto list = topObj.list("functions"); + for (auto & i : functionCalls) { + auto obj = list.object(); + if (i.first->name.set()) + obj.attr("name", (const string &) i.first->name); + else + obj.attr("name", nullptr); + if (i.first->pos) { + obj.attr("file", (const string &) i.first->pos.file); + obj.attr("line", i.first->pos.line); + obj.attr("column", i.first->pos.column); + } + obj.attr("count", i.second); + } + } + { + auto list = topObj.list("attributes"); + for (auto & i : attrSelects) { + auto obj = list.object(); + if (i.first) { + obj.attr("file", (const string &) i.first.file); + obj.attr("line", i.first.line); + obj.attr("column", i.first.column); + } + obj.attr("count", i.second); + } } - break; - case tThunk: - sz += doEnv(*v.thunk.env); - break; - case tApp: - sz += doValue(*v.app.left); - sz += doValue(*v.app.right); - break; - case tLambda: - sz += doEnv(*v.lambda.env); - break; - case tPrimOpApp: - sz += doValue(*v.primOpApp.left); - sz += doValue(*v.primOpApp.right); - break; - case tExternal: - if (seen.find(v.external) != seen.end()) break; - seen.insert(v.external); - sz += v.external->valueSize(seen); - break; - default: - ; } - return sz; - }; - - doEnv = [&](Env & env) -> size_t { - if (seen.find(&env) != seen.end()) return 0; - seen.insert(&env); - - size_t sz = sizeof(Env) + sizeof(Value *) * env.size; - - for (unsigned int i = 0; i < env.size; ++i) - if (env.values[i]) - sz += doValue(*env.values[i]); - - if (env.up) sz += doEnv(*env.up); - - return sz; - }; - - return doValue(v); + if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") { + auto list = topObj.list("symbols"); + symbols.dump([&](const std::string & s) { list.elem(s); }); + } + } } string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const { - throw TypeError(format("cannot coerce %1% to a string, at %2%") % - showType() % pos); + throw TypeError({ + .hint = hintfmt("cannot coerce %1% to a string", showType()), + .nixCode = NixCode { .errPos = pos } + }); } @@ -1813,4 +1931,25 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { } +EvalSettings::EvalSettings() +{ + auto var = getEnv("NIX_PATH"); + if (var) nixPath = parseNixPath(*var); +} + +Strings EvalSettings::getDefaultNixPath() +{ + Strings res; + auto add = [&](const Path & p) { if (pathExists(p)) { res.push_back(p); } }; + add(getHome() + "/.nix-defexpr/channels"); + add("nixpkgs=" + settings.nixStateDir + "/nix/profiles/per-user/root/channels/nixpkgs"); + add(settings.nixStateDir + "/nix/profiles/per-user/root/channels"); + return res; +} + +EvalSettings evalSettings; + +static GlobalConfig::Register r1(&evalSettings); + + } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 195cb0db3ac..1485dc7fe29 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -5,8 +5,12 @@ #include "nixexpr.hh" #include "symbol-table.hh" #include "hash.hh" +#include "config.hh" +#include #include +#include +#include namespace nix { @@ -14,6 +18,8 @@ namespace nix { class Store; class EvalState; +struct StorePath; +enum RepairFlag : bool; typedef void (* PrimOpFun) (EvalState & state, const Pos & pos, Value * * args, Value & v); @@ -32,21 +38,20 @@ struct PrimOp struct Env { Env * up; - unsigned short size; // used by ‘valueSize’ - unsigned short prevWith:15; // nr of levels up to next `with' environment - unsigned short haveWithAttrs:1; + unsigned short prevWith:14; // nr of levels up to next `with' environment + enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2; Value * values[0]; }; -Value & mkString(Value & v, const string & s, const PathSet & context = PathSet()); +Value & mkString(Value & v, std::string_view s, const PathSet & context = PathSet()); void copyContext(const Value & v, PathSet & context); /* Cache for calls to addToStore(); maps source paths to the store paths. */ -typedef std::map SrcToStore; +typedef std::map SrcToStore; std::ostream & operator << (std::ostream & str, const Value & v); @@ -68,16 +73,17 @@ public: const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, - sRight, sWrong; + sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, + sOutputHash, sOutputHashAlgo, sOutputHashMode; Symbol sDerivationNix; /* If set, force copying files to the Nix store even if they already exist there. */ - bool repair = false; + RepairFlag repair; - /* If set, don't allow access to files outside of the Nix search - path or to environment variables. */ - bool restricted; + /* The allowed filesystem paths in restricted or pure evaluation + mode. */ + std::optional allowedPaths; Value vEmptySet; @@ -86,6 +92,14 @@ public: private: SrcToStore srcToStore; + /* A cache from path names to parse trees. */ +#if HAVE_BOEHMGC + typedef std::map, traceable_allocator > > FileParseCache; +#else + typedef std::map FileParseCache; +#endif + FileParseCache fileParseCache; + /* A cache from path names to values. */ #if HAVE_BOEHMGC typedef std::map, traceable_allocator > > FileEvalCache; @@ -98,6 +112,12 @@ private: std::map> searchPathResolved; + /* Cache used by checkSourcePath(). */ + std::unordered_map resolvedPaths; + + /* Cache used by prim_match(). */ + std::unordered_map regexCache; + public: EvalState(const Strings & _searchPath, ref store); @@ -109,13 +129,26 @@ public: Path checkSourcePath(const Path & path); + void checkURI(const std::string & uri); + + /* When using a diverted store and 'path' is in the Nix store, map + 'path' to the diverted location (e.g. /nix/store/foo is mapped + to /home/alice/my-nix/nix/store/foo). However, this is only + done if the context is not empty, since otherwise we're + probably trying to read from the actual /nix/store. This is + intended to distinguish between import-from-derivation and + sources stored in the actual /nix/store. */ + Path toRealPath(const Path & path, const PathSet & context); + /* Parse a Nix expression from the specified file. */ Expr * parseExprFromFile(const Path & path); Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv); /* Parse a Nix expression from the specified string. */ - Expr * parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv); - Expr * parseExprFromString(const string & s, const Path & basePath); + Expr * parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv); + Expr * parseExprFromString(std::string_view s, const Path & basePath); + + Expr * parseStdin(); /* Evaluate an expression read from the given file to normal form. */ @@ -167,6 +200,9 @@ public: set with attribute `type = "derivation"'). */ bool isDerivation(Value & v); + std::optional tryAttrsToString(const Pos & pos, Value & v, + PathSet & context, bool coerceMore = false, bool copyToStore = true); + /* String coercion. Converts strings, paths and derivations to a string. If `coerceMore' is set, also converts nulls, integers, booleans and lists to a string. If `copyToStore' is set, @@ -196,10 +232,10 @@ private: void createBaseEnv(); - void addConstant(const string & name, Value & v); + Value * addConstant(const string & name, Value & v); - void addPrimOp(const string & name, - unsigned int arity, PrimOpFun primOp); + Value * addPrimOp(const string & name, + size_t arity, PrimOpFun primOp); public: @@ -233,18 +269,19 @@ public: /* Allocation primitives. */ Value * allocValue(); - Env & allocEnv(unsigned int size); + Env & allocEnv(size_t size); Value * allocAttr(Value & vAttrs, const Symbol & name); + Value * allocAttr(Value & vAttrs, const std::string & name); - Bindings * allocBindings(Bindings::size_t capacity); + Bindings * allocBindings(size_t capacity); - void mkList(Value & v, unsigned int length); - void mkAttrs(Value & v, unsigned int capacity); + void mkList(Value & v, size_t length); + void mkAttrs(Value & v, size_t capacity); void mkThunk_(Value & v, Expr * expr); void mkPos(Value & v, Pos * pos); - void concatLists(Value & v, unsigned int nrLists, Value * * lists, const Pos & pos); + void concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos); /* Print statistics. */ void printStats(); @@ -267,27 +304,31 @@ private: bool countCalls; - typedef std::map PrimOpCalls; + typedef std::map PrimOpCalls; PrimOpCalls primOpCalls; - typedef std::map FunctionCalls; + typedef std::map FunctionCalls; FunctionCalls functionCalls; void incrFunctionCall(ExprLambda * fun); - typedef std::map AttrSelects; + typedef std::map AttrSelects; AttrSelects attrSelects; friend struct ExprOpUpdate; friend struct ExprOpConcatLists; friend struct ExprSelect; friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v); + friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v); }; /* Return a string representing the type of the value `v'. */ string showType(const Value & v); +/* Decode a context string ‘!!’ into a pair . */ +std::pair decodeContext(const string & s); /* If `path' refers to a directory, then append "/default.nix". */ Path resolveExprPath(Path path); @@ -301,4 +342,35 @@ struct InvalidPathError : EvalError #endif }; +struct EvalSettings : Config +{ + EvalSettings(); + + static Strings getDefaultNixPath(); + + Setting enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", + "Whether builtin functions that allow executing native code should be enabled."}; + + Setting nixPath{this, getDefaultNixPath(), "nix-path", + "List of directories to be searched for <...> file references."}; + + Setting restrictEval{this, false, "restrict-eval", + "Whether to restrict file system access to paths in $NIX_PATH, " + "and network access to the URI prefixes listed in 'allowed-uris'."}; + + Setting pureEval{this, false, "pure-eval", + "Whether to restrict file system and network access to files specified by cryptographic hash."}; + + Setting enableImportFromDerivation{this, true, "allow-import-from-derivation", + "Whether the evaluator allows importing the result of a derivation."}; + + Setting allowedUris{this, {}, "allowed-uris", + "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; + + Setting traceFunctionCalls{this, false, "trace-function-calls", + "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."}; +}; + +extern EvalSettings evalSettings; + } diff --git a/src/libexpr/function-trace.cc b/src/libexpr/function-trace.cc new file mode 100644 index 00000000000..c6057b3842f --- /dev/null +++ b/src/libexpr/function-trace.cc @@ -0,0 +1,18 @@ +#include "function-trace.hh" +#include "logging.hh" + +namespace nix { + +FunctionCallTrace::FunctionCallTrace(const Pos & pos) : pos(pos) { + auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); + auto ns = std::chrono::duration_cast(duration); + printMsg(lvlInfo, "function-trace entered %1% at %2%", pos, ns.count()); +} + +FunctionCallTrace::~FunctionCallTrace() { + auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); + auto ns = std::chrono::duration_cast(duration); + printMsg(lvlInfo, "function-trace exited %1% at %2%", pos, ns.count()); +} + +} diff --git a/src/libexpr/function-trace.hh b/src/libexpr/function-trace.hh new file mode 100644 index 00000000000..472f2045ed6 --- /dev/null +++ b/src/libexpr/function-trace.hh @@ -0,0 +1,15 @@ +#pragma once + +#include "eval.hh" + +#include + +namespace nix { + +struct FunctionCallTrace +{ + const Pos & pos; + FunctionCallTrace(const Pos & pos); + ~FunctionCallTrace(); +}; +} diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index dc5def911ca..ca9c547faa7 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -1,14 +1,70 @@ #include "get-drvs.hh" #include "util.hh" #include "eval-inline.hh" +#include "derivations.hh" #include +#include namespace nix { -string DrvInfo::queryDrvPath() +DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs) + : state(&state), attrs(attrs), attrPath(attrPath) +{ +} + + +DrvInfo::DrvInfo(EvalState & state, ref store, const std::string & drvPathWithOutputs) + : state(&state), attrs(nullptr), attrPath("") +{ + auto [drvPath, selectedOutputs] = store->parsePathWithOutputs(drvPathWithOutputs); + + this->drvPath = store->printStorePath(drvPath); + + auto drv = store->derivationFromPath(drvPath); + + name = drvPath.name(); + + if (selectedOutputs.size() > 1) + throw Error("building more than one derivation output is not supported, in '%s'", drvPathWithOutputs); + + outputName = + selectedOutputs.empty() + ? get(drv.env, "outputName").value_or("out") + : *selectedOutputs.begin(); + + auto i = drv.outputs.find(outputName); + if (i == drv.outputs.end()) + throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName); + + outPath = store->printStorePath(i->second.path); +} + + +string DrvInfo::queryName() const +{ + if (name == "" && attrs) { + auto i = attrs->find(state->sName); + if (i == attrs->end()) throw TypeError("derivation name missing"); + name = state->forceStringNoCtx(*i->value); + } + return name; +} + + +string DrvInfo::querySystem() const +{ + if (system == "" && attrs) { + auto i = attrs->find(state->sSystem); + system = i == attrs->end() ? "unknown" : state->forceStringNoCtx(*i->value, *i->pos); + } + return system; +} + + +string DrvInfo::queryDrvPath() const { if (drvPath == "" && attrs) { Bindings::iterator i = attrs->find(state->sDrvPath); @@ -19,7 +75,7 @@ string DrvInfo::queryDrvPath() } -string DrvInfo::queryOutPath() +string DrvInfo::queryOutPath() const { if (outPath == "" && attrs) { Bindings::iterator i = attrs->find(state->sOutPath); @@ -61,7 +117,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) /* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */ const Value * outTI = queryMeta("outputsToInstall"); if (!outTI) return outputs; - const auto errMsg = Error("this derivation has bad ‘meta.outputsToInstall’"); + const auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'"); /* ^ this shows during `nix-env -i` right under the bad derivation */ if (!outTI->isList()) throw errMsg; Outputs result; @@ -75,7 +131,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) } -string DrvInfo::queryOutputName() +string DrvInfo::queryOutputName() const { if (outputName == "" && attrs) { Bindings::iterator i = attrs->find(state->sOutputName); @@ -221,20 +277,14 @@ static bool getDerivation(EvalState & state, Value & v, /* Remove spurious duplicates (e.g., a set like `rec { x = derivation {...}; y = x;}'. */ - if (done.find(v.attrs) != done.end()) return false; - done.insert(v.attrs); - - Bindings::iterator i = v.attrs->find(state.sName); - /* !!! We really would like to have a decent back trace here. */ - if (i == v.attrs->end()) throw TypeError("derivation name missing"); + if (!done.insert(v.attrs).second) return false; - Bindings::iterator i2 = v.attrs->find(state.sSystem); + DrvInfo drv(state, attrPath, v.attrs); - DrvInfo drv(state, state.forceStringNoCtx(*i->value), attrPath, - i2 == v.attrs->end() ? "unknown" : state.forceStringNoCtx(*i2->value, *i2->pos), - v.attrs); + drv.queryName(); drvs.push_back(drv); + return false; } catch (AssertionError & e) { @@ -244,15 +294,14 @@ static bool getDerivation(EvalState & state, Value & v, } -bool getDerivation(EvalState & state, Value & v, DrvInfo & drv, +std::optional getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures) { Done done; DrvInfos drvs; getDerivation(state, v, "", drvs, done, ignoreAssertionFailures); - if (drvs.size() != 1) return false; - drv = drvs.front(); - return true; + if (drvs.size() != 1) return {}; + return std::move(drvs.front()); } @@ -262,6 +311,9 @@ static string addToPath(const string & s1, const string & s2) } +static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*"); + + static void getDerivations(EvalState & state, Value & vIn, const string & pathPrefix, Bindings & autoArgs, DrvInfos & drvs, Done & done, @@ -284,25 +336,21 @@ static void getDerivations(EvalState & state, Value & vIn, there are names clashes between derivations, the derivation bound to the attribute with the "lower" name should take precedence). */ - typedef std::map SortedSymbols; - SortedSymbols attrs; - for (auto & i : *v.attrs) - attrs.insert(std::pair(i.name, i.name)); - - for (auto & i : attrs) { - Activity act(*logger, lvlDebug, format("evaluating attribute ‘%1%’") % i.first); - string pathPrefix2 = addToPath(pathPrefix, i.first); - Value & v2(*v.attrs->find(i.second)->value); + for (auto & i : v.attrs->lexicographicOrder()) { + debug("evaluating attribute '%1%'", i->name); + if (!std::regex_match(std::string(i->name), attrRegex)) + continue; + string pathPrefix2 = addToPath(pathPrefix, i->name); if (combineChannels) - getDerivations(state, v2, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); - else if (getDerivation(state, v2, pathPrefix2, drvs, done, ignoreAssertionFailures)) { + getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) { /* If the value of this attribute is itself a set, should we recurse into it? => Only if it has a `recurseForDerivations = true' attribute. */ - if (v2.type == tAttrs) { - Bindings::iterator j = v2.attrs->find(state.symbols.create("recurseForDerivations")); - if (j != v2.attrs->end() && state.forceBool(*j->value, *j->pos)) - getDerivations(state, v2, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + if (i->value->type == tAttrs) { + Bindings::iterator j = i->value->attrs->find(state.symbols.create("recurseForDerivations")); + if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos)) + getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); } } } @@ -310,7 +358,6 @@ static void getDerivations(EvalState & state, Value & vIn, else if (v.isList()) { for (unsigned int n = 0; n < v.listSize(); ++n) { - Activity act(*logger, lvlDebug, "evaluating list element"); string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str()); if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures)) getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index 37fcbe829d3..d7860fc6a4b 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -17,32 +17,34 @@ public: private: EvalState * state; - string drvPath; - string outPath; - string outputName; + mutable string name; + mutable string system; + mutable string drvPath; + mutable string outPath; + mutable string outputName; Outputs outputs; - bool failed; // set if we get an AssertionError + bool failed = false; // set if we get an AssertionError - Bindings * attrs, * meta; + Bindings * attrs = nullptr, * meta = nullptr; Bindings * getMeta(); bool checkMeta(Value & v); public: - string name; string attrPath; /* path towards the derivation */ - string system; - DrvInfo(EvalState & state) : state(&state), failed(false), attrs(0), meta(0) { }; - DrvInfo(EvalState & state, const string & name, const string & attrPath, const string & system, Bindings * attrs) - : state(&state), failed(false), attrs(attrs), meta(0), name(name), attrPath(attrPath), system(system) { }; + DrvInfo(EvalState & state) : state(&state) { }; + DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs); + DrvInfo(EvalState & state, ref store, const std::string & drvPathWithOutputs); - string queryDrvPath(); - string queryOutPath(); - string queryOutputName(); - /** Return the list of outputs. The "outputs to install" are determined by `mesa.outputsToInstall`. */ + string queryName() const; + string querySystem() const; + string queryDrvPath() const; + string queryOutPath() const; + string queryOutputName() const; + /** Return the list of outputs. The "outputs to install" are determined by `meta.outputsToInstall`. */ Outputs queryOutputs(bool onlyOutputsToInstall = false); StringSet queryMetaNames(); @@ -58,15 +60,9 @@ public: MetaValue queryMetaInfo(EvalState & state, const string & name) const; */ - void setDrvPath(const string & s) - { - drvPath = s; - } - - void setOutPath(const string & s) - { - outPath = s; - } + void setName(const string & s) { name = s; } + void setDrvPath(const string & s) { drvPath = s; } + void setOutPath(const string & s) { outPath = s; } void setFailed() { failed = true; }; bool hasFailed() { return failed; }; @@ -80,10 +76,10 @@ typedef list DrvInfos; #endif -/* If value `v' denotes a derivation, store information about the - derivation in `drv' and return true. Otherwise, return false. */ -bool getDerivation(EvalState & state, Value & v, DrvInfo & drv, - bool ignoreAssertionFailures); +/* If value `v' denotes a derivation, return a DrvInfo object + describing it. Otherwise return nothing. */ +std::optional getDerivation(EvalState & state, + Value & v, bool ignoreAssertionFailures); void getDerivations(EvalState & state, Value & v, const string & pathPrefix, Bindings & autoArgs, DrvInfos & drvs, diff --git a/corepkgs/imported-drv-to-derivation.nix b/src/libexpr/imported-drv-to-derivation.nix similarity index 100% rename from corepkgs/imported-drv-to-derivation.nix rename to src/libexpr/imported-drv-to-derivation.nix diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index f671802bcc2..76e1a26bff8 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -1,144 +1,160 @@ -#include "config.h" #include "json-to-value.hh" -#include +#include +#include + +using json = nlohmann::json; namespace nix { +// for more information, refer to +// https://github.com/nlohmann/json/blob/master/include/nlohmann/detail/input/json_sax.hpp +class JSONSax : nlohmann::json_sax { + class JSONState { + protected: + std::unique_ptr parent; + RootValue v; + public: + virtual std::unique_ptr resolve(EvalState &) + { + throw std::logic_error("tried to close toplevel json parser state"); + } + explicit JSONState(std::unique_ptr && p) : parent(std::move(p)) {} + explicit JSONState(Value * v) : v(allocRootValue(v)) {} + JSONState(JSONState & p) = delete; + Value & value(EvalState & state) + { + if (!v) + v = allocRootValue(state.allocValue()); + return **v; + } + virtual ~JSONState() {} + virtual void add() {} + }; -static void skipWhitespace(const char * & s) -{ - while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; -} + class JSONObjectState : public JSONState { + using JSONState::JSONState; + ValueMap attrs; + std::unique_ptr resolve(EvalState & state) override + { + Value & v = parent->value(state); + state.mkAttrs(v, attrs.size()); + for (auto & i : attrs) + v.attrs->push_back(Attr(i.first, i.second)); + return std::move(parent); + } + void add() override { v = nullptr; } + public: + void key(string_t & name, EvalState & state) + { + attrs.insert_or_assign(state.symbols.create(name), &value(state)); + } + }; + class JSONListState : public JSONState { + ValueVector values; + std::unique_ptr resolve(EvalState & state) override + { + Value & v = parent->value(state); + state.mkList(v, values.size()); + for (size_t n = 0; n < values.size(); ++n) { + v.listElems()[n] = values[n]; + } + return std::move(parent); + } + void add() override { + values.push_back(*v); + v = nullptr; + } + public: + JSONListState(std::unique_ptr && p, std::size_t reserve) : JSONState(std::move(p)) + { + values.reserve(reserve); + } + }; -static string parseJSONString(const char * & s) -{ - string res; - if (*s++ != '"') throw JSONParseError("expected JSON string"); - while (*s != '"') { - if (!*s) throw JSONParseError("got end-of-string in JSON string"); - if (*s == '\\') { - s++; - if (*s == '"') res += '"'; - else if (*s == '\\') res += '\\'; - else if (*s == '/') res += '/'; - else if (*s == '/') res += '/'; - else if (*s == 'b') res += '\b'; - else if (*s == 'f') res += '\f'; - else if (*s == 'n') res += '\n'; - else if (*s == 'r') res += '\r'; - else if (*s == 't') res += '\t'; - else if (*s == 'u') throw JSONParseError("\\u characters in JSON strings are currently not supported"); - else throw JSONParseError("invalid escaped character in JSON string"); - s++; - } else - res += *s++; - } - s++; - return res; -} + EvalState & state; + std::unique_ptr rs; + template inline bool handle_value(T f, Args... args) + { + f(rs->value(state), args...); + rs->add(); + return true; + } -static void parseJSON(EvalState & state, const char * & s, Value & v) -{ - skipWhitespace(s); +public: + JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {}; - if (!*s) throw JSONParseError("expected JSON value"); + bool null() + { + return handle_value(mkNull); + } - if (*s == '[') { - s++; - ValueVector values; - values.reserve(128); - skipWhitespace(s); - while (1) { - if (values.empty() && *s == ']') break; - Value * v2 = state.allocValue(); - parseJSON(state, s, *v2); - values.push_back(v2); - skipWhitespace(s); - if (*s == ']') break; - if (*s != ',') throw JSONParseError("expected ‘,’ or ‘]’ after JSON array element"); - s++; - } - s++; - state.mkList(v, values.size()); - for (size_t n = 0; n < values.size(); ++n) - v.listElems()[n] = values[n]; + bool boolean(bool val) + { + return handle_value(mkBool, val); } - else if (*s == '{') { - s++; - ValueMap attrs; - while (1) { - skipWhitespace(s); - if (attrs.empty() && *s == '}') break; - string name = parseJSONString(s); - skipWhitespace(s); - if (*s != ':') throw JSONParseError("expected ‘:’ in JSON object"); - s++; - Value * v2 = state.allocValue(); - parseJSON(state, s, *v2); - attrs[state.symbols.create(name)] = v2; - skipWhitespace(s); - if (*s == '}') break; - if (*s != ',') throw JSONParseError("expected ‘,’ or ‘}’ after JSON member"); - s++; - } - state.mkAttrs(v, attrs.size()); - for (auto & i : attrs) - v.attrs->push_back(Attr(i.first, i.second)); - v.attrs->sort(); - s++; + bool number_integer(number_integer_t val) + { + return handle_value(mkInt, val); } - else if (*s == '"') { - mkString(v, parseJSONString(s)); + bool number_unsigned(number_unsigned_t val) + { + return handle_value(mkInt, val); } - else if (isdigit(*s) || *s == '-' || *s == '.' ) { - // Buffer into a string first, then use built-in C++ conversions - std::string tmp_number; - ValueType number_type = tInt; + bool number_float(number_float_t val, const string_t & s) + { + return handle_value(mkFloat, val); + } - while (isdigit(*s) || *s == '-' || *s == '.' || *s == 'e' || *s == 'E') { - if (*s == '.' || *s == 'e' || *s == 'E') - number_type = tFloat; - tmp_number += *s++; - } + bool string(string_t & val) + { + return handle_value(mkString, val.c_str()); + } - if (number_type == tFloat) - mkFloat(v, stod(tmp_number)); - else - mkInt(v, stoi(tmp_number)); + bool start_object(std::size_t len) + { + rs = std::make_unique(std::move(rs)); + return true; } - else if (strncmp(s, "true", 4) == 0) { - s += 4; - mkBool(v, true); + bool key(string_t & name) + { + dynamic_cast(rs.get())->key(name, state); + return true; } - else if (strncmp(s, "false", 5) == 0) { - s += 5; - mkBool(v, false); + bool end_object() { + rs = rs->resolve(state); + rs->add(); + return true; } - else if (strncmp(s, "null", 4) == 0) { - s += 4; - mkNull(v); + bool end_array() { + return end_object(); } - else throw JSONParseError("unrecognised JSON value"); -} + bool start_array(size_t len) { + rs = std::make_unique(std::move(rs), + len != std::numeric_limits::max() ? len : 128); + return true; + } + bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) { + throw JSONParseError(ex.what()); + } +}; void parseJSON(EvalState & state, const string & s_, Value & v) { - const char * s = s_.c_str(); - parseJSON(state, s, v); - skipWhitespace(s); - if (*s) throw JSONParseError(format("expected end-of-string while parsing JSON value: %1%") % s); + JSONSax parser(state, v); + bool res = json::sax_parse(s_, &parser); + if (!res) + throw JSONParseError("Invalid JSON Value"); } - } diff --git a/src/libexpr/json-to-value.hh b/src/libexpr/json-to-value.hh index 33f35b16ce8..3b0fdae115e 100644 --- a/src/libexpr/json-to-value.hh +++ b/src/libexpr/json-to-value.hh @@ -6,7 +6,7 @@ namespace nix { -MakeError(JSONParseError, EvalError) +MakeError(JSONParseError, EvalError); void parseJSON(EvalState & state, const string & s, Value & v); diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 5b1ff0350cd..f6e83926bef 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -6,12 +6,14 @@ %option nounput noyy_top_state +%s DEFAULT %x STRING %x IND_STRING -%x INSIDE_DOLLAR_CURLY %{ +#include + #include "nixexpr.hh" #include "parser-tab.hh" @@ -49,9 +51,10 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) } -static Expr * unescapeStr(SymbolTable & symbols, const char * s) +static Expr * unescapeStr(SymbolTable & symbols, const char * s, size_t length) { string t; + t.reserve(length); char c; while ((c = *s++)) { if (c == '\\') { @@ -84,6 +87,7 @@ static Expr * unescapeStr(SymbolTable & symbols, const char * s) %} +ANY .|\n ID [a-zA-Z\_][a-zA-Z0-9\_\'\-]* INT [0-9]+ FLOAT (([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)? @@ -95,8 +99,6 @@ URI [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~ %% -{ - if { return IF; } then { return THEN; } @@ -122,45 +124,58 @@ or { return OR_KW; } {ID} { yylval->id = strdup(yytext); return ID; } {INT} { errno = 0; - yylval->n = strtol(yytext, 0, 10); - if (errno != 0) - throw ParseError(format("invalid integer ‘%1%’") % yytext); + try { + yylval->n = boost::lexical_cast(yytext); + } catch (const boost::bad_lexical_cast &) { + throw ParseError("invalid integer '%1%'", yytext); + } return INT; } {FLOAT} { errno = 0; yylval->nf = strtod(yytext, 0); if (errno != 0) - throw ParseError(format("invalid float ‘%1%’") % yytext); + throw ParseError("invalid float '%1%'", yytext); return FLOAT; } -\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } -} +\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } -\} { return '}'; } -\} { POP_STATE(); return '}'; } -\{ { return '{'; } -\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return '{'; } - -\" { PUSH_STATE(STRING); return '"'; } -([^\$\"\\]|\$[^\{\"\\]|\\.|\$\\.)*\$/\" | -([^\$\"\\]|\$[^\{\"\\]|\\.|\$\\.)+ { - /* It is impossible to match strings ending with '$' with one - regex because trailing contexts are only valid at the end - of a rule. (A sane but undocumented limitation.) */ - yylval->e = unescapeStr(data->symbols, yytext); - return STR; +\} { /* State INITIAL only exists at the bottom of the stack and is + used as a marker. DEFAULT replaces it everywhere else. + Popping when in INITIAL state causes an empty stack exception, + so don't */ + if (YYSTATE != INITIAL) + POP_STATE(); + return '}'; } -\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } -\" { POP_STATE(); return '"'; } -. return yytext[0]; /* just in case: shouldn't be reached */ - -\'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; } +\{ { PUSH_STATE(DEFAULT); return '{'; } + +\" { PUSH_STATE(STRING); return '"'; } +([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})*\$/\" | +([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})+ { + /* It is impossible to match strings ending with '$' with one + regex because trailing contexts are only valid at the end + of a rule. (A sane but undocumented limitation.) */ + yylval->e = unescapeStr(data->symbols, yytext, yyleng); + return STR; + } +\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } +\" { POP_STATE(); return '"'; } +\$|\\|\$\\ { + /* This can only occur when we reach EOF, otherwise the above + (...|\$[^\{\"\\]|\\.|\$\\.)+ would have triggered. + This is technically invalid, but we leave the problem to the + parser who fails with exact location. */ + return STR; + } + +\'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; } ([^\$\']|\$[^\{\']|\'[^\'\$])+ { yylval->e = new ExprIndStr(yytext); return IND_STR; } -\'\'\$ { +\'\'\$ | +\$ { yylval->e = new ExprIndStr("$"); return IND_STR; } @@ -168,27 +183,25 @@ or { return OR_KW; } yylval->e = new ExprIndStr("''"); return IND_STR; } -\'\'\\. { - yylval->e = unescapeStr(data->symbols, yytext + 2); +\'\'\\{ANY} { + yylval->e = unescapeStr(data->symbols, yytext + 2, yyleng - 2); return IND_STR; } -\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } +\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } \'\' { POP_STATE(); return IND_STRING_CLOSE; } \' { yylval->e = new ExprIndStr("'"); return IND_STR; } -. return yytext[0]; /* just in case: shouldn't be reached */ -{ {PATH} { if (yytext[yyleng-1] == '/') - throw ParseError("path ‘%s’ has a trailing slash", yytext); + throw ParseError("path '%s' has a trailing slash", yytext); yylval->path = strdup(yytext); return PATH; } {HPATH} { if (yytext[yyleng-1] == '/') - throw ParseError("path ‘%s’ has a trailing slash", yytext); + throw ParseError("path '%s' has a trailing slash", yytext); yylval->path = strdup(yytext); return HPATH; } @@ -199,11 +212,10 @@ or { return OR_KW; } \#[^\r\n]* /* single-line comments */ \/\*([^*]|\*+[^*/])*\*+\/ /* long comments */ -. return yytext[0]; - -} - -<> { data->atEnd = true; return 0; } +{ANY} { + /* Don't return a negative number, as this will cause + Bison to stop parsing without an error. */ + return (unsigned char) yytext[0]; + } %% - diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 620050a13b0..917e8a1c7df 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -6,9 +6,9 @@ libexpr_DIR := $(d) libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc -libexpr_CXXFLAGS := -Wno-deprecated-register +libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr -libexpr_LIBS = libutil libstore libformat +libexpr_LIBS = libutil libstore libfetchers libnixrust libexpr_LDFLAGS = ifneq ($(OS), FreeBSD) @@ -33,3 +33,5 @@ clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexe dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644)) + +$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh diff --git a/src/libexpr/nix-expr.pc.in b/src/libexpr/nix-expr.pc.in index 21b6c38dd13..80f7a492b1a 100644 --- a/src/libexpr/nix-expr.pc.in +++ b/src/libexpr/nix-expr.pc.in @@ -7,4 +7,4 @@ Description: Nix Package Manager Version: @PACKAGE_VERSION@ Requires: nix-store bdw-gc Libs: -L${libdir} -lnixexpr -Cflags: -I${includedir}/nix +Cflags: -I${includedir}/nix -std=c++17 diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index b2c9f0528ca..b4b65883dbd 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -10,7 +10,7 @@ namespace nix { /* Displaying abstract syntax trees. */ -std::ostream & operator << (std::ostream & str, Expr & e) +std::ostream & operator << (std::ostream & str, const Expr & e) { e.show(str); return str; @@ -58,48 +58,48 @@ std::ostream & operator << (std::ostream & str, const Symbol & sym) return str; } -void Expr::show(std::ostream & str) +void Expr::show(std::ostream & str) const { abort(); } -void ExprInt::show(std::ostream & str) +void ExprInt::show(std::ostream & str) const { str << n; } -void ExprFloat::show(std::ostream & str) +void ExprFloat::show(std::ostream & str) const { str << nf; } -void ExprString::show(std::ostream & str) +void ExprString::show(std::ostream & str) const { showString(str, s); } -void ExprPath::show(std::ostream & str) +void ExprPath::show(std::ostream & str) const { str << s; } -void ExprVar::show(std::ostream & str) +void ExprVar::show(std::ostream & str) const { str << name; } -void ExprSelect::show(std::ostream & str) +void ExprSelect::show(std::ostream & str) const { str << "(" << *e << ")." << showAttrPath(attrPath); if (def) str << " or (" << *def << ")"; } -void ExprOpHasAttr::show(std::ostream & str) +void ExprOpHasAttr::show(std::ostream & str) const { str << "((" << *e << ") ? " << showAttrPath(attrPath) << ")"; } -void ExprAttrs::show(std::ostream & str) +void ExprAttrs::show(std::ostream & str) const { if (recursive) str << "rec "; str << "{ "; @@ -113,7 +113,7 @@ void ExprAttrs::show(std::ostream & str) str << "}"; } -void ExprList::show(std::ostream & str) +void ExprList::show(std::ostream & str) const { str << "[ "; for (auto & i : elems) @@ -121,7 +121,7 @@ void ExprList::show(std::ostream & str) str << "]"; } -void ExprLambda::show(std::ostream & str) +void ExprLambda::show(std::ostream & str) const { str << "("; if (matchAttrs) { @@ -143,7 +143,7 @@ void ExprLambda::show(std::ostream & str) str << ": " << *body << ")"; } -void ExprLet::show(std::ostream & str) +void ExprLet::show(std::ostream & str) const { str << "(let "; for (auto & i : attrs->attrs) @@ -155,27 +155,27 @@ void ExprLet::show(std::ostream & str) str << "in " << *body << ")"; } -void ExprWith::show(std::ostream & str) +void ExprWith::show(std::ostream & str) const { str << "(with " << *attrs << "; " << *body << ")"; } -void ExprIf::show(std::ostream & str) +void ExprIf::show(std::ostream & str) const { str << "(if " << *cond << " then " << *then << " else " << *else_ << ")"; } -void ExprAssert::show(std::ostream & str) +void ExprAssert::show(std::ostream & str) const { str << "assert " << *cond << "; " << *body; } -void ExprOpNot::show(std::ostream & str) +void ExprOpNot::show(std::ostream & str) const { str << "(! " << *e << ")"; } -void ExprConcatStrings::show(std::ostream & str) +void ExprConcatStrings::show(std::ostream & str) const { bool first = true; str << "("; @@ -186,7 +186,7 @@ void ExprConcatStrings::show(std::ostream & str) str << ")"; } -void ExprPos::show(std::ostream & str) +void ExprPos::show(std::ostream & str) const { str << "__curPos"; } @@ -267,8 +267,11 @@ void ExprVar::bindVars(const StaticEnv & env) /* Otherwise, the variable must be obtained from the nearest enclosing `with'. If there is no `with', then we can issue an "undefined variable" error now. */ - if (withLevel == -1) throw UndefinedVarError(format("undefined variable ‘%1%’ at %2%") % name % pos); - + if (withLevel == -1) + throw UndefinedVarError({ + .hint = hintfmt("undefined variable '%1%'", name), + .nixCode = NixCode { .errPos = pos } + }); fromWith = true; this->level = withLevel; } @@ -419,7 +422,7 @@ void ExprLambda::setName(Symbol & name) string ExprLambda::showNamePos() const { - return (format("%1% at %2%") % (name.set() ? "‘" + (string) name + "’" : "anonymous function") % pos).str(); + return (format("%1% at %2%") % (name.set() ? "'" + (string) name + "'" : "anonymous function") % pos).str(); } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index d2ca09b3a5b..ec6fd3190be 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -2,6 +2,7 @@ #include "value.hh" #include "symbol-table.hh" +#include "error.hh" #include @@ -9,15 +10,14 @@ namespace nix { -MakeError(EvalError, Error) -MakeError(ParseError, Error) -MakeError(IncompleteParseError, ParseError) -MakeError(AssertionError, EvalError) -MakeError(ThrownError, AssertionError) -MakeError(Abort, EvalError) -MakeError(TypeError, EvalError) -MakeError(UndefinedVarError, Error) -MakeError(RestrictedPathError, Error) +MakeError(EvalError, Error); +MakeError(ParseError, Error); +MakeError(AssertionError, EvalError); +MakeError(ThrownError, AssertionError); +MakeError(Abort, EvalError); +MakeError(TypeError, EvalError); +MakeError(UndefinedVarError, Error); +MakeError(RestrictedPathError, Error); /* Position objects. */ @@ -76,17 +76,17 @@ string showAttrPath(const AttrPath & attrPath); struct Expr { virtual ~Expr() { }; - virtual void show(std::ostream & str); + virtual void show(std::ostream & str) const; virtual void bindVars(const StaticEnv & env); virtual void eval(EvalState & state, Env & env, Value & v); virtual Value * maybeThunk(EvalState & state, Env & env); virtual void setName(Symbol & name); }; -std::ostream & operator << (std::ostream & str, Expr & e); +std::ostream & operator << (std::ostream & str, const Expr & e); #define COMMON_METHODS \ - void show(std::ostream & str); \ + void show(std::ostream & str) const; \ void eval(EvalState & state, Env & env, Value & v); \ void bindVars(const StaticEnv & env); @@ -210,9 +210,10 @@ struct ExprList : Expr struct Formal { + Pos pos; Symbol name; Expr * def; - Formal(const Symbol & name, Expr * def) : name(name), def(def) { }; + Formal(const Pos & pos, const Symbol & name, Expr * def) : pos(pos), name(name), def(def) { }; }; struct Formals @@ -235,8 +236,10 @@ struct ExprLambda : Expr : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body) { if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end()) - throw ParseError(format("duplicate formal function argument ‘%1%’ at %2%") - % arg % pos); + throw ParseError({ + .hint = hintfmt("duplicate formal function argument '%1%'", arg), + .nixCode = NixCode { .errPos = pos } + }); }; void setName(Symbol & name); string showNamePos() const; @@ -255,15 +258,16 @@ struct ExprWith : Expr { Pos pos; Expr * attrs, * body; - unsigned int prevWith; + size_t prevWith; ExprWith(const Pos & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { }; COMMON_METHODS }; struct ExprIf : Expr { + Pos pos; Expr * cond, * then, * else_; - ExprIf(Expr * cond, Expr * then, Expr * else_) : cond(cond), then(then), else_(else_) { }; + ExprIf(const Pos & pos, Expr * cond, Expr * then, Expr * else_) : pos(pos), cond(cond), then(then), else_(else_) { }; COMMON_METHODS }; @@ -283,13 +287,13 @@ struct ExprOpNot : Expr }; #define MakeBinOp(name, s) \ - struct Expr##name : Expr \ + struct name : Expr \ { \ Pos pos; \ Expr * e1, * e2; \ - Expr##name(Expr * e1, Expr * e2) : e1(e1), e2(e2) { }; \ - Expr##name(const Pos & pos, Expr * e1, Expr * e2) : pos(pos), e1(e1), e2(e2) { }; \ - void show(std::ostream & str) \ + name(Expr * e1, Expr * e2) : e1(e1), e2(e2) { }; \ + name(const Pos & pos, Expr * e1, Expr * e2) : pos(pos), e1(e1), e2(e2) { }; \ + void show(std::ostream & str) const \ { \ str << "(" << *e1 << " " s " " << *e2 << ")"; \ } \ @@ -300,14 +304,14 @@ struct ExprOpNot : Expr void eval(EvalState & state, Env & env, Value & v); \ }; -MakeBinOp(App, "") -MakeBinOp(OpEq, "==") -MakeBinOp(OpNEq, "!=") -MakeBinOp(OpAnd, "&&") -MakeBinOp(OpOr, "||") -MakeBinOp(OpImpl, "->") -MakeBinOp(OpUpdate, "//") -MakeBinOp(OpConcatLists, "++") +MakeBinOp(ExprApp, "") +MakeBinOp(ExprOpEq, "==") +MakeBinOp(ExprOpNEq, "!=") +MakeBinOp(ExprOpAnd, "&&") +MakeBinOp(ExprOpOr, "||") +MakeBinOp(ExprOpImpl, "->") +MakeBinOp(ExprOpUpdate, "//") +MakeBinOp(ExprOpConcatLists, "++") struct ExprConcatStrings : Expr { diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index d07eedddaf6..a639be64e3e 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -1,7 +1,7 @@ %glr-parser -%pure-parser +%define api.pure %locations -%error-verbose +%define parse.error verbose %defines /* %no-lines */ %parse-param { void * scanner } @@ -20,6 +20,7 @@ #include "nixexpr.hh" #include "eval.hh" +#include "globals.hh" namespace nix { @@ -30,13 +31,11 @@ namespace nix { Expr * result; Path basePath; Symbol path; - string error; - bool atEnd; + ErrorInfo error; Symbol sLetBody; ParseData(EvalState & state) : state(state) , symbols(state.symbols) - , atEnd(false) , sLetBody(symbols.create("")) { }; }; @@ -65,15 +64,20 @@ namespace nix { static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos) { - throw ParseError(format("attribute ‘%1%’ at %2% already defined at %3%") - % showAttrPath(attrPath) % pos % prevPos); + throw ParseError({ + .hint = hintfmt("attribute '%1%' already defined at %2%", + showAttrPath(attrPath), prevPos), + .nixCode = NixCode { .errPos = pos }, + }); } static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos) { - throw ParseError(format("attribute ‘%1%’ at %2% already defined at %3%") - % attr % pos % prevPos); + throw ParseError({ + .hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos), + .nixCode = NixCode { .errPos = pos }, + }); } @@ -83,6 +87,8 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, AttrPath::iterator i; // All attrpaths have at least one attr assert(!attrPath.empty()); + // Checking attrPath validity. + // =========================== for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { if (i->symbol.set()) { ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); @@ -104,11 +110,29 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, attrs = nested; } } + // Expr insertion. + // ========================== if (i->symbol.set()) { ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); if (j != attrs->attrs.end()) { - dupAttr(attrPath, pos, j->second.pos); + // This attr path is already defined. However, if both + // e and the expr pointed by the attr path are two attribute sets, + // we want to merge them. + // Otherwise, throw an error. + auto ae = dynamic_cast(e); + auto jAttrs = dynamic_cast(j->second.e); + if (jAttrs && ae) { + for (auto & ad : ae->attrs) { + auto j2 = jAttrs->attrs.find(ad.first); + if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. + dupAttr(ad.first, j2->second.pos, ad.second.pos); + jAttrs->attrs[ad.first] = ad.second; + } + } else { + dupAttr(attrPath, pos, j->second.pos); + } } else { + // This attr path is not defined. Let's create it. attrs->attrs[i->symbol] = ExprAttrs::AttrDef(e, pos); e->setName(i->symbol); } @@ -120,11 +144,13 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, static void addFormal(const Pos & pos, Formals * formals, const Formal & formal) { - if (formals->argNames.find(formal.name) != formals->argNames.end()) - throw ParseError(format("duplicate formal function argument ‘%1%’ at %2%") - % formal.name % pos); + if (!formals->argNames.insert(formal.name).second) + throw ParseError({ + .hint = hintfmt("duplicate formal function argument '%1%'", + formal.name), + .nixCode = NixCode { .errPos = pos }, + }); formals->formals.push_front(formal); - formals->argNames.insert(formal.name); } @@ -136,8 +162,8 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector(i); if (!e) { @@ -148,7 +174,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vectors.size(); ++j) { + for (size_t j = 0; j < e->s.size(); ++j) { if (atStartOfLine) { if (e->s[j] == ' ') curIndent++; @@ -170,8 +196,8 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector * es2 = new vector; atStartOfLine = true; - unsigned int curDropped = 0; - unsigned int n = es.size(); + size_t curDropped = 0; + size_t n = es.size(); for (vector::iterator i = es.begin(); i != es.end(); ++i, --n) { ExprIndStr * e = dynamic_cast(*i); if (!e) { @@ -182,7 +208,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vectors.size(); ++j) { + for (size_t j = 0; j < e->s.size(); ++j) { if (atStartOfLine) { if (e->s[j] == ' ') { if (curDropped++ >= minIndent) @@ -231,8 +257,10 @@ static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data) void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error) { - data->error = (format("%1%, at %2%") - % error % makeCurPos(*loc, data)).str(); + data->error = { + .hint = hintfmt(error), + .nixCode = NixCode { .errPos = makeCurPos(*loc, data) } + }; } @@ -275,11 +303,11 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err %token IND_STRING_OPEN IND_STRING_CLOSE %token ELLIPSIS -%nonassoc IMPL +%right IMPL %left OR %left AND %nonassoc EQ NEQ -%left '<' '>' LEQ GEQ +%nonassoc '<' '>' LEQ GEQ %right UPDATE %left NOT %left '+' '-' @@ -309,15 +337,17 @@ expr_function { $$ = new ExprWith(CUR_POS, $2, $4); } | LET binds IN expr_function { if (!$2->dynamicAttrs.empty()) - throw ParseError(format("dynamic attributes not allowed in let at %1%") - % CUR_POS); + throw ParseError({ + .hint = hintfmt("dynamic attributes not allowed in let"), + .nixCode = NixCode { .errPos = CUR_POS }, + }); $$ = new ExprLet($2, $4); } | expr_if ; expr_if - : IF expr THEN expr ELSE expr { $$ = new ExprIf($2, $4, $6); } + : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); } | expr_op ; @@ -376,7 +406,7 @@ expr_simple $$ = stripIndentation(CUR_POS, data->symbols, *$2); } | PATH { $$ = new ExprPath(absPath($1, data->basePath)); } - | HPATH { $$ = new ExprPath(getEnv("HOME", "") + string{$1 + 1}); } + | HPATH { $$ = new ExprPath(getHome() + string{$1 + 1}); } | SPATH { string path($1 + 1, strlen($1) - 2); $$ = new ExprApp(CUR_POS, @@ -384,7 +414,15 @@ expr_simple new ExprVar(data->symbols.create("__nixPath"))), new ExprString(data->symbols.create(path))); } - | URI { $$ = new ExprString(data->symbols.create($1)); } + | URI { + static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals"); + if (noURLLiterals) + throw ParseError({ + .hint = hintfmt("URL literals are disabled"), + .nixCode = NixCode { .errPos = CUR_POS } + }); + $$ = new ExprString(data->symbols.create($1)); + } | '(' expr ')' { $$ = $2; } /* Let expressions `let {..., body = ...}' are just desugared into `(rec {..., body = ...}).body'. */ @@ -452,8 +490,10 @@ attrs $$->push_back(AttrName(str->s)); delete str; } else - throw ParseError(format("dynamic attributes not allowed in inherit at %1%") - % makeCurPos(@2, data)); + throw ParseError({ + .hint = hintfmt("dynamic attributes not allowed in inherit"), + .nixCode = NixCode { .errPos = makeCurPos(@2, data) }, + }); } | { $$ = new AttrPath; } ; @@ -508,8 +548,8 @@ formals ; formal - : ID { $$ = new Formal(data->symbols.create($1), 0); } - | ID '?' expr { $$ = new Formal(data->symbols.create($1), $3); } + : ID { $$ = new Formal(CUR_POS, data->symbols.create($1), 0); } + | ID '?' expr { $$ = new Formal(CUR_POS, data->symbols.create($1), $3); } ; %% @@ -521,9 +561,9 @@ formal #include #include "eval.hh" -#include "download.hh" +#include "filetransfer.hh" +#include "fetchers.hh" #include "store-api.hh" -#include "primops/fetchgit.hh" namespace nix { @@ -542,12 +582,7 @@ Expr * EvalState::parse(const char * text, int res = yyparse(scanner, &data); yylex_destroy(scanner); - if (res) { - if (data.atEnd) - throw IncompleteParseError(data.error); - else - throw ParseError(data.error); - } + if (res) throw ParseError(data.error); data.result->bindVars(staticEnv); @@ -559,12 +594,17 @@ Path resolveExprPath(Path path) { assert(path[0] == '/'); + unsigned int followCount = 0, maxFollow = 1024; + /* If `path' is a symlink, follow it. This is so that relative path references work. */ struct stat st; while (true) { + // Basic cycle/depth limit to avoid infinite loops. + if (++followCount >= maxFollow) + throw Error("too many symbolic links encountered while traversing the path '%s'", path); if (lstat(path.c_str(), &st)) - throw SysError(format("getting status of ‘%1%’") % path); + throw SysError("getting status of '%s'", path); if (!S_ISLNK(st.st_mode)) break; path = absPath(readLink(path), dirOf(path)); } @@ -589,18 +629,25 @@ Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv) } -Expr * EvalState::parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv) +Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv) { - return parse(s.c_str(), "(string)", basePath, staticEnv); + return parse(s.data(), "(string)", basePath, staticEnv); } -Expr * EvalState::parseExprFromString(const string & s, const Path & basePath) +Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath) { return parseExprFromString(s, basePath, staticBaseEnv); } +Expr * EvalState::parseStdin() +{ + //Activity act(*logger, lvlTalkative, format("parsing standard input")); + return parseExprFromString(drainFD(0), absPath(".")); +} + + void EvalState::addToSearchPath(const string & s) { size_t pos = s.find('='); @@ -641,11 +688,13 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos Path res = r.second + suffix; if (pathExists(res)) return canonPath(res); } - format f = format( - "file ‘%1%’ was not found in the Nix search path (add it using $NIX_PATH or -I)" - + string(pos ? ", at %2%" : "")); - f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); - throw ThrownError(f % path % pos); + throw ThrownError({ + .hint = hintfmt(evalSettings.pureEval + ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" + : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)", + path), + .nixCode = NixCode { .errPos = pos } + }); } @@ -658,13 +707,13 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { - if (hasPrefix(elem.second, "git://") || hasSuffix(elem.second, ".git")) - // FIXME: support specifying revision/branch - res = { true, exportGit(store, elem.second, "master") }; - else - res = { true, getDownloader()->downloadCached(store, elem.second, true) }; - } catch (DownloadError & e) { - printError(format("warning: Nix search path entry ‘%1%’ cannot be downloaded, ignoring") % elem.second); + res = { true, store->toRealPath(fetchers::downloadTarball( + store, resolveUri(elem.second), "source", false).storePath) }; + } catch (FileTransferError & e) { + logWarning({ + .name = "Entry download", + .hint = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second) + }); res = { false, "" }; } } else { @@ -672,12 +721,15 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl if (pathExists(path)) res = { true, path }; else { - printError(format("warning: Nix search path entry ‘%1%’ does not exist, ignoring") % elem.second); + logWarning({ + .name = "Entry not found", + .hint = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second) + }); res = { false, "" }; } } - debug(format("resolved search path element ‘%s’ to ‘%s’") % elem.second % res.second); + debug(format("resolved search path element '%s' to '%s'") % elem.second % res.second); searchPathResolved[elem.second] = res; return res; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 59623874c3f..62e5163c902 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1,6 +1,5 @@ #include "archive.hh" #include "derivations.hh" -#include "download.hh" #include "eval-inline.hh" #include "eval.hh" #include "globals.hh" @@ -8,6 +7,7 @@ #include "names.hh" #include "store-api.hh" #include "util.hh" +#include "json.hh" #include "value-to-json.hh" #include "value-to-xml.hh" #include "primops.hh" @@ -38,32 +38,48 @@ std::pair decodeContext(const string & s) size_t index = s.find("!", 1); return std::pair(string(s, index + 1), string(s, 1, index - 1)); } else - return std::pair(s.at(0) == '/' ? s: string(s, 1), ""); + return std::pair(s.at(0) == '/' ? s : string(s, 1), ""); } InvalidPathError::InvalidPathError(const Path & path) : - EvalError(format("path ‘%1%’ is not valid") % path), path(path) {} + EvalError("path '%s' is not valid", path), path(path) {} void EvalState::realiseContext(const PathSet & context) { - PathSet drvs; + std::vector drvs; + for (auto & i : context) { std::pair decoded = decodeContext(i); - Path ctx = decoded.first; - assert(store->isStorePath(ctx)); + auto ctx = store->parseStorePath(decoded.first); if (!store->isValidPath(ctx)) - throw InvalidPathError(ctx); - if (!decoded.second.empty() && nix::isDerivation(ctx)) - drvs.insert(decoded.first + "!" + decoded.second); - } - if (!drvs.empty()) { - /* For performance, prefetch all substitute info. */ - PathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; - store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); - store->buildPaths(drvs); + throw InvalidPathError(store->printStorePath(ctx)); + if (!decoded.second.empty() && ctx.isDerivation()) { + drvs.push_back(StorePathWithOutputs{ctx.clone(), {decoded.second}}); + + /* Add the output of this derivation to the allowed + paths. */ + if (allowedPaths) { + auto drv = store->derivationFromPath(store->parseStorePath(decoded.first)); + DerivationOutputs::iterator i = drv.outputs.find(decoded.second); + if (i == drv.outputs.end()) + throw Error("derivation '%s' does not have an output named '%s'", decoded.first, decoded.second); + allowedPaths->insert(store->printStorePath(i->second.path)); + } + } } + + if (drvs.empty()) return; + + if (!evalSettings.enableImportFromDerivation) + throw EvalError("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false", + store->printStorePath(drvs.begin()->path)); + + /* For performance, prefetch all substitute info. */ + StorePathSet willBuild, willSubstitute, unknown; + unsigned long long downloadSize, narSize; + store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); + store->buildPaths(drvs); } @@ -77,14 +93,17 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError(format("cannot import ‘%1%’, since path ‘%2%’ is not valid, at %3%") - % path % e.path % pos); + throw EvalError({ + .hint = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path), + .nixCode = NixCode { .errPos = pos } + }); } - path = state.checkSourcePath(path); + Path realPath = state.checkSourcePath(state.toRealPath(path, context)); - if (state.store->isStorePath(path) && state.store->isValidPath(path) && isDerivation(path)) { - Derivation drv = readDerivation(path); + // FIXME + if (state.store->isStorePath(path) && state.store->isValidPath(state.store->parseStorePath(path)) && isDerivation(path)) { + Derivation drv = readDerivation(*state.store, realPath); Value & w = *state.allocValue(); state.mkAttrs(w, 3 + drv.outputs.size()); Value * v2 = state.allocAttr(w, state.sDrvPath); @@ -98,20 +117,27 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args for (const auto & o : drv.outputs) { v2 = state.allocAttr(w, state.symbols.create(o.first)); - mkString(*v2, o.second.path, {"!" + o.first + "!" + path}); + mkString(*v2, state.store->printStorePath(o.second.path), {"!" + o.first + "!" + path}); outputsVal->listElems()[outputs_index] = state.allocValue(); mkString(*(outputsVal->listElems()[outputs_index++]), o.first); } w.attrs->sort(); - Value fun; - state.evalFile(settings.nixDataDir + "/nix/corepkgs/imported-drv-to-derivation.nix", fun); - state.forceFunction(fun, pos); - mkApp(v, fun, w); + + static RootValue fun; + if (!fun) { + fun = allocRootValue(state.allocValue()); + state.eval(state.parseExprFromString( + #include "imported-drv-to-derivation.nix.gen.hh" + , "/"), **fun); + } + + state.forceFunction(**fun, pos); + mkApp(v, **fun, w); state.forceAttrs(v, pos); } else { state.forceAttrs(*args[0]); if (args[0]->attrs->empty()) - state.evalFile(path, v); + state.evalFile(realPath, v); else { Env * env = &state.allocEnv(args[0]->attrs->size()); env->up = &state.baseEnv; @@ -124,8 +150,8 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args env->values[displ++] = attr.value; } - Activity act(*logger, lvlTalkative, format("evaluating file ‘%1%’") % path); - Expr * e = state.parseExprFromFile(resolveExprPath(path), staticEnv); + printTalkative("evaluating file '%1%'", realPath); + Expr * e = state.parseExprFromFile(resolveExprPath(realPath), staticEnv); e->eval(state, *env, v); } @@ -138,7 +164,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v); /* Load a ValueInitializer from a DSO and return whatever it initializes */ -static void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v) +void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v) { PathSet context; Path path = state.coerceToPath(pos, *args[0], context); @@ -146,8 +172,12 @@ static void prim_importNative(EvalState & state, const Pos & pos, Value * * args try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError(format("cannot import ‘%1%’, since path ‘%2%’ is not valid, at %3%") - % path % e.path % pos); + throw EvalError({ + .hint = hintfmt( + "cannot import '%1%', since path '%2%' is not valid", + path, e.path), + .nixCode = NixCode { .errPos = pos } + }); } path = state.checkSourcePath(path); @@ -156,17 +186,17 @@ static void prim_importNative(EvalState & state, const Pos & pos, Value * * args void *handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL); if (!handle) - throw EvalError(format("could not open ‘%1%’: %2%") % path % dlerror()); + throw EvalError("could not open '%1%': %2%", path, dlerror()); dlerror(); ValueInitializer func = (ValueInitializer) dlsym(handle, sym.c_str()); if(!func) { char *message = dlerror(); if (message) - throw EvalError(format("could not load symbol ‘%1%’ from ‘%2%’: %3%") % sym % path % message); + throw EvalError("could not load symbol '%1%' from '%2%': %3%", sym, path, message); else - throw EvalError(format("symbol ‘%1%’ from ‘%2%’ resolved to NULL when a function pointer was expected") - % sym % path); + throw EvalError("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected", + sym, path); } (func)(state, v); @@ -175,10 +205,55 @@ static void prim_importNative(EvalState & state, const Pos & pos, Value * * args } +/* Execute a program and parse its output */ +void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceList(*args[0], pos); + auto elems = args[0]->listElems(); + auto count = args[0]->listSize(); + if (count == 0) { + throw EvalError({ + .hint = hintfmt("at least one argument to 'exec' required"), + .nixCode = NixCode { .errPos = pos } + }); + } + PathSet context; + auto program = state.coerceToString(pos, *elems[0], context, false, false); + Strings commandArgs; + for (unsigned int i = 1; i < args[0]->listSize(); ++i) { + commandArgs.emplace_back(state.coerceToString(pos, *elems[i], context, false, false)); + } + try { + state.realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError({ + .hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid", + program, e.path), + .nixCode = NixCode { .errPos = pos } + }); + } + + auto output = runProgram(program, true, commandArgs); + Expr * parsed; + try { + parsed = state.parseExprFromString(output, pos.file); + } catch (Error & e) { + e.addPrefix(fmt("While parsing the output from '%1%', at %2%\n", program, pos)); + throw; + } + try { + state.eval(parsed, v); + } catch (Error & e) { + e.addPrefix(fmt("While evaluating the output from '%1%', at %2%\n", program, pos)); + throw; + } +} + + /* Return a string representing the type of the expression. */ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); string t; switch (args[0]->type) { case tInt: t = "int"; break; @@ -206,7 +281,7 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu /* Determine whether the argument is the null value. */ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tNull); } @@ -214,29 +289,40 @@ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Valu /* Determine whether the argument is a function. */ static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); - mkBool(v, args[0]->type == tLambda); + state.forceValue(*args[0], pos); + bool res; + switch (args[0]->type) { + case tLambda: + case tPrimOp: + case tPrimOpApp: + res = true; + break; + default: + res = false; + break; + } + mkBool(v, res); } /* Determine whether the argument is an integer. */ static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tInt); } /* Determine whether the argument is a float. */ static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tFloat); } /* Determine whether the argument is a string. */ static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tString); } @@ -244,10 +330,16 @@ static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Va /* Determine whether the argument is a Boolean. */ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tBool); } +/* Determine whether the argument is a path. */ +static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0], pos); + mkBool(v, args[0]->type == tPath); +} struct CompareValues { @@ -258,7 +350,7 @@ struct CompareValues if (v1->type == tInt && v2->type == tFloat) return v1->integer < v2->fpoint; if (v1->type != v2->type) - throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2)); + throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2)); switch (v1->type) { case tInt: return v1->integer < v2->integer; @@ -269,7 +361,7 @@ struct CompareValues case tPath: return strcmp(v1->path, v2->path) < 0; default: - throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2)); + throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2)); } } }; @@ -284,15 +376,16 @@ typedef list ValueList; static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * args, Value & v) { - Activity act(*logger, lvlDebug, "finding dependencies"); - state.forceAttrs(*args[0], pos); /* Get the start set. */ Bindings::iterator startSet = args[0]->attrs->find(state.symbols.create("startSet")); if (startSet == args[0]->attrs->end()) - throw EvalError(format("attribute ‘startSet’ required, at %1%") % pos); + throw EvalError({ + .hint = hintfmt("attribute 'startSet' required"), + .nixCode = NixCode { .errPos = pos } + }); state.forceList(*startSet->value, pos); ValueList workSet; @@ -303,8 +396,11 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar Bindings::iterator op = args[0]->attrs->find(state.symbols.create("operator")); if (op == args[0]->attrs->end()) - throw EvalError(format("attribute ‘operator’ required, at %1%") % pos); - state.forceValue(*op->value); + throw EvalError({ + .hint = hintfmt("attribute 'operator' required"), + .nixCode = NixCode { .errPos = pos } + }); + state.forceValue(*op->value, pos); /* Construct the closure by applying the operator to element of `workSet', adding the result to `workSet', continuing until @@ -322,11 +418,13 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar Bindings::iterator key = e->attrs->find(state.symbols.create("key")); if (key == e->attrs->end()) - throw EvalError(format("attribute ‘key’ required, at %1%") % pos); - state.forceValue(*key->value); + throw EvalError({ + .hint = hintfmt("attribute 'key' required"), + .nixCode = NixCode { .errPos = pos } + }); + state.forceValue(*key->value, pos); - if (doneKeys.find(key->value) != doneKeys.end()) continue; - doneKeys.insert(key->value); + if (!doneKeys.insert(key->value).second) continue; res.push_back(e); /* Call the `operator' function with `e' as argument. */ @@ -336,7 +434,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar /* Add the values returned by the operator to the work set. */ for (unsigned int n = 0; n < call.listSize(); ++n) { - state.forceValue(*call.listElems()[n]); + state.forceValue(*call.listElems()[n], pos); workSet.push_back(call.listElems()[n]); } } @@ -353,7 +451,7 @@ static void prim_abort(EvalState & state, const Pos & pos, Value * * args, Value { PathSet context; string s = state.coerceToString(pos, *args[0], context); - throw Abort(format("evaluation aborted with the following error message: ‘%1%’") % s); + throw Abort("evaluation aborted with the following error message: '%1%'", s); } @@ -368,7 +466,7 @@ static void prim_throw(EvalState & state, const Pos & pos, Value * * args, Value static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * args, Value & v) { try { - state.forceValue(*args[1]); + state.forceValue(*args[1], pos); v = *args[1]; } catch (Error & e) { PathSet context; @@ -384,7 +482,7 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val { state.mkAttrs(v, 2); try { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); v.attrs->push_back(Attr(state.sValue, args[0])); mkBool(*state.allocAttr(v, state.symbols.create("success")), true); } catch (AssertionError & e) { @@ -399,15 +497,15 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v) { string name = state.forceStringNoCtx(*args[0], pos); - mkString(v, state.restricted ? "" : getEnv(name)); + mkString(v, evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name).value_or("")); } /* Evaluate the first argument, then return the second argument. */ static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); - state.forceValue(*args[1]); + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); v = *args[1]; } @@ -417,7 +515,7 @@ static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value & static void prim_deepSeq(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValueDeep(*args[0]); - state.forceValue(*args[1]); + state.forceValue(*args[1], pos); v = *args[1]; } @@ -426,23 +524,16 @@ static void prim_deepSeq(EvalState & state, const Pos & pos, Value * * args, Val return the second expression. Useful for debugging. */ static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); if (args[0]->type == tString) - printError(format("trace: %1%") % args[0]->string.s); + printError("trace: %1%", args[0]->string.s); else - printError(format("trace: %1%") % *args[0]); - state.forceValue(*args[1]); + printError("trace: %1%", *args[0]); + state.forceValue(*args[1], pos); v = *args[1]; } -void prim_valueSize(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - /* We're not forcing the argument on purpose. */ - mkInt(v, valueSize(*args[0])); -} - - /************************************************************* * Derivations *************************************************************/ @@ -457,23 +548,31 @@ void prim_valueSize(EvalState & state, const Pos & pos, Value * * args, Value & derivation. */ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * args, Value & v) { - Activity act(*logger, lvlVomit, "evaluating derivation"); - state.forceAttrs(*args[0], pos); /* Figure out the name first (for stack backtraces). */ Bindings::iterator attr = args[0]->attrs->find(state.sName); if (attr == args[0]->attrs->end()) - throw EvalError(format("required attribute ‘name’ missing, at %1%") % pos); + throw EvalError({ + .hint = hintfmt("required attribute 'name' missing"), + .nixCode = NixCode { .errPos = pos } + }); string drvName; Pos & posDrvName(*attr->pos); try { drvName = state.forceStringNoCtx(*attr->value, pos); } catch (Error & e) { - e.addPrefix(format("while evaluating the derivation attribute ‘name’ at %1%:\n") % posDrvName); + e.addPrefix(fmt("while evaluating the derivation attribute 'name' at %1%:\n", posDrvName)); throw; } + /* Check whether attributes should be passed as a JSON file. */ + std::ostringstream jsonBuf; + std::unique_ptr jsonObject; + attr = args[0]->attrs->find(state.sStructuredAttrs); + if (attr != args[0]->attrs->end() && state.forceBool(*attr->value, pos)) + jsonObject = std::make_unique(jsonBuf); + /* Check whether null attributes should be ignored. */ bool ignoreNulls = false; attr = args[0]->attrs->find(state.sIgnoreNulls); @@ -485,30 +584,68 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * PathSet context; - string outputHash, outputHashAlgo; - bool outputHashRecursive = false; + std::optional outputHash; + std::string outputHashAlgo; + auto ingestionMethod = FileIngestionMethod::Flat; StringSet outputs; outputs.insert("out"); - for (auto & i : *args[0]->attrs) { - if (i.name == state.sIgnoreNulls) continue; - string key = i.name; - Activity act(*logger, lvlVomit, format("processing attribute ‘%1%’") % key); + for (auto & i : args[0]->attrs->lexicographicOrder()) { + if (i->name == state.sIgnoreNulls) continue; + const string & key = i->name; + vomit("processing attribute '%1%'", key); + + auto handleHashMode = [&](const std::string & s) { + if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive; + else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat; + else + throw EvalError({ + .hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), + .nixCode = NixCode { .errPos = posDrvName } + }); + }; + + auto handleOutputs = [&](const Strings & ss) { + outputs.clear(); + for (auto & j : ss) { + if (outputs.find(j) != outputs.end()) + throw EvalError({ + .hint = hintfmt("duplicate derivation output '%1%'", j), + .nixCode = NixCode { .errPos = posDrvName } + }); + /* !!! Check whether j is a valid attribute + name. */ + /* Derivations cannot be named ‘drv’, because + then we'd have an attribute ‘drvPath’ in + the resulting set. */ + if (j == "drv") + throw EvalError({ + .hint = hintfmt("invalid derivation output name 'drv'" ), + .nixCode = NixCode { .errPos = posDrvName } + }); + outputs.insert(j); + } + if (outputs.empty()) + throw EvalError({ + .hint = hintfmt("derivation cannot have an empty set of outputs"), + .nixCode = NixCode { .errPos = posDrvName } + }); + }; try { if (ignoreNulls) { - state.forceValue(*i.value); - if (i.value->type == tNull) continue; + state.forceValue(*i->value, pos); + if (i->value->type == tNull) continue; } /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ - if (key == "args") { - state.forceList(*i.value, pos); - for (unsigned int n = 0; n < i.value->listSize(); ++n) { - string s = state.coerceToString(posDrvName, *i.value->listElems()[n], context, true); + if (i->name == state.sArgs) { + state.forceList(*i->value, pos); + for (unsigned int n = 0; n < i->value->listSize(); ++n) { + string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true); drv.args.push_back(s); } } @@ -516,48 +653,59 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* All other attributes are passed to the builder through the environment. */ else { - string s = state.coerceToString(posDrvName, *i.value, context, true); - drv.env[key] = s; - if (key == "builder") drv.builder = s; - else if (i.name == state.sSystem) drv.platform = s; - else if (i.name == state.sName) { - drvName = s; - printMsg(lvlVomit, format("derivation name is ‘%1%’") % drvName); - } - else if (key == "outputHash") outputHash = s; - else if (key == "outputHashAlgo") outputHashAlgo = s; - else if (key == "outputHashMode") { - if (s == "recursive") outputHashRecursive = true; - else if (s == "flat") outputHashRecursive = false; - else throw EvalError(format("invalid value ‘%1%’ for ‘outputHashMode’ attribute, at %2%") % s % posDrvName); - } - else if (key == "outputs") { - Strings tmp = tokenizeString(s); - outputs.clear(); - for (auto & j : tmp) { - if (outputs.find(j) != outputs.end()) - throw EvalError(format("duplicate derivation output ‘%1%’, at %2%") % j % posDrvName); - /* !!! Check whether j is a valid attribute - name. */ - /* Derivations cannot be named ‘drv’, because - then we'd have an attribute ‘drvPath’ in - the resulting set. */ - if (j == "drv") - throw EvalError(format("invalid derivation output name ‘drv’, at %1%") % posDrvName); - outputs.insert(j); + + if (jsonObject) { + + if (i->name == state.sStructuredAttrs) continue; + + auto placeholder(jsonObject->placeholder(key)); + printValueAsJSON(state, true, *i->value, placeholder, context); + + if (i->name == state.sBuilder) + drv.builder = state.forceString(*i->value, context, posDrvName); + else if (i->name == state.sSystem) + drv.platform = state.forceStringNoCtx(*i->value, posDrvName); + else if (i->name == state.sOutputHash) + outputHash = state.forceStringNoCtx(*i->value, posDrvName); + else if (i->name == state.sOutputHashAlgo) + outputHashAlgo = state.forceStringNoCtx(*i->value, posDrvName); + else if (i->name == state.sOutputHashMode) + handleHashMode(state.forceStringNoCtx(*i->value, posDrvName)); + else if (i->name == state.sOutputs) { + /* Require ‘outputs’ to be a list of strings. */ + state.forceList(*i->value, posDrvName); + Strings ss; + for (unsigned int n = 0; n < i->value->listSize(); ++n) + ss.emplace_back(state.forceStringNoCtx(*i->value->listElems()[n], posDrvName)); + handleOutputs(ss); } - if (outputs.empty()) - throw EvalError(format("derivation cannot have an empty set of outputs, at %1%") % posDrvName); + + } else { + auto s = state.coerceToString(posDrvName, *i->value, context, true); + drv.env.emplace(key, s); + if (i->name == state.sBuilder) drv.builder = s; + else if (i->name == state.sSystem) drv.platform = s; + else if (i->name == state.sOutputHash) outputHash = s; + else if (i->name == state.sOutputHashAlgo) outputHashAlgo = s; + else if (i->name == state.sOutputHashMode) handleHashMode(s); + else if (i->name == state.sOutputs) + handleOutputs(tokenizeString(s)); } + } } catch (Error & e) { - e.addPrefix(format("while evaluating the attribute ‘%1%’ of the derivation ‘%2%’ at %3%:\n") + e.addPrefix(format("while evaluating the attribute '%1%' of the derivation '%2%' at %3%:\n") % key % drvName % posDrvName); throw; } } + if (jsonObject) { + jsonObject.reset(); + drv.env.emplace("__json", jsonBuf.str()); + } + /* Everything in the context of the strings in the derivation attributes should be added as dependencies of the resulting derivation. */ @@ -572,103 +720,108 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * runs. */ if (path.at(0) == '=') { /* !!! This doesn't work if readOnlyMode is set. */ - PathSet refs; - state.store->computeFSClosure(string(path, 1), refs); + StorePathSet refs; + state.store->computeFSClosure(state.store->parseStorePath(std::string_view(path).substr(1)), refs); for (auto & j : refs) { - drv.inputSrcs.insert(j); - if (isDerivation(j)) - drv.inputDrvs[j] = state.store->queryDerivationOutputNames(j); + drv.inputSrcs.insert(j.clone()); + if (j.isDerivation()) + drv.inputDrvs[j.clone()] = state.store->readDerivation(j).outputNames(); } } - /* See prim_unsafeDiscardOutputDependency. */ - else if (path.at(0) == '~') - drv.inputSrcs.insert(string(path, 1)); - /* Handle derivation outputs of the form ‘!!’. */ else if (path.at(0) == '!') { std::pair ctx = decodeContext(path); - drv.inputDrvs[ctx.first].insert(ctx.second); + drv.inputDrvs[state.store->parseStorePath(ctx.first)].insert(ctx.second); } - /* Handle derivation contexts returned by - ‘builtins.storePath’. */ - else if (isDerivation(path)) - drv.inputDrvs[path] = state.store->queryDerivationOutputNames(path); - /* Otherwise it's a source file. */ else - drv.inputSrcs.insert(path); + drv.inputSrcs.insert(state.store->parseStorePath(path)); } /* Do we have all required attributes? */ if (drv.builder == "") - throw EvalError(format("required attribute ‘builder’ missing, at %1%") % posDrvName); + throw EvalError({ + .hint = hintfmt("required attribute 'builder' missing"), + .nixCode = NixCode { .errPos = posDrvName } + }); + if (drv.platform == "") - throw EvalError(format("required attribute ‘system’ missing, at %1%") % posDrvName); + throw EvalError({ + .hint = hintfmt("required attribute 'system' missing"), + .nixCode = NixCode { .errPos = posDrvName } + }); /* Check whether the derivation name is valid. */ - checkStoreName(drvName); if (isDerivation(drvName)) - throw EvalError(format("derivation names are not allowed to end in ‘%1%’, at %2%") - % drvExtension % posDrvName); + throw EvalError({ + .hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension), + .nixCode = NixCode { .errPos = posDrvName } + }); - if (outputHash != "") { + if (outputHash) { /* Handle fixed-output derivations. */ if (outputs.size() != 1 || *(outputs.begin()) != "out") - throw Error(format("multiple outputs are not supported in fixed-output derivations, at %1%") % posDrvName); - - HashType ht = parseHashType(outputHashAlgo); - if (ht == htUnknown) - throw EvalError(format("unknown hash algorithm ‘%1%’, at %2%") % outputHashAlgo % posDrvName); - Hash h = parseHash16or32(ht, outputHash); - outputHash = printHash(h); - if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo; - - Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName); - drv.env["out"] = outPath; - drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, outputHash); + throw Error({ + .hint = hintfmt("multiple outputs are not supported in fixed-output derivations"), + .nixCode = NixCode { .errPos = posDrvName } + }); + + HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo); + + Hash h = newHashAllowEmpty(*outputHash, ht); + + auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); + if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); + drv.outputs.insert_or_assign("out", DerivationOutput { + std::move(outPath), + (ingestionMethod == FileIngestionMethod::Recursive ? "r:" : "") + + printHashType(h.type), + h.to_string(Base16, false), + }); } else { - /* Construct the "masked" store derivation, which is the final - one except that in the list of outputs, the output paths - are empty, and the corresponding environment variables have - an empty value. This ensures that changes in the set of - output names do get reflected in the hash. */ + /* Compute a hash over the "masked" store derivation, which is + the final one except that in the list of outputs, the + output paths are empty strings, and the corresponding + environment variables have an empty value. This ensures + that changes in the set of output names do get reflected in + the hash. */ for (auto & i : outputs) { - drv.env[i] = ""; - drv.outputs[i] = DerivationOutput("", "", ""); + if (!jsonObject) drv.env[i] = ""; + drv.outputs.insert_or_assign(i, + DerivationOutput(StorePath::dummy.clone(), "", "")); } - /* Use the masked derivation expression to compute the output - path. */ - Hash h = hashDerivationModulo(*state.store, drv); + Hash h = hashDerivationModulo(*state.store, Derivation(drv), true); - for (auto & i : drv.outputs) - if (i.second.path == "") { - Path outPath = state.store->makeOutputPath(i.first, h, drvName); - drv.env[i.first] = outPath; - i.second.path = outPath; - } + for (auto & i : outputs) { + auto outPath = state.store->makeOutputPath(i, h, drvName); + if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath); + drv.outputs.insert_or_assign(i, + DerivationOutput(std::move(outPath), "", "")); + } } /* Write the resulting term into the Nix store directory. */ - Path drvPath = writeDerivation(state.store, drv, drvName, state.repair); + auto drvPath = writeDerivation(state.store, drv, drvName, state.repair); + auto drvPathS = state.store->printStorePath(drvPath); - printMsg(lvlChatty, format("instantiated ‘%1%’ -> ‘%2%’") - % drvName % drvPath); + printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS); /* Optimisation, but required in read-only mode! because in that case we don't actually write store derivations, so we can't read them later. */ - drvHashes[drvPath] = hashDerivationModulo(*state.store, drv); + drvHashes.insert_or_assign(drvPath.clone(), + hashDerivationModulo(*state.store, Derivation(drv), false)); state.mkAttrs(v, 1 + drv.outputs.size()); - mkString(*state.allocAttr(v, state.sDrvPath), drvPath, {"=" + drvPath}); + mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS}); for (auto & i : drv.outputs) { mkString(*state.allocAttr(v, state.symbols.create(i.first)), - i.second.path, {"!" + i.first + "!" + drvPath}); + state.store->printStorePath(i.second.path), {"!" + i.first + "!" + drvPathS}); } v.attrs->sort(); } @@ -676,7 +829,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Return a placeholder string for the specified output that will be substituted by the corresponding output path at build time. For - example, ‘placeholder "out"’ returns the string + example, 'placeholder "out"' returns the string /1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9. At build time, any occurence of this string in an derivation attribute will be replaced with the concrete path in the Nix store of the output @@ -718,10 +871,13 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V e.g. nix-push does the right thing. */ if (!state.store->isStorePath(path)) path = canonPath(path, true); if (!state.store->isInStore(path)) - throw EvalError(format("path ‘%1%’ is not in the Nix store, at %2%") % path % pos); + throw EvalError({ + .hint = hintfmt("path '%1%' is not in the Nix store", path), + .nixCode = NixCode { .errPos = pos } + }); Path path2 = state.store->toStorePath(path); if (!settings.readOnlyMode) - state.store->ensurePath(path2); + state.store->ensurePath(state.store->parseStorePath(path2)); context.insert(path2); mkString(v, path, context); } @@ -731,8 +887,17 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, { PathSet context; Path path = state.coerceToPath(pos, *args[0], context); - if (!context.empty()) - throw EvalError(format("string ‘%1%’ cannot refer to other paths, at %2%") % path % pos); + try { + state.realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError({ + .hint = hintfmt( + "cannot check the existence of '%1%', since path '%2%' is not valid", + path, e.path), + .nixCode = NixCode { .errPos = pos } + }); + } + try { mkBool(v, pathExists(state.checkSourcePath(path))); } catch (SysError & e) { @@ -760,7 +925,7 @@ static void prim_baseNameOf(EvalState & state, const Pos & pos, Value * * args, static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value & v) { PathSet context; - Path dir = dirOf(state.coerceToPath(pos, *args[0], context)); + Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false)); if (args[0]->type == tPath) mkPath(v, dir.c_str()); else mkString(v, dir, context); } @@ -773,18 +938,20 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError(format("cannot read ‘%1%’, since path ‘%2%’ is not valid, at %3%") - % path % e.path % pos); + throw EvalError({ + .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), + .nixCode = NixCode { .errPos = pos } + }); } - string s = readFile(state.checkSourcePath(path)); + string s = readFile(state.checkSourcePath(state.toRealPath(path, context))); if (s.find((char) 0) != string::npos) - throw Error(format("the contents of the file ‘%1%’ cannot be represented as a Nix string") % path); + throw Error("the contents of the file '%1%' cannot be represented as a Nix string", path); mkString(v, s.c_str()); } /* Find a file in the Nix search path. Used to implement paths, - which are desugared to ‘findFile __nixPath "x"’. */ + which are desugared to 'findFile __nixPath "x"'. */ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceList(*args[0], pos); @@ -802,7 +969,10 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va i = v2.attrs->find(state.symbols.create("path")); if (i == v2.attrs->end()) - throw EvalError(format("attribute ‘path’ missing, at %1%") % pos); + throw EvalError({ + .hint = hintfmt("attribute 'path' missing"), + .nixCode = NixCode { .errPos = pos } + }); PathSet context; string path = state.coerceToString(pos, *i->value, context, false, false); @@ -810,8 +980,10 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError(format("cannot find ‘%1%’, since path ‘%2%’ is not valid, at %3%") - % path % e.path % pos); + throw EvalError({ + .hint = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path), + .nixCode = NixCode { .errPos = pos } + }); } searchPath.emplace_back(prefix, path); @@ -822,6 +994,23 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str()); } +/* Return the cryptographic hash of a file in base-16. */ +static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + string type = state.forceStringNoCtx(*args[0], pos); + HashType ht = parseHashType(type); + if (ht == htUnknown) + throw Error({ + .hint = hintfmt("unknown hash type '%1%'", type), + .nixCode = NixCode { .errPos = pos } + }); + + PathSet context; // discarded + Path p = state.coerceToPath(pos, *args[1], context); + + mkString(v, hashFile(ht, state.checkSourcePath(p)).to_string(Base16, false), context); +} + /* Read a directory (without . or ..) */ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -830,8 +1019,10 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val try { state.realiseContext(ctx); } catch (InvalidPathError & e) { - throw EvalError(format("cannot read ‘%1%’, since path ‘%2%’ is not valid, at %3%") - % path % e.path % pos); + throw EvalError({ + .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), + .nixCode = NixCode { .errPos = pos } + }); } DirEntries entries = readDirectory(state.checkSourcePath(path)); @@ -897,22 +1088,23 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu string name = state.forceStringNoCtx(*args[0], pos); string contents = state.forceString(*args[1], context, pos); - PathSet refs; + StorePathSet refs; for (auto path : context) { - if (path.at(0) == '=') path = string(path, 1); - if (isDerivation(path)) { - /* See prim_unsafeDiscardOutputDependency. */ - if (path.at(0) != '~') - throw EvalError(format("in ‘toFile’: the file ‘%1%’ cannot refer to derivation outputs, at %2%") % name % pos); - path = string(path, 1); - } - refs.insert(path); + if (path.at(0) != '/') + throw EvalError( { + .hint = hintfmt( + "in 'toFile': the file named '%1%' must not contain a reference " + "to a derivation but contains (%2%)", + name, path), + .nixCode = NixCode { .errPos = pos } + }); + refs.insert(state.store->parseStorePath(path)); } - Path storePath = settings.readOnlyMode + auto storePath = state.store->printStorePath(settings.readOnlyMode ? state.store->computeStorePathForText(name, contents, refs) - : state.store->addTextToStore(name, contents, refs, state.repair); + : state.store->addTextToStore(name, contents, refs, state.repair)); /* Note: we don't need to add `context' to the context of the result, since `storePath' itself has references to the paths @@ -922,22 +1114,14 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu } -struct FilterFromExpr : PathFilter +static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, + Value * filterFun, FileIngestionMethod method, const Hash & expectedHash, Value & v) { - EvalState & state; - Value & filter; - Pos pos; - - FilterFromExpr(EvalState & state, Value & filter, const Pos & pos) - : state(state), filter(filter), pos(pos) - { - } - - bool operator () (const Path & path) - { - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path ‘%1%’") % path); + const auto path = evalSettings.pureEval && expectedHash ? + path_ : + state.checkSourcePath(path_); + PathFilter filter = filterFun ? ([&](const Path & path) { + auto st = lstat(path); /* Call the filter function. The first argument is the path, the second is a string indicating the type of the file. */ @@ -945,7 +1129,7 @@ struct FilterFromExpr : PathFilter mkString(arg1, path); Value fun2; - state.callFunction(filter, arg1, fun2, noPos); + state.callFunction(*filterFun, arg1, fun2, noPos); Value arg2; mkString(arg2, @@ -958,8 +1142,23 @@ struct FilterFromExpr : PathFilter state.callFunction(fun2, arg2, res, noPos); return state.forceBool(res, pos); - } -}; + }) : defaultPathFilter; + + std::optional expectedStorePath; + if (expectedHash) + expectedStorePath = state.store->makeFixedOutputPath(method, expectedHash, name); + Path dstPath; + if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { + dstPath = state.store->printStorePath(settings.readOnlyMode + ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first + : state.store->addToStore(name, path, method, htSHA256, filter, state.repair)); + if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath)) + throw Error("store path mismatch in (possibly filtered) path added from '%s'", path); + } else + dstPath = state.store->printStorePath(*expectedStorePath); + + mkString(v, dstPath, {dstPath}); +} static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -967,21 +1166,66 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args PathSet context; Path path = state.coerceToPath(pos, *args[1], context); if (!context.empty()) - throw EvalError(format("string ‘%1%’ cannot refer to other paths, at %2%") % path % pos); + throw EvalError({ + .hint = hintfmt("string '%1%' cannot refer to other paths", path), + .nixCode = NixCode { .errPos = pos } + }); - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); if (args[0]->type != tLambda) - throw TypeError(format("first argument in call to ‘filterSource’ is not a function but %1%, at %2%") % showType(*args[0]) % pos); + throw TypeError({ + .hint = hintfmt( + "first argument in call to 'filterSource' is not a function but %1%", + showType(*args[0])), + .nixCode = NixCode { .errPos = pos } + }); - FilterFromExpr filter(state, *args[0], pos); + addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v); +} - path = state.checkSourcePath(path); +static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[0], pos); + Path path; + string name; + Value * filterFun = nullptr; + auto method = FileIngestionMethod::Recursive; + Hash expectedHash; - Path dstPath = settings.readOnlyMode - ? state.store->computeStorePathForPath(path, true, htSHA256, filter).first - : state.store->addToStore(baseNameOf(path), path, true, htSHA256, filter, state.repair); + for (auto & attr : *args[0]->attrs) { + const string & n(attr.name); + if (n == "path") { + PathSet context; + path = state.coerceToPath(*attr.pos, *attr.value, context); + if (!context.empty()) + throw EvalError({ + .hint = hintfmt("string '%1%' cannot refer to other paths", path), + .nixCode = NixCode { .errPos = *attr.pos } + }); + } else if (attr.name == state.sName) + name = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "filter") { + state.forceValue(*attr.value, pos); + filterFun = attr.value; + } else if (n == "recursive") + method = FileIngestionMethod { state.forceBool(*attr.value, *attr.pos) }; + else if (n == "sha256") + expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + else + throw EvalError({ + .hint = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name), + .nixCode = NixCode { .errPos = *attr.pos } + }); + } + if (path.empty()) + throw EvalError({ + .hint = hintfmt("'path' required"), + .nixCode = NixCode { .errPos = pos } + }); + if (name.empty()) + name = baseNameOf(path); - mkString(v, dstPath, {dstPath}); + addPath(state, pos, name, path, filterFun, method, expectedHash, v); } @@ -998,12 +1242,12 @@ static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, V state.mkList(v, args[0]->attrs->size()); - unsigned int n = 0; + size_t n = 0; for (auto & i : *args[0]->attrs) mkString(*(v.listElems()[n++] = state.allocValue()), i.name); std::sort(v.listElems(), v.listElems() + n, - [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; }); + [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; }); } @@ -1035,10 +1279,13 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) // !!! Should we create a symbol here or just do a lookup? Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); if (i == args[1]->attrs->end()) - throw EvalError(format("attribute ‘%1%’ missing, at %2%") % attr % pos); + throw EvalError({ + .hint = hintfmt("attribute '%1%' missing", attr), + .nixCode = NixCode { .errPos = pos } + }); // !!! add to stack trace? if (state.countCalls && i->pos) state.attrSelects[*i->pos]++; - state.forceValue(*i->value); + state.forceValue(*i->value, pos); v = *i->value; } @@ -1068,7 +1315,7 @@ static void prim_hasAttr(EvalState & state, const Pos & pos, Value * * args, Val /* Determine whether the argument is a set. */ static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tAttrs); } @@ -1115,17 +1362,21 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, Bindings::iterator j = v2.attrs->find(state.sName); if (j == v2.attrs->end()) - throw TypeError(format("‘name’ attribute missing in a call to ‘listToAttrs’, at %1%") % pos); + throw TypeError({ + .hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"), + .nixCode = NixCode { .errPos = pos } + }); string name = state.forceStringNoCtx(*j->value, pos); Symbol sym = state.symbols.create(name); - if (seen.find(sym) == seen.end()) { + if (seen.insert(sym).second) { Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue)); if (j2 == v2.attrs->end()) - throw TypeError(format("‘value’ attribute missing in a call to ‘listToAttrs’, at %1%") % pos); - + throw TypeError({ + .hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"), + .nixCode = NixCode { .errPos = pos } + }); v.attrs->push_back(Attr(sym, j2->value, j2->pos)); - seen.insert(sym); } } @@ -1195,9 +1446,12 @@ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Va */ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); if (args[0]->type != tLambda) - throw TypeError(format("‘functionArgs’ requires a function, at %1%") % pos); + throw TypeError({ + .hint = hintfmt("'functionArgs' requires a function"), + .nixCode = NixCode { .errPos = pos } + }); if (!args[0]->lambda.fun->matchAttrs) { state.mkAttrs(v, 0); @@ -1205,13 +1459,34 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args } state.mkAttrs(v, args[0]->lambda.fun->formals->formals.size()); - for (auto & i : args[0]->lambda.fun->formals->formals) + for (auto & i : args[0]->lambda.fun->formals->formals) { // !!! should optimise booleans (allocate only once) - mkBool(*state.allocAttr(v, i.name), i.def); + Value * value = state.allocValue(); + v.attrs->push_back(Attr(i.name, value, &i.pos)); + mkBool(*value, i.def); + } v.attrs->sort(); } +/* Apply a function to every element of an attribute set. */ +static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[1], pos); + + state.mkAttrs(v, args[1]->attrs->size()); + + for (auto & i : *args[1]->attrs) { + Value * vName = state.allocValue(); + Value * vFun2 = state.allocValue(); + mkString(*vName, i.name); + mkApp(*vFun2, *args[0], *vName); + mkApp(*state.allocAttr(v, i.name), *vFun2, *i.value); + } +} + + + /************************************************************* * Lists *************************************************************/ @@ -1220,7 +1495,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args /* Determine whether the argument is a list. */ static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->isList()); } @@ -1229,8 +1504,11 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu { state.forceList(list, pos); if (n < 0 || (unsigned int) n >= list.listSize()) - throw Error(format("list index %1% is out of bounds, at %2%") % n % pos); - state.forceValue(*list.listElems()[n]); + throw Error({ + .hint = hintfmt("list index %1% is out of bounds", n), + .nixCode = NixCode { .errPos = pos } + }); + state.forceValue(*list.listElems()[n], pos); v = *list.listElems()[n]; } @@ -1256,7 +1534,11 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value { state.forceList(*args[0], pos); if (args[0]->listSize() == 0) - throw Error(format("‘tail’ called on an empty list, at %1%") % pos); + throw Error({ + .hint = hintfmt("'tail' called on an empty list"), + .nixCode = NixCode { .errPos = pos } + }); + state.mkList(v, args[0]->listSize() - 1); for (unsigned int n = 0; n < v.listSize(); ++n) v.listElems()[n] = args[0]->listElems()[n + 1]; @@ -1266,7 +1548,6 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value /* Apply a function to every element of a list. */ static void prim_map(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); state.forceList(*args[1], pos); state.mkList(v, args[1]->listSize()); @@ -1345,19 +1626,20 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args, state.forceFunction(*args[0], pos); state.forceList(*args[2], pos); - Value * vCur = args[1]; + if (args[2]->listSize()) { + Value * vCur = args[1]; - if (args[2]->listSize()) for (unsigned int n = 0; n < args[2]->listSize(); ++n) { Value vTmp; state.callFunction(*args[0], *vCur, vTmp, pos); vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue(); state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos); } - else - v = *vCur; - - state.forceValue(v); + state.forceValue(v, pos); + } else { + state.forceValue(*args[1], pos); + v = *args[1]; + } } @@ -1394,11 +1676,13 @@ static void prim_all(EvalState & state, const Pos & pos, Value * * args, Value & static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); auto len = state.forceInt(*args[1], pos); if (len < 0) - throw EvalError(format("cannot create list of size %1%, at %2%") % len % pos); + throw EvalError({ + .hint = hintfmt("cannot create list of size %1%", len), + .nixCode = NixCode { .errPos = pos } + }); state.mkList(v, len); @@ -1421,7 +1705,7 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value auto len = args[1]->listSize(); state.mkList(v, len); for (unsigned int n = 0; n < len; ++n) { - state.forceValue(*args[1]->listElems()[n]); + state.forceValue(*args[1]->listElems()[n], pos); v.listElems()[n] = args[1]->listElems()[n]; } @@ -1456,7 +1740,7 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V for (unsigned int n = 0; n < len; ++n) { auto vElem = args[1]->listElems()[n]; - state.forceValue(*vElem); + state.forceValue(*vElem, pos); Value res; state.callFunction(*args[0], *vElem, res, pos); if (state.forceBool(res, pos)) @@ -1468,17 +1752,50 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V state.mkAttrs(v, 2); Value * vRight = state.allocAttr(v, state.sRight); - state.mkList(*vRight, right.size()); - memcpy(vRight->listElems(), right.data(), sizeof(Value *) * right.size()); + auto rsize = right.size(); + state.mkList(*vRight, rsize); + if (rsize) + memcpy(vRight->listElems(), right.data(), sizeof(Value *) * rsize); Value * vWrong = state.allocAttr(v, state.sWrong); - state.mkList(*vWrong, wrong.size()); - memcpy(vWrong->listElems(), wrong.data(), sizeof(Value *) * wrong.size()); + auto wsize = wrong.size(); + state.mkList(*vWrong, wsize); + if (wsize) + memcpy(vWrong->listElems(), wrong.data(), sizeof(Value *) * wsize); v.attrs->sort(); } +/* concatMap = f: list: concatLists (map f list); */ +/* C++-version is to avoid allocating `mkApp', call `f' eagerly */ +static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + auto nrLists = args[1]->listSize(); + + Value lists[nrLists]; + size_t len = 0; + + for (unsigned int n = 0; n < nrLists; ++n) { + Value * vElem = args[1]->listElems()[n]; + state.callFunction(*args[0], *vElem, lists[n], pos); + state.forceList(lists[n], pos); + len += lists[n].listSize(); + } + + state.mkList(v, len); + auto out = v.listElems(); + for (unsigned int n = 0, pos = 0; n < nrLists; ++n) { + auto l = lists[n].listSize(); + if (l) + memcpy(out + pos, lists[n].listElems(), l * sizeof(Value *)); + pos += l; + } +} + + /************************************************************* * Integer arithmetic *************************************************************/ @@ -1486,6 +1803,8 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value & v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); if (args[0]->type == tFloat || args[1]->type == tFloat) mkFloat(v, state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos)); else @@ -1495,6 +1814,8 @@ static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value & static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); if (args[0]->type == tFloat || args[1]->type == tFloat) mkFloat(v, state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos)); else @@ -1504,6 +1825,8 @@ static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); if (args[0]->type == tFloat || args[1]->type == tFloat) mkFloat(v, state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos)); else @@ -1513,8 +1836,15 @@ static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); + NixFloat f2 = state.forceFloat(*args[1], pos); - if (f2 == 0) throw EvalError(format("division by zero, at %1%") % pos); + if (f2 == 0) + throw EvalError({ + .hint = hintfmt("division by zero"), + .nixCode = NixCode { .errPos = pos } + }); if (args[0]->type == tFloat || args[1]->type == tFloat) { mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); @@ -1523,16 +1853,34 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & NixInt i2 = state.forceInt(*args[1], pos); /* Avoid division overflow as it might raise SIGFPE. */ if (i1 == std::numeric_limits::min() && i2 == -1) - throw EvalError(format("overflow in integer division, at %1%") % pos); + throw EvalError({ + .hint = hintfmt("overflow in integer division"), + .nixCode = NixCode { .errPos = pos } + }); + mkInt(v, i1 / i2); } } +static void prim_bitAnd(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + mkInt(v, state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos)); +} + +static void prim_bitOr(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + mkInt(v, state.forceInt(*args[0], pos) | state.forceInt(*args[1], pos)); +} + +static void prim_bitXor(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + mkInt(v, state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos)); +} static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); - state.forceValue(*args[1]); + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); CompareValues comp; mkBool(v, comp(args[0], args[1])); } @@ -1565,7 +1913,11 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V PathSet context; string s = state.coerceToString(pos, *args[2], context); - if (start < 0) throw EvalError(format("negative start position in ‘substring’, at %1%") % pos); + if (start < 0) + throw EvalError({ + .hint = hintfmt("negative start position in 'substring'"), + .nixCode = NixCode { .errPos = pos } + }); mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context); } @@ -1579,72 +1931,141 @@ static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args } -static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(pos, *args[0], context); - mkString(v, s, PathSet()); -} - - -/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a - builder without causing the derivation to be built (for instance, - in the derivation that builds NARs in nix-push, when doing - source-only deployment). This primop marks the string context so - that builtins.derivation adds the path to drv.inputSrcs rather than - drv.inputDrvs. */ -static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(pos, *args[0], context); - - PathSet context2; - for (auto & p : context) - context2.insert(p.at(0) == '=' ? "~" + string(p, 1) : p); - - mkString(v, s, context2); -} - - /* Return the cryptographic hash of a string in base-16. */ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v) { string type = state.forceStringNoCtx(*args[0], pos); HashType ht = parseHashType(type); if (ht == htUnknown) - throw Error(format("unknown hash type ‘%1%’, at %2%") % type % pos); + throw Error({ + .hint = hintfmt("unknown hash type '%1%'", type), + .nixCode = NixCode { .errPos = pos } + }); PathSet context; // discarded string s = state.forceString(*args[1], context, pos); - mkString(v, printHash(hashString(ht, s)), context); + mkString(v, hashString(ht, s).to_string(Base16, false), context); } /* Match a regular expression against a string and return either ‘null’ or a list containing substring matches. */ -static void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) +void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) { - std::regex regex(state.forceStringNoCtx(*args[0], pos), std::regex::extended); + auto re = state.forceStringNoCtx(*args[0], pos); - PathSet context; - const std::string str = state.forceString(*args[1], context, pos); + try { + auto regex = state.regexCache.find(re); + if (regex == state.regexCache.end()) + regex = state.regexCache.emplace(re, std::regex(re, std::regex::extended)).first; - std::smatch match; - if (!std::regex_match(str, match, regex)) { - mkNull(v); - return; + PathSet context; + const std::string str = state.forceString(*args[1], context, pos); + + std::smatch match; + if (!std::regex_match(str, match, regex->second)) { + mkNull(v); + return; + } + + // the first match is the whole string + const size_t len = match.size() - 1; + state.mkList(v, len); + for (size_t i = 0; i < len; ++i) { + if (!match[i+1].matched) + mkNull(*(v.listElems()[i] = state.allocValue())); + else + mkString(*(v.listElems()[i] = state.allocValue()), match[i + 1].str().c_str()); + } + + } catch (std::regex_error &e) { + if (e.code() == std::regex_constants::error_space) { + // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ + throw EvalError({ + .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), + .nixCode = NixCode { .errPos = pos } + }); + } else { + throw EvalError({ + .hint = hintfmt("invalid regular expression '%s'", re), + .nixCode = NixCode { .errPos = pos } + }); + } } +} - // the first match is the whole string - const size_t len = match.size() - 1; - state.mkList(v, len); - for (size_t i = 0; i < len; ++i) { - if (!match[i+1].matched) - mkNull(*(v.listElems()[i] = state.allocValue())); - else - mkString(*(v.listElems()[i] = state.allocValue()), match[i + 1].str().c_str()); + +/* Split a string with a regular expression, and return a list of the + non-matching parts interleaved by the lists of the matching groups. */ +static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto re = state.forceStringNoCtx(*args[0], pos); + + try { + + std::regex regex(re, std::regex::extended); + + PathSet context; + const std::string str = state.forceString(*args[1], context, pos); + + auto begin = std::sregex_iterator(str.begin(), str.end(), regex); + auto end = std::sregex_iterator(); + + // Any matches results are surrounded by non-matching results. + const size_t len = std::distance(begin, end); + state.mkList(v, 2 * len + 1); + size_t idx = 0; + Value * elem; + + if (len == 0) { + v.listElems()[idx++] = args[1]; + return; + } + + for (std::sregex_iterator i = begin; i != end; ++i) { + assert(idx <= 2 * len + 1 - 3); + std::smatch match = *i; + + // Add a string for non-matched characters. + elem = v.listElems()[idx++] = state.allocValue(); + mkString(*elem, match.prefix().str().c_str()); + + // Add a list for matched substrings. + const size_t slen = match.size() - 1; + elem = v.listElems()[idx++] = state.allocValue(); + + // Start at 1, beacause the first match is the whole string. + state.mkList(*elem, slen); + for (size_t si = 0; si < slen; ++si) { + if (!match[si + 1].matched) + mkNull(*(elem->listElems()[si] = state.allocValue())); + else + mkString(*(elem->listElems()[si] = state.allocValue()), match[si + 1].str().c_str()); + } + + // Add a string for non-matched suffix characters. + if (idx == 2 * len) { + elem = v.listElems()[idx++] = state.allocValue(); + mkString(*elem, match.suffix().str().c_str()); + } + } + assert(idx == 2 * len + 1); + + } catch (std::regex_error &e) { + if (e.code() == std::regex_constants::error_space) { + // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ + throw EvalError({ + .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), + .nixCode = NixCode { .errPos = pos } + }); + } else { + throw EvalError({ + .hint = hintfmt("invalid regular expression '%s'", re), + .nixCode = NixCode { .errPos = pos } + }); + } } } @@ -1674,7 +2095,10 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar state.forceList(*args[0], pos); state.forceList(*args[1], pos); if (args[0]->listSize() != args[1]->listSize()) - throw EvalError(format("‘from’ and ‘to’ arguments to ‘replaceStrings’ have different lengths, at %1%") % pos); + throw EvalError({ + .hint = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"), + .nixCode = NixCode { .errPos = pos } + }); vector from; from.reserve(args[0]->listSize()); @@ -1693,21 +2117,32 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar auto s = state.forceString(*args[2], context, pos); string res; - for (size_t p = 0; p < s.size(); ) { + // Loops one past last character to handle the case where 'from' contains an empty string. + for (size_t p = 0; p <= s.size(); ) { bool found = false; auto i = from.begin(); auto j = to.begin(); for (; i != from.end(); ++i, ++j) if (s.compare(p, i->size(), *i) == 0) { found = true; - p += i->size(); res += j->first; + if (i->empty()) { + if (p < s.size()) + res += s[p]; + p++; + } else { + p += i->size(); + } for (auto& path : j->second) context.insert(path); j->second.clear(); break; } - if (!found) res += s[p++]; + if (!found) { + if (p < s.size()) + res += s[p]; + p++; + } } mkString(v, res, context); @@ -1738,59 +2173,23 @@ static void prim_compareVersions(EvalState & state, const Pos & pos, Value * * a } -/************************************************************* - * Networking - *************************************************************/ - - -void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, - const string & who, bool unpack) -{ - string url; - Hash expectedHash; - string name; - - state.forceValue(*args[0]); - - if (args[0]->type == tAttrs) { - - state.forceAttrs(*args[0], pos); - - for (auto & attr : *args[0]->attrs) { - string n(attr.name); - if (n == "url") - url = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "sha256") - expectedHash = parseHash16or32(htSHA256, state.forceStringNoCtx(*attr.value, *attr.pos)); - else if (n == "name") - name = state.forceStringNoCtx(*attr.value, *attr.pos); - else - throw EvalError(format("unsupported argument ‘%1%’ to ‘%2%’, at %3%") % attr.name % who % attr.pos); - } - - if (url.empty()) - throw EvalError(format("‘url’ argument required, at %1%") % pos); - - } else - url = state.forceStringNoCtx(*args[0], pos); - - if (state.restricted && !expectedHash) - throw Error(format("‘%1%’ is not allowed in restricted mode") % who); - - Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash); - mkString(v, res, PathSet({res})); -} - - -static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args, Value & v) { - fetch(state, pos, args, v, "fetchurl", false); -} - - -static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - fetch(state, pos, args, v, "fetchTarball", true); + string version = state.forceStringNoCtx(*args[0], pos); + auto iter = version.cbegin(); + Strings components; + while (iter != version.cend()) { + auto component = nextComponent(iter, version.cend()); + if (component.empty()) + break; + components.emplace_back(std::move(component)); + } + state.mkList(v, components.size()); + unsigned int n = 0; + for (auto & component : components) { + auto listElem = v.listElems()[n++] = state.allocValue(); + mkString(*listElem, std::move(component)); + } } @@ -1829,11 +2228,24 @@ void EvalState::createBaseEnv() mkNull(v); addConstant("null", v); - mkInt(v, time(0)); - addConstant("__currentTime", v); + auto vThrow = addPrimOp("throw", 1, prim_throw); + + auto addPurityError = [&](const std::string & name) { + Value * v2 = allocValue(); + mkString(*v2, fmt("'%s' is not allowed in pure evaluation mode", name)); + mkApp(v, *vThrow, *v2); + addConstant(name, v); + }; + + if (!evalSettings.pureEval) { + mkInt(v, time(0)); + addConstant("__currentTime", v); + } - mkString(v, settings.thisSystem); - addConstant("__currentSystem", v); + if (!evalSettings.pureEval) { + mkString(v, settings.thisSystem.get()); + addConstant("__currentSystem", v); + } mkString(v, nixVersion); addConstant("__nixVersion", v); @@ -1845,18 +2257,20 @@ void EvalState::createBaseEnv() language feature gets added. It's not necessary to increase it when primops get added, because you can just use `builtins ? primOp' to check. */ - mkInt(v, 4); + mkInt(v, 5); addConstant("__langVersion", v); // Miscellaneous - addPrimOp("scopedImport", 2, prim_scopedImport); + auto vScopedImport = addPrimOp("scopedImport", 2, prim_scopedImport); Value * v2 = allocValue(); mkAttrs(*v2, 0); - mkApp(v, *baseEnv.values[baseEnvDispl - 1], *v2); + mkApp(v, *vScopedImport, *v2); forceValue(v); addConstant("import", v); - if (settings.enableImportNative) + if (evalSettings.enableNativeCode) { addPrimOp("__importNative", 2, prim_importNative); + addPrimOp("__exec", 1, prim_exec); + } addPrimOp("__typeOf", 1, prim_typeOf); addPrimOp("isNull", 1, prim_isNull); addPrimOp("__isFunction", 1, prim_isFunction); @@ -1864,9 +2278,9 @@ void EvalState::createBaseEnv() addPrimOp("__isInt", 1, prim_isInt); addPrimOp("__isFloat", 1, prim_isFloat); addPrimOp("__isBool", 1, prim_isBool); + addPrimOp("__isPath", 1, prim_isPath); addPrimOp("__genericClosure", 1, prim_genericClosure); addPrimOp("abort", 1, prim_abort); - addPrimOp("throw", 1, prim_throw); addPrimOp("__addErrorContext", 2, prim_addErrorContext); addPrimOp("__tryEval", 1, prim_tryEval); addPrimOp("__getEnv", 1, prim_getEnv); @@ -1877,17 +2291,20 @@ void EvalState::createBaseEnv() // Debugging addPrimOp("__trace", 2, prim_trace); - addPrimOp("__valueSize", 1, prim_valueSize); // Paths addPrimOp("__toPath", 1, prim_toPath); - addPrimOp("__storePath", 1, prim_storePath); + if (evalSettings.pureEval) + addPurityError("__storePath"); + else + addPrimOp("__storePath", 1, prim_storePath); addPrimOp("__pathExists", 1, prim_pathExists); addPrimOp("baseNameOf", 1, prim_baseNameOf); addPrimOp("dirOf", 1, prim_dirOf); addPrimOp("__readFile", 1, prim_readFile); addPrimOp("__readDir", 1, prim_readDir); addPrimOp("__findFile", 2, prim_findFile); + addPrimOp("__hashFile", 2, prim_hashFile); // Creating files addPrimOp("__toXML", 1, prim_toXML); @@ -1895,6 +2312,7 @@ void EvalState::createBaseEnv() addPrimOp("__fromJSON", 1, prim_fromJSON); addPrimOp("__toFile", 2, prim_toFile); addPrimOp("__filterSource", 2, prim_filterSource); + addPrimOp("__path", 1, prim_path); // Sets addPrimOp("__attrNames", 1, prim_attrNames); @@ -1908,6 +2326,7 @@ void EvalState::createBaseEnv() addPrimOp("__intersectAttrs", 2, prim_intersectAttrs); addPrimOp("__catAttrs", 2, prim_catAttrs); addPrimOp("__functionArgs", 1, prim_functionArgs); + addPrimOp("__mapAttrs", 2, prim_mapAttrs); // Lists addPrimOp("__isList", 1, prim_isList); @@ -1925,40 +2344,40 @@ void EvalState::createBaseEnv() addPrimOp("__genList", 2, prim_genList); addPrimOp("__sort", 2, prim_sort); addPrimOp("__partition", 2, prim_partition); + addPrimOp("__concatMap", 2, prim_concatMap); // Integer arithmetic addPrimOp("__add", 2, prim_add); addPrimOp("__sub", 2, prim_sub); addPrimOp("__mul", 2, prim_mul); addPrimOp("__div", 2, prim_div); + addPrimOp("__bitAnd", 2, prim_bitAnd); + addPrimOp("__bitOr", 2, prim_bitOr); + addPrimOp("__bitXor", 2, prim_bitXor); addPrimOp("__lessThan", 2, prim_lessThan); // String manipulation addPrimOp("toString", 1, prim_toString); addPrimOp("__substring", 3, prim_substring); addPrimOp("__stringLength", 1, prim_stringLength); - addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); - addPrimOp("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); addPrimOp("__hashString", 2, prim_hashString); addPrimOp("__match", 2, prim_match); + addPrimOp("__split", 2, prim_split); addPrimOp("__concatStringsSep", 2, prim_concatStringSep); addPrimOp("__replaceStrings", 3, prim_replaceStrings); // Versions addPrimOp("__parseDrvName", 1, prim_parseDrvName); addPrimOp("__compareVersions", 2, prim_compareVersions); + addPrimOp("__splitVersion", 1, prim_splitVersion); // Derivations addPrimOp("derivationStrict", 1, prim_derivationStrict); addPrimOp("placeholder", 1, prim_placeholder); - // Networking - addPrimOp("__fetchurl", 1, prim_fetchurl); - addPrimOp("fetchTarball", 1, prim_fetchTarball); - /* Add a wrapper around the derivation primop that computes the `drvPath' and `outPath' attributes lazily. */ - string path = settings.nixDataDir + "/nix/corepkgs/derivation.nix"; + string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true); sDerivationNix = symbols.create(path); evalFile(path, v); addConstant("derivation", v); diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 39d23b04a5c..05d0792efc7 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -9,7 +9,19 @@ struct RegisterPrimOp { typedef std::vector> PrimOps; static PrimOps * primOps; + /* You can register a constant by passing an arity of 0. fun + will get called during EvalState initialization, so there + may be primops not yet added and builtins is not yet sorted. */ RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun); }; +/* These primops are disabled without enableNativeCode, but plugins + may wish to use them in limited contexts without globally enabling + them. */ +/* Load a ValueInitializer from a DSO and return whatever it initializes */ +void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v); + +/* Execute a program and parse its output */ +void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v); + } diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc new file mode 100644 index 00000000000..efa2e9576f7 --- /dev/null +++ b/src/libexpr/primops/context.cc @@ -0,0 +1,196 @@ +#include "primops.hh" +#include "eval-inline.hh" +#include "derivations.hh" + +namespace nix { + +static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + string s = state.coerceToString(pos, *args[0], context); + mkString(v, s, PathSet()); +} + +static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); + + +static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + state.forceString(*args[0], context, pos); + mkBool(v, !context.empty()); +} + +static RegisterPrimOp r2("__hasContext", 1, prim_hasContext); + + +/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a + builder without causing the derivation to be built (for instance, + in the derivation that builds NARs in nix-push, when doing + source-only deployment). This primop marks the string context so + that builtins.derivation adds the path to drv.inputSrcs rather than + drv.inputDrvs. */ +static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + string s = state.coerceToString(pos, *args[0], context); + + PathSet context2; + for (auto & p : context) + context2.insert(p.at(0) == '=' ? string(p, 1) : p); + + mkString(v, s, context2); +} + +static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); + + +/* Extract the context of a string as a structured Nix value. + + The context is represented as an attribute set whose keys are the + paths in the context set and whose values are attribute sets with + the following keys: + path: True if the relevant path is in the context as a plain store + path (i.e. the kind of context you get when interpolating + a Nix path (e.g. ./.) into a string). False if missing. + allOutputs: True if the relevant path is a derivation and it is + in the context as a drv file with all of its outputs + (i.e. the kind of context you get when referencing + .drvPath of some derivation). False if missing. + outputs: If a non-empty list, the relevant path is a derivation + and the provided outputs are referenced in the context + (i.e. the kind of context you get when referencing + .outPath of some derivation). Empty list if missing. + Note that for a given path any combination of the above attributes + may be present. +*/ +static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + struct ContextInfo { + bool path = false; + bool allOutputs = false; + Strings outputs; + }; + PathSet context; + state.forceString(*args[0], context, pos); + auto contextInfos = std::map(); + for (const auto & p : context) { + Path drv; + string output; + const Path * path = &p; + if (p.at(0) == '=') { + drv = string(p, 1); + path = &drv; + } else if (p.at(0) == '!') { + std::pair ctx = decodeContext(p); + drv = ctx.first; + output = ctx.second; + path = &drv; + } + auto isPath = drv.empty(); + auto isAllOutputs = (!drv.empty()) && output.empty(); + + auto iter = contextInfos.find(*path); + if (iter == contextInfos.end()) { + contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}}); + } else { + if (isPath) + iter->second.path = true; + else if (isAllOutputs) + iter->second.allOutputs = true; + else + iter->second.outputs.emplace_back(std::move(output)); + } + } + + state.mkAttrs(v, contextInfos.size()); + + auto sPath = state.symbols.create("path"); + auto sAllOutputs = state.symbols.create("allOutputs"); + for (const auto & info : contextInfos) { + auto & infoVal = *state.allocAttr(v, state.symbols.create(info.first)); + state.mkAttrs(infoVal, 3); + if (info.second.path) + mkBool(*state.allocAttr(infoVal, sPath), true); + if (info.second.allOutputs) + mkBool(*state.allocAttr(infoVal, sAllOutputs), true); + if (!info.second.outputs.empty()) { + auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs); + state.mkList(outputsVal, info.second.outputs.size()); + size_t i = 0; + for (const auto & output : info.second.outputs) { + mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output); + } + } + infoVal.attrs->sort(); + } + v.attrs->sort(); +} + +static RegisterPrimOp r4("__getContext", 1, prim_getContext); + + +/* Append the given context to a given string. + + See the commentary above unsafeGetContext for details of the + context representation. +*/ +static void prim_appendContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + auto orig = state.forceString(*args[0], context, pos); + + state.forceAttrs(*args[1], pos); + + auto sPath = state.symbols.create("path"); + auto sAllOutputs = state.symbols.create("allOutputs"); + for (auto & i : *args[1]->attrs) { + if (!state.store->isStorePath(i.name)) + throw EvalError({ + .hint = hintfmt("Context key '%s' is not a store path", i.name), + .nixCode = NixCode { .errPos = *i.pos } + }); + if (!settings.readOnlyMode) + state.store->ensurePath(state.store->parseStorePath(i.name)); + state.forceAttrs(*i.value, *i.pos); + auto iter = i.value->attrs->find(sPath); + if (iter != i.value->attrs->end()) { + if (state.forceBool(*iter->value, *iter->pos)) + context.insert(i.name); + } + + iter = i.value->attrs->find(sAllOutputs); + if (iter != i.value->attrs->end()) { + if (state.forceBool(*iter->value, *iter->pos)) { + if (!isDerivation(i.name)) { + throw EvalError({ + .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name), + .nixCode = NixCode { .errPos = *i.pos } + }); + } + context.insert("=" + string(i.name)); + } + } + + iter = i.value->attrs->find(state.sOutputs); + if (iter != i.value->attrs->end()) { + state.forceList(*iter->value, *iter->pos); + if (iter->value->listSize() && !isDerivation(i.name)) { + throw EvalError({ + .hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name), + .nixCode = NixCode { .errPos = *i.pos } + }); + } + for (unsigned int n = 0; n < iter->value->listSize(); ++n) { + auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos); + context.insert("!" + name + "!" + string(i.name)); + } + } + } + + mkString(v, orig, context); +} + +static RegisterPrimOp r5("__appendContext", 2, prim_appendContext); + +} diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc new file mode 100644 index 00000000000..dd7229a3d54 --- /dev/null +++ b/src/libexpr/primops/fetchGit.cc @@ -0,0 +1,91 @@ +#include "primops.hh" +#include "eval-inline.hh" +#include "store-api.hh" +#include "hash.hh" +#include "fetchers.hh" +#include "url.hh" + +namespace nix { + +static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + std::string url; + std::optional ref; + std::optional rev; + std::string name = "source"; + bool fetchSubmodules = false; + PathSet context; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + + state.forceAttrs(*args[0], pos); + + for (auto & attr : *args[0]->attrs) { + string n(attr.name); + if (n == "url") + url = state.coerceToString(*attr.pos, *attr.value, context, false, false); + else if (n == "ref") + ref = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "rev") + rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1); + else if (n == "name") + name = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "submodules") + fetchSubmodules = state.forceBool(*attr.value, *attr.pos); + else + throw EvalError({ + .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name), + .nixCode = NixCode { .errPos = *attr.pos } + }); + } + + if (url.empty()) + throw EvalError({ + .hint = hintfmt("'url' argument required"), + .nixCode = NixCode { .errPos = pos } + }); + + } else + url = state.coerceToString(pos, *args[0], context, false, false); + + // FIXME: git externals probably can be used to bypass the URI + // whitelist. Ah well. + state.checkURI(url); + + if (evalSettings.pureEval && !rev) + throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); + + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "git"); + attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); + if (ref) attrs.insert_or_assign("ref", *ref); + if (rev) attrs.insert_or_assign("rev", rev->gitRev()); + if (fetchSubmodules) attrs.insert_or_assign("submodules", true); + auto input = fetchers::inputFromAttrs(attrs); + + // FIXME: use name? + auto [tree, input2] = input->fetchTree(state.store); + + state.mkAttrs(v, 8); + auto storePath = state.store->printStorePath(tree.storePath); + mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); + // Backward compatibility: set 'rev' to + // 0000000000000000000000000000000000000000 for a dirty tree. + auto rev2 = input2->getRev().value_or(Hash(htSHA1)); + mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev()); + // Backward compatibility: set 'revCount' to 0 for a dirty tree. + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), + tree.info.revCount.value_or(0)); + mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules); + v.attrs->sort(); + + if (state.allowedPaths) + state.allowedPaths->insert(tree.actualPath); +} + +static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); + +} diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc new file mode 100644 index 00000000000..9bace8f8925 --- /dev/null +++ b/src/libexpr/primops/fetchMercurial.cc @@ -0,0 +1,93 @@ +#include "primops.hh" +#include "eval-inline.hh" +#include "store-api.hh" +#include "fetchers.hh" +#include "url.hh" + +#include + +namespace nix { + +static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + std::string url; + std::optional rev; + std::optional ref; + std::string name = "source"; + PathSet context; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + + state.forceAttrs(*args[0], pos); + + for (auto & attr : *args[0]->attrs) { + string n(attr.name); + if (n == "url") + url = state.coerceToString(*attr.pos, *attr.value, context, false, false); + else if (n == "rev") { + // Ugly: unlike fetchGit, here the "rev" attribute can + // be both a revision or a branch/tag name. + auto value = state.forceStringNoCtx(*attr.value, *attr.pos); + if (std::regex_match(value, revRegex)) + rev = Hash(value, htSHA1); + else + ref = value; + } + else if (n == "name") + name = state.forceStringNoCtx(*attr.value, *attr.pos); + else + throw EvalError({ + .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name), + .nixCode = NixCode { .errPos = *attr.pos } + }); + } + + if (url.empty()) + throw EvalError({ + .hint = hintfmt("'url' argument required"), + .nixCode = NixCode { .errPos = pos } + }); + + } else + url = state.coerceToString(pos, *args[0], context, false, false); + + // FIXME: git externals probably can be used to bypass the URI + // whitelist. Ah well. + state.checkURI(url); + + if (evalSettings.pureEval && !rev) + throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); + + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "hg"); + attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); + if (ref) attrs.insert_or_assign("ref", *ref); + if (rev) attrs.insert_or_assign("rev", rev->gitRev()); + auto input = fetchers::inputFromAttrs(attrs); + + // FIXME: use name + auto [tree, input2] = input->fetchTree(state.store); + + state.mkAttrs(v, 8); + auto storePath = state.store->printStorePath(tree.storePath); + mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); + if (input2->getRef()) + mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef()); + // Backward compatibility: set 'rev' to + // 0000000000000000000000000000000000000000 for a dirty tree. + auto rev2 = input2->getRev().value_or(Hash(htSHA1)); + mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12)); + if (tree.info.revCount) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); + v.attrs->sort(); + + if (state.allowedPaths) + state.allowedPaths->insert(tree.actualPath); +} + +static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); + +} diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc new file mode 100644 index 00000000000..9be93710ad3 --- /dev/null +++ b/src/libexpr/primops/fetchTree.cc @@ -0,0 +1,172 @@ +#include "primops.hh" +#include "eval-inline.hh" +#include "store-api.hh" +#include "fetchers.hh" +#include "filetransfer.hh" + +#include +#include + +namespace nix { + +void emitTreeAttrs( + EvalState & state, + const fetchers::Tree & tree, + std::shared_ptr input, + Value & v) +{ + state.mkAttrs(v, 8); + + auto storePath = state.store->printStorePath(tree.storePath); + + mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); + + assert(tree.info.narHash); + mkString(*state.allocAttr(v, state.symbols.create("narHash")), + tree.info.narHash.to_string(SRI, true)); + + if (input->getRev()) { + mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev()); + } + + if (tree.info.revCount) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); + + if (tree.info.lastModified) + mkString(*state.allocAttr(v, state.symbols.create("lastModified")), + fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S"))); + + v.attrs->sort(); +} + +static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + settings.requireExperimentalFeature("flakes"); + + std::shared_ptr input; + PathSet context; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + state.forceAttrs(*args[0], pos); + + fetchers::Attrs attrs; + + for (auto & attr : *args[0]->attrs) { + state.forceValue(*attr.value); + if (attr.value->type == tString) + attrs.emplace(attr.name, attr.value->string.s); + else if (attr.value->type == tBool) + attrs.emplace(attr.name, attr.value->boolean); + else + throw TypeError("fetchTree argument '%s' is %s while a string or Boolean is expected", + attr.name, showType(*attr.value)); + } + + if (!attrs.count("type")) + throw Error({ + .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), + .nixCode = NixCode { .errPos = pos } + }); + + input = fetchers::inputFromAttrs(attrs); + } else + input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false)); + + if (evalSettings.pureEval && !input->isImmutable()) + throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input"); + + // FIXME: use fetchOrSubstituteTree + auto [tree, input2] = input->fetchTree(state.store); + + if (state.allowedPaths) + state.allowedPaths->insert(tree.actualPath); + + emitTreeAttrs(state, tree, input2, v); +} + +static RegisterPrimOp r("fetchTree", 1, prim_fetchTree); + +static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, + const string & who, bool unpack, std::string name) +{ + std::optional url; + std::optional expectedHash; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + + state.forceAttrs(*args[0], pos); + + for (auto & attr : *args[0]->attrs) { + string n(attr.name); + if (n == "url") + url = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "sha256") + expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + else if (n == "name") + name = state.forceStringNoCtx(*attr.value, *attr.pos); + else + throw EvalError({ + .hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who), + .nixCode = NixCode { .errPos = *attr.pos } + }); + } + + if (!url) + throw EvalError({ + .hint = hintfmt("'url' argument required"), + .nixCode = NixCode { .errPos = pos } + }); + } else + url = state.forceStringNoCtx(*args[0], pos); + + url = resolveUri(*url); + + state.checkURI(*url); + + if (name == "") + name = baseNameOf(*url); + + if (evalSettings.pureEval && !expectedHash) + throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); + + auto storePath = + unpack + ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath + : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; + + auto path = state.store->toRealPath(storePath); + + if (expectedHash) { + auto hash = unpack + ? state.store->queryPathInfo(storePath)->narHash + : hashFile(htSHA256, path); + if (hash != *expectedHash) + throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", + *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true)); + } + + if (state.allowedPaths) + state.allowedPaths->insert(path); + + mkString(v, path, PathSet({path})); +} + +static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + fetch(state, pos, args, v, "fetchurl", false, ""); +} + +static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + fetch(state, pos, args, v, "fetchTarball", true, "source"); +} + +static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl); +static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball); + +} diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc deleted file mode 100644 index bd440c8c62a..00000000000 --- a/src/libexpr/primops/fetchgit.cc +++ /dev/null @@ -1,82 +0,0 @@ -#include "primops.hh" -#include "eval-inline.hh" -#include "download.hh" -#include "store-api.hh" - -namespace nix { - -Path exportGit(ref store, const std::string & uri, const std::string & rev) -{ - if (!isUri(uri)) - throw EvalError(format("‘%s’ is not a valid URI") % uri); - - Path cacheDir = getCacheDir() + "/nix/git"; - - if (!pathExists(cacheDir)) { - createDirs(cacheDir); - runProgram("git", true, { "init", "--bare", cacheDir }); - } - - Activity act(*logger, lvlInfo, format("fetching Git repository ‘%s’") % uri); - - std::string localRef = "pid-" + std::to_string(getpid()); - Path localRefFile = cacheDir + "/refs/heads/" + localRef; - - runProgram("git", true, { "-C", cacheDir, "fetch", uri, rev + ":" + localRef }); - - std::string commitHash = chomp(readFile(localRefFile)); - - unlink(localRefFile.c_str()); - - debug(format("got revision ‘%s’") % commitHash); - - // FIXME: should pipe this, or find some better way to extract a - // revision. - auto tar = runProgram("git", true, { "-C", cacheDir, "archive", commitHash }); - - Path tmpDir = createTempDir(); - AutoDelete delTmpDir(tmpDir, true); - - runProgram("tar", true, { "x", "-C", tmpDir }, tar); - - return store->addToStore("git-export", tmpDir); -} - -static void prim_fetchgit(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - // FIXME: cut&paste from fetch(). - if (state.restricted) throw Error("‘fetchgit’ is not allowed in restricted mode"); - - std::string url; - std::string rev = "master"; - - state.forceValue(*args[0]); - - if (args[0]->type == tAttrs) { - - state.forceAttrs(*args[0], pos); - - for (auto & attr : *args[0]->attrs) { - string name(attr.name); - if (name == "url") - url = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (name == "rev") - rev = state.forceStringNoCtx(*attr.value, *attr.pos); - else - throw EvalError(format("unsupported argument ‘%1%’ to ‘fetchgit’, at %3%") % attr.name % attr.pos); - } - - if (url.empty()) - throw EvalError(format("‘url’ argument required, at %1%") % pos); - - } else - url = state.forceStringNoCtx(*args[0], pos); - - Path storePath = exportGit(state.store, url, rev); - - mkString(v, storePath, PathSet({storePath})); -} - -static RegisterPrimOp r("__fetchgit", 1, prim_fetchgit); - -} diff --git a/src/libexpr/primops/fetchgit.hh b/src/libexpr/primops/fetchgit.hh deleted file mode 100644 index 6ffb21a96da..00000000000 --- a/src/libexpr/primops/fetchgit.hh +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include - -#include "ref.hh" - -namespace nix { - -class Store; - -Path exportGit(ref store, - const std::string & uri, const std::string & rev); - -} diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc new file mode 100644 index 00000000000..7615d1379f7 --- /dev/null +++ b/src/libexpr/primops/fromTOML.cc @@ -0,0 +1,93 @@ +#include "primops.hh" +#include "eval-inline.hh" + +#include "../../cpptoml/cpptoml.h" + +namespace nix { + +static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + using namespace cpptoml; + + auto toml = state.forceStringNoCtx(*args[0], pos); + + std::istringstream tomlStream(toml); + + std::function)> visit; + + visit = [&](Value & v, std::shared_ptr t) { + + if (auto t2 = t->as_table()) { + + size_t size = 0; + for (auto & i : *t2) { (void) i; size++; } + + state.mkAttrs(v, size); + + for (auto & i : *t2) { + auto & v2 = *state.allocAttr(v, state.symbols.create(i.first)); + + if (auto i2 = i.second->as_table_array()) { + size_t size2 = i2->get().size(); + state.mkList(v2, size2); + for (size_t j = 0; j < size2; ++j) + visit(*(v2.listElems()[j] = state.allocValue()), i2->get()[j]); + } + else + visit(v2, i.second); + } + + v.attrs->sort(); + } + + else if (auto t2 = t->as_array()) { + size_t size = t2->get().size(); + + state.mkList(v, size); + + for (size_t i = 0; i < size; ++i) + visit(*(v.listElems()[i] = state.allocValue()), t2->get()[i]); + } + + // Handle cases like 'a = [[{ a = true }]]', which IMHO should be + // parsed as a array containing an array containing a table, + // but instead are parsed as an array containing a table array + // containing a table. + else if (auto t2 = t->as_table_array()) { + size_t size = t2->get().size(); + + state.mkList(v, size); + + for (size_t j = 0; j < size; ++j) + visit(*(v.listElems()[j] = state.allocValue()), t2->get()[j]); + } + + else if (t->is_value()) { + if (auto val = t->as()) + mkInt(v, val->get()); + else if (auto val = t->as()) + mkFloat(v, val->get()); + else if (auto val = t->as()) + mkBool(v, val->get()); + else if (auto val = t->as()) + mkString(v, val->get()); + else + throw EvalError("unsupported value type in TOML"); + } + + else abort(); + }; + + try { + visit(v, parser(tomlStream).parse()); + } catch (std::runtime_error & e) { + throw EvalError({ + .hint = hintfmt("while parsing a TOML string: %s", e.what()), + .nixCode = NixCode { .errPos = pos } + }); + } +} + +static RegisterPrimOp r("fromTOML", 1, prim_fromTOML); + +} diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index 2fdf820211c..7ba5e1c14f1 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -1,7 +1,5 @@ #pragma once -#include "config.h" - #include #include @@ -40,7 +38,12 @@ public: return s < s2.s; } - operator const string & () const + operator const std::string & () const + { + return *s; + } + + operator const std::string_view () const { return *s; } @@ -71,12 +74,19 @@ public: return Symbol(&*res.first); } - unsigned int size() const + size_t size() const { return symbols.size(); } size_t totalSize() const; + + template + void dump(T callback) + { + for (auto & s : symbols) + callback(s); + } }; } diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 72e413e4491..6ec8315bab0 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -40,7 +40,12 @@ void printValueAsJSON(EvalState & state, bool strict, break; case tAttrs: { - Bindings::iterator i = v.attrs->find(state.sOutPath); + auto maybeString = state.tryAttrsToString(noPos, v, context, false, false); + if (maybeString) { + out.write(*maybeString); + break; + } + auto i = v.attrs->find(state.sOutPath); if (i == v.attrs->end()) { auto obj(out.object()); StringSet names; @@ -74,7 +79,7 @@ void printValueAsJSON(EvalState & state, bool strict, break; default: - throw TypeError(format("cannot convert %1% to JSON") % showType(v)); + throw TypeError("cannot convert %1% to JSON", showType(v)); } } @@ -88,7 +93,7 @@ void printValueAsJSON(EvalState & state, bool strict, void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, JSONPlaceholder & out, PathSet & context) const { - throw TypeError(format("cannot convert %1% to JSON") % showType()); + throw TypeError("cannot convert %1% to JSON", showType()); } diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 00b1918a82a..1f0b1541d6e 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -105,10 +105,9 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, XMLOpenElement _(doc, "derivation", xmlAttrs); - if (drvPath != "" && drvsSeen.find(drvPath) == drvsSeen.end()) { - drvsSeen.insert(drvPath); + if (drvPath != "" && drvsSeen.insert(drvPath).second) showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen); - } else + else doc.writeEmptyElement("repeated"); } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 271e6a1b24a..71025824ef0 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -1,6 +1,5 @@ #pragma once -#include "config.h" #include "symbol-table.hh" #if HAVE_BOEHMGC @@ -36,7 +35,6 @@ struct Env; struct Expr; struct ExprLambda; struct PrimOp; -struct PrimOp; class Symbol; struct Pos; class EvalState; @@ -44,8 +42,8 @@ class XMLWriter; class JSONPlaceholder; -typedef long NixInt; -typedef float NixFloat; +typedef int64_t NixInt; +typedef double NixFloat; /* External values must descend from ExternalValueBase, so that * type-agnostic nix functions (e.g. showType) can be implemented @@ -64,9 +62,6 @@ class ExternalValueBase /* Return a string to be used in builtins.typeOf */ virtual string typeOf() const = 0; - /* How much space does this value take up */ - virtual size_t valueSize(std::set & seen) const = 0; - /* Coerce the value to a string. Defaults to uncoercable, i.e. throws an * error */ @@ -129,7 +124,7 @@ struct Value const char * path; Bindings * attrs; struct { - unsigned int size; + size_t size; Value * * elems; } bigList; Value * smallList[2]; @@ -167,7 +162,7 @@ struct Value return type == tList1 || type == tList2 ? smallList : bigList.elems; } - unsigned int listSize() const + size_t listSize() const { return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size; } @@ -221,6 +216,14 @@ static inline void mkApp(Value & v, Value & left, Value & right) } +static inline void mkPrimOpApp(Value & v, Value & left, Value & right) +{ + v.type = tPrimOpApp; + v.app.left = &left; + v.app.right = &right; +} + + static inline void mkStringNoCopy(Value & v, const char * s) { v.type = tString; @@ -249,19 +252,18 @@ static inline void mkPathNoCopy(Value & v, const char * s) void mkPath(Value & v, const char * s); -/* Compute the size in bytes of the given value, including all values - and environments reachable from it. Static expressions (Exprs) are - not included. */ -size_t valueSize(Value & v); - - #if HAVE_BOEHMGC -typedef std::vector > ValueVector; -typedef std::map, gc_allocator > ValueMap; +typedef std::vector > ValueVector; +typedef std::map, traceable_allocator > > ValueMap; #else typedef std::vector ValueVector; typedef std::map ValueMap; #endif +/* A value allocated in traceable memory. */ +typedef std::shared_ptr RootValue; + +RootValue allocRootValue(Value * v); + } diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc new file mode 100644 index 00000000000..feb0a608567 --- /dev/null +++ b/src/libfetchers/attrs.cc @@ -0,0 +1,107 @@ +#include "attrs.hh" +#include "fetchers.hh" + +#include + +namespace nix::fetchers { + +Attrs jsonToAttrs(const nlohmann::json & json) +{ + Attrs attrs; + + for (auto & i : json.items()) { + if (i.value().is_number()) + attrs.emplace(i.key(), i.value().get()); + else if (i.value().is_string()) + attrs.emplace(i.key(), i.value().get()); + else if (i.value().is_boolean()) + attrs.emplace(i.key(), i.value().get()); + else + throw Error("unsupported input attribute type in lock file"); + } + + return attrs; +} + +nlohmann::json attrsToJson(const Attrs & attrs) +{ + nlohmann::json json; + for (auto & attr : attrs) { + if (auto v = std::get_if(&attr.second)) { + json[attr.first] = *v; + } else if (auto v = std::get_if(&attr.second)) { + json[attr.first] = *v; + } else if (auto v = std::get_if>(&attr.second)) { + json[attr.first] = v->t; + } else abort(); + } + return json; +} + +std::optional maybeGetStrAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) return {}; + if (auto v = std::get_if(&i->second)) + return *v; + throw Error("input attribute '%s' is not a string %s", name, attrsToJson(attrs).dump()); +} + +std::string getStrAttr(const Attrs & attrs, const std::string & name) +{ + auto s = maybeGetStrAttr(attrs, name); + if (!s) + throw Error("input attribute '%s' is missing", name); + return *s; +} + +std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) return {}; + if (auto v = std::get_if(&i->second)) + return *v; + throw Error("input attribute '%s' is not an integer", name); +} + +int64_t getIntAttr(const Attrs & attrs, const std::string & name) +{ + auto s = maybeGetIntAttr(attrs, name); + if (!s) + throw Error("input attribute '%s' is missing", name); + return *s; +} + +std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) return {}; + if (auto v = std::get_if(&i->second)) + return *v; + throw Error("input attribute '%s' is not a Boolean", name); +} + +bool getBoolAttr(const Attrs & attrs, const std::string & name) +{ + auto s = maybeGetBoolAttr(attrs, name); + if (!s) + throw Error("input attribute '%s' is missing", name); + return *s; +} + +std::map attrsToQuery(const Attrs & attrs) +{ + std::map query; + for (auto & attr : attrs) { + if (auto v = std::get_if(&attr.second)) { + query.insert_or_assign(attr.first, fmt("%d", *v)); + } else if (auto v = std::get_if(&attr.second)) { + query.insert_or_assign(attr.first, *v); + } else if (auto v = std::get_if>(&attr.second)) { + query.insert_or_assign(attr.first, v->t ? "1" : "0"); + } else abort(); + } + return query; +} + +} diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh new file mode 100644 index 00000000000..d6e0ae00053 --- /dev/null +++ b/src/libfetchers/attrs.hh @@ -0,0 +1,39 @@ +#pragma once + +#include "types.hh" + +#include + +#include + +namespace nix::fetchers { + +/* Wrap bools to prevent string literals (i.e. 'char *') from being + cast to a bool in Attr. */ +template +struct Explicit { + T t; +}; + +typedef std::variant> Attr; +typedef std::map Attrs; + +Attrs jsonToAttrs(const nlohmann::json & json); + +nlohmann::json attrsToJson(const Attrs & attrs); + +std::optional maybeGetStrAttr(const Attrs & attrs, const std::string & name); + +std::string getStrAttr(const Attrs & attrs, const std::string & name); + +std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name); + +int64_t getIntAttr(const Attrs & attrs, const std::string & name); + +std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & name); + +bool getBoolAttr(const Attrs & attrs, const std::string & name); + +std::map attrsToQuery(const Attrs & attrs); + +} diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc new file mode 100644 index 00000000000..e1c7f3dee32 --- /dev/null +++ b/src/libfetchers/cache.cc @@ -0,0 +1,121 @@ +#include "cache.hh" +#include "sqlite.hh" +#include "sync.hh" +#include "store-api.hh" + +#include + +namespace nix::fetchers { + +static const char * schema = R"sql( + +create table if not exists Cache ( + input text not null, + info text not null, + path text not null, + immutable integer not null, + timestamp integer not null, + primary key (input) +); +)sql"; + +struct CacheImpl : Cache +{ + struct State + { + SQLite db; + SQLiteStmt add, lookup; + }; + + Sync _state; + + CacheImpl() + { + auto state(_state.lock()); + + auto dbPath = getCacheDir() + "/nix/fetcher-cache-v1.sqlite"; + createDirs(dirOf(dbPath)); + + state->db = SQLite(dbPath); + state->db.isCache(); + state->db.exec(schema); + + state->add.create(state->db, + "insert or replace into Cache(input, info, path, immutable, timestamp) values (?, ?, ?, ?, ?)"); + + state->lookup.create(state->db, + "select info, path, immutable, timestamp from Cache where input = ?"); + } + + void add( + ref store, + const Attrs & inAttrs, + const Attrs & infoAttrs, + const StorePath & storePath, + bool immutable) override + { + _state.lock()->add.use() + (attrsToJson(inAttrs).dump()) + (attrsToJson(infoAttrs).dump()) + (store->printStorePath(storePath)) + (immutable) + (time(0)).exec(); + } + + std::optional> lookup( + ref store, + const Attrs & inAttrs) override + { + if (auto res = lookupExpired(store, inAttrs)) { + if (!res->expired) + return std::make_pair(std::move(res->infoAttrs), std::move(res->storePath)); + debug("ignoring expired cache entry '%s'", + attrsToJson(inAttrs).dump()); + } + return {}; + } + + std::optional lookupExpired( + ref store, + const Attrs & inAttrs) override + { + auto state(_state.lock()); + + auto inAttrsJson = attrsToJson(inAttrs).dump(); + + auto stmt(state->lookup.use()(inAttrsJson)); + if (!stmt.next()) { + debug("did not find cache entry for '%s'", inAttrsJson); + return {}; + } + + auto infoJson = stmt.getStr(0); + auto storePath = store->parseStorePath(stmt.getStr(1)); + auto immutable = stmt.getInt(2) != 0; + auto timestamp = stmt.getInt(3); + + store->addTempRoot(storePath); + if (!store->isValidPath(storePath)) { + // FIXME: we could try to substitute 'storePath'. + debug("ignoring disappeared cache entry '%s'", inAttrsJson); + return {}; + } + + debug("using cache entry '%s' -> '%s', '%s'", + inAttrsJson, infoJson, store->printStorePath(storePath)); + + return Result { + .expired = !immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)), + .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJson)), + .storePath = std::move(storePath) + }; + } +}; + +ref getCache() +{ + static auto cache = std::make_shared(); + return ref(cache); +} + +} diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh new file mode 100644 index 00000000000..d76ab12331d --- /dev/null +++ b/src/libfetchers/cache.hh @@ -0,0 +1,34 @@ +#pragma once + +#include "fetchers.hh" + +namespace nix::fetchers { + +struct Cache +{ + virtual void add( + ref store, + const Attrs & inAttrs, + const Attrs & infoAttrs, + const StorePath & storePath, + bool immutable) = 0; + + virtual std::optional> lookup( + ref store, + const Attrs & inAttrs) = 0; + + struct Result + { + bool expired = false; + Attrs infoAttrs; + StorePath storePath; + }; + + virtual std::optional lookupExpired( + ref store, + const Attrs & inAttrs) = 0; +}; + +ref getCache(); + +} diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc new file mode 100644 index 00000000000..11cac4c5570 --- /dev/null +++ b/src/libfetchers/fetchers.cc @@ -0,0 +1,75 @@ +#include "fetchers.hh" +#include "store-api.hh" + +#include + +namespace nix::fetchers { + +std::unique_ptr>> inputSchemes = nullptr; + +void registerInputScheme(std::unique_ptr && inputScheme) +{ + if (!inputSchemes) inputSchemes = std::make_unique>>(); + inputSchemes->push_back(std::move(inputScheme)); +} + +std::unique_ptr inputFromURL(const ParsedURL & url) +{ + for (auto & inputScheme : *inputSchemes) { + auto res = inputScheme->inputFromURL(url); + if (res) return res; + } + throw Error("input '%s' is unsupported", url.url); +} + +std::unique_ptr inputFromURL(const std::string & url) +{ + return inputFromURL(parseURL(url)); +} + +std::unique_ptr inputFromAttrs(const Attrs & attrs) +{ + auto attrs2(attrs); + attrs2.erase("narHash"); + for (auto & inputScheme : *inputSchemes) { + auto res = inputScheme->inputFromAttrs(attrs2); + if (res) { + if (auto narHash = maybeGetStrAttr(attrs, "narHash")) + // FIXME: require SRI hash. + res->narHash = newHashAllowEmpty(*narHash, htUnknown); + return res; + } + } + throw Error("input '%s' is unsupported", attrsToJson(attrs)); +} + +Attrs Input::toAttrs() const +{ + auto attrs = toAttrsInternal(); + if (narHash) + attrs.emplace("narHash", narHash->to_string(SRI, true)); + attrs.emplace("type", type()); + return attrs; +} + +std::pair> Input::fetchTree(ref store) const +{ + auto [tree, input] = fetchTreeInternal(store); + + if (tree.actualPath == "") + tree.actualPath = store->toRealPath(tree.storePath); + + if (!tree.info.narHash) + tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash; + + if (input->narHash) + assert(input->narHash == tree.info.narHash); + + if (narHash && narHash != input->narHash) + throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", + to_string(), tree.actualPath, narHash->to_string(SRI, true), input->narHash->to_string(SRI, true)); + + return {std::move(tree), input}; +} + +} diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh new file mode 100644 index 00000000000..59a58ae6781 --- /dev/null +++ b/src/libfetchers/fetchers.hh @@ -0,0 +1,103 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" +#include "path.hh" +#include "tree-info.hh" +#include "attrs.hh" +#include "url.hh" + +#include + +namespace nix { class Store; } + +namespace nix::fetchers { + +struct Input; + +struct Tree +{ + Path actualPath; + StorePath storePath; + TreeInfo info; +}; + +struct Input : std::enable_shared_from_this +{ + std::optional narHash; // FIXME: implement + + virtual std::string type() const = 0; + + virtual ~Input() { } + + virtual bool operator ==(const Input & other) const { return false; } + + /* Check whether this is a "direct" input, that is, not + one that goes through a registry. */ + virtual bool isDirect() const { return true; } + + /* Check whether this is an "immutable" input, that is, + one that contains a commit hash or content hash. */ + virtual bool isImmutable() const { return (bool) narHash; } + + virtual bool contains(const Input & other) const { return false; } + + virtual std::optional getRef() const { return {}; } + + virtual std::optional getRev() const { return {}; } + + virtual ParsedURL toURL() const = 0; + + std::string to_string() const + { + return toURL().to_string(); + } + + Attrs toAttrs() const; + + std::pair> fetchTree(ref store) const; + +private: + + virtual std::pair> fetchTreeInternal(ref store) const = 0; + + virtual Attrs toAttrsInternal() const = 0; +}; + +struct InputScheme +{ + virtual ~InputScheme() { } + + virtual std::unique_ptr inputFromURL(const ParsedURL & url) = 0; + + virtual std::unique_ptr inputFromAttrs(const Attrs & attrs) = 0; +}; + +std::unique_ptr inputFromURL(const ParsedURL & url); + +std::unique_ptr inputFromURL(const std::string & url); + +std::unique_ptr inputFromAttrs(const Attrs & attrs); + +void registerInputScheme(std::unique_ptr && fetcher); + +struct DownloadFileResult +{ + StorePath storePath; + std::string etag; + std::string effectiveUrl; +}; + +DownloadFileResult downloadFile( + ref store, + const std::string & url, + const std::string & name, + bool immutable); + +Tree downloadTarball( + ref store, + const std::string & url, + const std::string & name, + bool immutable); + +} diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc new file mode 100644 index 00000000000..75ce5ee8b46 --- /dev/null +++ b/src/libfetchers/git.cc @@ -0,0 +1,441 @@ +#include "fetchers.hh" +#include "cache.hh" +#include "globals.hh" +#include "tarfile.hh" +#include "store-api.hh" + +#include + +using namespace std::string_literals; + +namespace nix::fetchers { + +static std::string readHead(const Path & path) +{ + return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" })); +} + +static bool isNotDotGitDirectory(const Path & path) +{ + static const std::regex gitDirRegex("^(?:.*/)?\\.git$"); + + return not std::regex_match(path, gitDirRegex); +} + +struct GitInput : Input +{ + ParsedURL url; + std::optional ref; + std::optional rev; + bool shallow = false; + bool submodules = false; + + GitInput(const ParsedURL & url) : url(url) + { } + + std::string type() const override { return "git"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && url == other2->url + && rev == other2->rev + && ref == other2->ref; + } + + bool isImmutable() const override + { + return (bool) rev || narHash; + } + + std::optional getRef() const override { return ref; } + + std::optional getRev() const override { return rev; } + + ParsedURL toURL() const override + { + ParsedURL url2(url); + if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme; + if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); + if (ref) url2.query.insert_or_assign("ref", *ref); + if (shallow) url2.query.insert_or_assign("shallow", "1"); + return url2; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("url", url.to_string()); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + if (shallow) + attrs.emplace("shallow", true); + if (submodules) + attrs.emplace("submodules", true); + return attrs; + } + + std::pair getActualUrl() const + { + // Don't clone file:// URIs (but otherwise treat them the + // same as remote URIs, i.e. don't use the working tree or + // HEAD). + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing + bool isLocal = url.scheme == "file" && !forceHttp; + return {isLocal, isLocal ? url.path : url.base}; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto name = "source"; + + auto input = std::make_shared(*this); + + assert(!rev || rev->type == htSHA1); + + std::string cacheType = "git"; + if (shallow) cacheType += "-shallow"; + if (submodules) cacheType += "-submodules"; + + auto getImmutableAttrs = [&]() + { + return Attrs({ + {"type", cacheType}, + {"name", name}, + {"rev", input->rev->gitRev()}, + }); + }; + + auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) + -> std::pair> + { + assert(input->rev); + assert(!rev || rev == input->rev); + return { + Tree { + .actualPath = store->toRealPath(storePath), + .storePath = std::move(storePath), + .info = TreeInfo { + .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")), + .lastModified = getIntAttr(infoAttrs, "lastModified"), + }, + }, + input + }; + }; + + if (rev) { + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + } + + auto [isLocal, actualUrl_] = getActualUrl(); + auto actualUrl = actualUrl_; // work around clang bug + + // If this is a local directory and no ref or revision is + // given, then allow the use of an unclean working tree. + if (!input->ref && !input->rev && isLocal) { + bool clean = false; + + /* Check whether this repo has any commits. There are + probably better ways to do this. */ + auto gitDir = actualUrl + "/.git"; + auto commonGitDir = chomp(runProgram( + "git", + true, + { "-C", actualUrl, "rev-parse", "--git-common-dir" } + )); + if (commonGitDir != ".git") + gitDir = commonGitDir; + + bool haveCommits = !readDirectory(gitDir + "/refs/heads").empty(); + + try { + if (haveCommits) { + runProgram("git", true, { "-C", actualUrl, "diff-index", "--quiet", "HEAD", "--" }); + clean = true; + } + } catch (ExecError & e) { + if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; + } + + if (!clean) { + + /* This is an unclean working tree. So copy all tracked files. */ + + if (!settings.allowDirty) + throw Error("Git tree '%s' is dirty", actualUrl); + + if (settings.warnDirty) + warn("Git tree '%s' is dirty", actualUrl); + + auto gitOpts = Strings({ "-C", actualUrl, "ls-files", "-z" }); + if (submodules) + gitOpts.emplace_back("--recurse-submodules"); + + auto files = tokenizeString>( + runProgram("git", true, gitOpts), "\0"s); + + PathFilter filter = [&](const Path & p) -> bool { + assert(hasPrefix(p, actualUrl)); + std::string file(p, actualUrl.size() + 1); + + auto st = lstat(p); + + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && hasPrefix(*i, prefix); + } + + return files.count(file); + }; + + auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); + + auto tree = Tree { + .actualPath = store->printStorePath(storePath), + .storePath = std::move(storePath), + .info = TreeInfo { + // FIXME: maybe we should use the timestamp of the last + // modified dirty file? + .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0, + } + }; + + return {std::move(tree), input}; + } + } + + if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master"; + + Attrs mutableAttrs({ + {"type", cacheType}, + {"name", name}, + {"url", actualUrl}, + {"ref", *input->ref}, + }); + + Path repoDir; + + if (isLocal) { + + if (!input->rev) + input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1); + + repoDir = actualUrl; + + } else { + + if (auto res = getCache()->lookup(store, mutableAttrs)) { + auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); + if (!rev || rev == rev2) { + input->rev = rev2; + return makeResult(res->first, std::move(res->second)); + } + } + + Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false); + repoDir = cacheDir; + + if (!pathExists(cacheDir)) { + createDirs(dirOf(cacheDir)); + runProgram("git", true, { "init", "--bare", repoDir }); + } + + Path localRefFile = + input->ref->compare(0, 5, "refs/") == 0 + ? cacheDir + "/" + *input->ref + : cacheDir + "/refs/heads/" + *input->ref; + + bool doFetch; + time_t now = time(0); + + /* If a rev was specified, we need to fetch if it's not in the + repo. */ + if (input->rev) { + try { + runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() }); + doFetch = false; + } catch (ExecError & e) { + if (WIFEXITED(e.status)) { + doFetch = true; + } else { + throw; + } + } + } else { + /* If the local ref is older than ‘tarball-ttl’ seconds, do a + git fetch to update the local ref to the remote ref. */ + struct stat st; + doFetch = stat(localRefFile.c_str(), &st) != 0 || + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; + } + + if (doFetch) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl)); + + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + try { + auto fetchRef = input->ref->compare(0, 5, "refs/") == 0 + ? *input->ref + : "refs/heads/" + *input->ref; + runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }); + } catch (Error & e) { + if (!pathExists(localRefFile)) throw; + warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); + } + + struct timeval times[2]; + times[0].tv_sec = now; + times[0].tv_usec = 0; + times[1].tv_sec = now; + times[1].tv_usec = 0; + + utimes(localRefFile.c_str(), times); + } + + if (!input->rev) + input->rev = Hash(chomp(readFile(localRefFile)), htSHA1); + } + + bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; + + if (isShallow && !shallow) + throw Error("'%s' is a shallow Git repository, but a non-shallow repository is needed", actualUrl); + + // FIXME: check whether rev is an ancestor of ref. + + printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl); + + /* Now that we know the ref, check again whether we have it in + the store. */ + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + PathFilter filter = defaultPathFilter; + + if (submodules) { + Path tmpGitDir = createTempDir(); + AutoDelete delTmpGitDir(tmpGitDir, true); + + runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir }); + // TODO: repoDir might lack the ref (it only checks if rev + // exists, see FIXME above) so use a big hammer and fetch + // everything to ensure we get the rev. + runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", + "--update-head-ok", "--", repoDir, "refs/*:refs/*" }); + + runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() }); + runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl }); + runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); + + filter = isNotDotGitDirectory; + } else { + // FIXME: should pipe this, or find some better way to extract a + // revision. + auto source = sinkToSource([&](Sink & sink) { + RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() }); + gitOptions.standardOut = &sink; + runProgram2(gitOptions); + }); + + unpackTarfile(*source, tmpDir); + } + + auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter); + + auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); + + Attrs infoAttrs({ + {"rev", input->rev->gitRev()}, + {"lastModified", lastModified}, + }); + + if (!shallow) + infoAttrs.insert_or_assign("revCount", + std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }))); + + if (!this->rev) + getCache()->add( + store, + mutableAttrs, + infoAttrs, + storePath, + false); + + getCache()->add( + store, + getImmutableAttrs(), + infoAttrs, + storePath, + true); + + return makeResult(infoAttrs, std::move(storePath)); + } +}; + +struct GitInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "git" && + url.scheme != "git+http" && + url.scheme != "git+https" && + url.scheme != "git+ssh" && + url.scheme != "git+file") return nullptr; + + auto url2(url); + if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); + url2.query.clear(); + + Attrs attrs; + attrs.emplace("type", "git"); + + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); + } + + attrs.emplace("url", url2.to_string()); + + return inputFromAttrs(attrs); + } + + std::unique_ptr inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "git") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules") + throw Error("unsupported Git input attribute '%s'", name); + + auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (std::regex_search(*ref, badGitRefRegex)) + throw BadURL("invalid Git branch/tag name '%s'", *ref); + input->ref = *ref; + } + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + + input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false); + + input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false); + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc new file mode 100644 index 00000000000..0bee1d6b3f9 --- /dev/null +++ b/src/libfetchers/github.cc @@ -0,0 +1,195 @@ +#include "filetransfer.hh" +#include "cache.hh" +#include "fetchers.hh" +#include "globals.hh" +#include "store-api.hh" + +#include + +namespace nix::fetchers { + +std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); +std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); + +struct GitHubInput : Input +{ + std::string owner; + std::string repo; + std::optional ref; + std::optional rev; + + std::string type() const override { return "github"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && owner == other2->owner + && repo == other2->repo + && rev == other2->rev + && ref == other2->ref; + } + + bool isImmutable() const override + { + return (bool) rev || narHash; + } + + std::optional getRef() const override { return ref; } + + std::optional getRev() const override { return rev; } + + ParsedURL toURL() const override + { + auto path = owner + "/" + repo; + assert(!(ref && rev)); + if (ref) path += "/" + *ref; + if (rev) path += "/" + rev->to_string(Base16, false); + return ParsedURL { + .scheme = "github", + .path = path, + }; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("owner", owner); + attrs.emplace("repo", repo); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + return attrs; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto rev = this->rev; + auto ref = this->ref.value_or("master"); + + if (!rev) { + auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s", + owner, repo, ref); + auto json = nlohmann::json::parse( + readFile( + store->toRealPath( + downloadFile(store, url, "source", false).storePath))); + rev = Hash(std::string { json["sha"] }, htSHA1); + debug("HEAD revision for '%s' is %s", url, rev->gitRev()); + } + + auto input = std::make_shared(*this); + input->ref = {}; + input->rev = *rev; + + Attrs immutableAttrs({ + {"type", "git-tarball"}, + {"rev", rev->gitRev()}, + }); + + if (auto res = getCache()->lookup(store, immutableAttrs)) { + return { + Tree{ + .actualPath = store->toRealPath(res->second), + .storePath = std::move(res->second), + .info = TreeInfo { + .lastModified = getIntAttr(res->first, "lastModified"), + }, + }, + input + }; + } + + // FIXME: use regular /archive URLs instead? api.github.com + // might have stricter rate limits. + + auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", + owner, repo, rev->to_string(Base16, false)); + + std::string accessToken = settings.githubAccessToken.get(); + if (accessToken != "") + url += "?access_token=" + accessToken; + + auto tree = downloadTarball(store, url, "source", true); + + getCache()->add( + store, + immutableAttrs, + { + {"rev", rev->gitRev()}, + {"lastModified", *tree.info.lastModified} + }, + tree.storePath, + true); + + return {std::move(tree), input}; + } +}; + +struct GitHubInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "github") return nullptr; + + auto path = tokenizeString>(url.path, "/"); + auto input = std::make_unique(); + + if (path.size() == 2) { + } else if (path.size() == 3) { + if (std::regex_match(path[2], revRegex)) + input->rev = Hash(path[2], htSHA1); + else if (std::regex_match(path[2], refRegex)) + input->ref = path[2]; + else + throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); + } else + throw BadURL("GitHub URL '%s' is invalid", url.url); + + for (auto &[name, value] : url.query) { + if (name == "rev") { + if (input->rev) + throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url); + input->rev = Hash(value, htSHA1); + } + else if (name == "ref") { + if (!std::regex_match(value, refRegex)) + throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url); + if (input->ref) + throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url); + input->ref = value; + } + } + + if (input->ref && input->rev) + throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url); + + input->owner = path[0]; + input->repo = path[1]; + + return input; + } + + std::unique_ptr inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "github") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev") + throw Error("unsupported GitHub input attribute '%s'", name); + + auto input = std::make_unique(); + input->owner = getStrAttr(attrs, "owner"); + input->repo = getStrAttr(attrs, "repo"); + input->ref = maybeGetStrAttr(attrs, "ref"); + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk new file mode 100644 index 00000000000..d7143d8a60c --- /dev/null +++ b/src/libfetchers/local.mk @@ -0,0 +1,11 @@ +libraries += libfetchers + +libfetchers_NAME = libnixfetchers + +libfetchers_DIR := $(d) + +libfetchers_SOURCES := $(wildcard $(d)/*.cc) + +libfetchers_CXXFLAGS += -I src/libutil -I src/libstore + +libfetchers_LIBS = libutil libstore libnixrust diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc new file mode 100644 index 00000000000..2e0d4bf4d83 --- /dev/null +++ b/src/libfetchers/mercurial.cc @@ -0,0 +1,303 @@ +#include "fetchers.hh" +#include "cache.hh" +#include "globals.hh" +#include "tarfile.hh" +#include "store-api.hh" + +#include + +using namespace std::string_literals; + +namespace nix::fetchers { + +struct MercurialInput : Input +{ + ParsedURL url; + std::optional ref; + std::optional rev; + + MercurialInput(const ParsedURL & url) : url(url) + { } + + std::string type() const override { return "hg"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && url == other2->url + && rev == other2->rev + && ref == other2->ref; + } + + bool isImmutable() const override + { + return (bool) rev || narHash; + } + + std::optional getRef() const override { return ref; } + + std::optional getRev() const override { return rev; } + + ParsedURL toURL() const override + { + ParsedURL url2(url); + url2.scheme = "hg+" + url2.scheme; + if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); + if (ref) url2.query.insert_or_assign("ref", *ref); + return url; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("url", url.to_string()); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + return attrs; + } + + std::pair getActualUrl() const + { + bool isLocal = url.scheme == "file"; + return {isLocal, isLocal ? url.path : url.base}; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto name = "source"; + + auto input = std::make_shared(*this); + + auto [isLocal, actualUrl_] = getActualUrl(); + auto actualUrl = actualUrl_; // work around clang bug + + // FIXME: return lastModified. + + // FIXME: don't clone local repositories. + + if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) { + + bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == ""; + + if (!clean) { + + /* This is an unclean working tree. So copy all tracked + files. */ + + if (!settings.allowDirty) + throw Error("Mercurial tree '%s' is unclean", actualUrl); + + if (settings.warnDirty) + warn("Mercurial tree '%s' is unclean", actualUrl); + + input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl })); + + auto files = tokenizeString>( + runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); + + PathFilter filter = [&](const Path & p) -> bool { + assert(hasPrefix(p, actualUrl)); + std::string file(p, actualUrl.size() + 1); + + auto st = lstat(p); + + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && hasPrefix(*i, prefix); + } + + return files.count(file); + }; + + auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); + + return {Tree { + .actualPath = store->printStorePath(storePath), + .storePath = std::move(storePath), + }, input}; + } + } + + if (!input->ref) input->ref = "default"; + + auto getImmutableAttrs = [&]() + { + return Attrs({ + {"type", "hg"}, + {"name", name}, + {"rev", input->rev->gitRev()}, + }); + }; + + auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) + -> std::pair> + { + assert(input->rev); + assert(!rev || rev == input->rev); + return { + Tree{ + .actualPath = store->toRealPath(storePath), + .storePath = std::move(storePath), + .info = TreeInfo { + .revCount = getIntAttr(infoAttrs, "revCount"), + }, + }, + input + }; + }; + + if (input->rev) { + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + } + + assert(input->rev || input->ref); + auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref; + + Attrs mutableAttrs({ + {"type", "hg"}, + {"name", name}, + {"url", actualUrl}, + {"ref", *input->ref}, + }); + + if (auto res = getCache()->lookup(store, mutableAttrs)) { + auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); + if (!rev || rev == rev2) { + input->rev = rev2; + return makeResult(res->first, std::move(res->second)); + } + } + + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false)); + + /* If this is a commit hash that we already have, we don't + have to pull again. */ + if (!(input->rev + && pathExists(cacheDir) + && runProgram( + RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" }) + .killStderr(true)).second == "1")) + { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl)); + + if (pathExists(cacheDir)) { + try { + runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); + } + catch (ExecError & e) { + string transJournal = cacheDir + "/.hg/store/journal"; + /* hg throws "abandoned transaction" error only if this file exists */ + if (pathExists(transJournal)) { + runProgram("hg", true, { "recover", "-R", cacheDir }); + runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); + } else { + throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); + } + } + } else { + createDirs(dirOf(cacheDir)); + runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir }); + } + } + + auto tokens = tokenizeString>( + runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); + assert(tokens.size() == 3); + + input->rev = Hash(tokens[0], htSHA1); + auto revCount = std::stoull(tokens[1]); + input->ref = tokens[2]; + + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + + runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir }); + + deletePath(tmpDir + "/.hg_archival.txt"); + + auto storePath = store->addToStore(name, tmpDir); + + Attrs infoAttrs({ + {"rev", input->rev->gitRev()}, + {"revCount", (int64_t) revCount}, + }); + + if (!this->rev) + getCache()->add( + store, + mutableAttrs, + infoAttrs, + storePath, + false); + + getCache()->add( + store, + getImmutableAttrs(), + infoAttrs, + storePath, + true); + + return makeResult(infoAttrs, std::move(storePath)); + } +}; + +struct MercurialInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "hg+http" && + url.scheme != "hg+https" && + url.scheme != "hg+ssh" && + url.scheme != "hg+file") return nullptr; + + auto url2(url); + url2.scheme = std::string(url2.scheme, 3); + url2.query.clear(); + + Attrs attrs; + attrs.emplace("type", "hg"); + + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); + } + + attrs.emplace("url", url2.to_string()); + + return inputFromAttrs(attrs); + } + + std::unique_ptr inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "hg") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev") + throw Error("unsupported Mercurial input attribute '%s'", name); + + auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (!std::regex_match(*ref, refRegex)) + throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); + input->ref = *ref; + } + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc new file mode 100644 index 00000000000..ba2cc192e54 --- /dev/null +++ b/src/libfetchers/path.cc @@ -0,0 +1,148 @@ +#include "fetchers.hh" +#include "store-api.hh" + +namespace nix::fetchers { + +struct PathInput : Input +{ + Path path; + + /* Allow the user to pass in "fake" tree info attributes. This is + useful for making a pinned tree work the same as the repository + from which is exported + (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ + std::optional rev; + std::optional revCount; + std::optional lastModified; + + std::string type() const override { return "path"; } + + std::optional getRev() const override { return rev; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && path == other2->path + && rev == other2->rev + && revCount == other2->revCount + && lastModified == other2->lastModified; + } + + bool isImmutable() const override + { + return (bool) narHash; + } + + ParsedURL toURL() const override + { + auto query = attrsToQuery(toAttrsInternal()); + query.erase("path"); + return ParsedURL { + .scheme = "path", + .path = path, + .query = query, + }; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("path", path); + if (rev) + attrs.emplace("rev", rev->gitRev()); + if (revCount) + attrs.emplace("revCount", *revCount); + if (lastModified) + attrs.emplace("lastModified", *lastModified); + return attrs; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto input = std::make_shared(*this); + + // FIXME: check whether access to 'path' is allowed. + + auto storePath = store->maybeParseStorePath(path); + + if (storePath) + store->addTempRoot(*storePath); + + if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) + // FIXME: try to substitute storePath. + storePath = store->addToStore("source", path); + + return + { + Tree { + .actualPath = store->toRealPath(*storePath), + .storePath = std::move(*storePath), + .info = TreeInfo { + .revCount = revCount, + .lastModified = lastModified + } + }, + input + }; + } + +}; + +struct PathInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "path") return nullptr; + + auto input = std::make_unique(); + input->path = url.path; + + for (auto & [name, value] : url.query) + if (name == "rev") + input->rev = Hash(value, htSHA1); + else if (name == "revCount") { + uint64_t revCount; + if (!string2Int(value, revCount)) + throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); + input->revCount = revCount; + } + else if (name == "lastModified") { + time_t lastModified; + if (!string2Int(value, lastModified)) + throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); + input->lastModified = lastModified; + } + else + throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); + + return input; + } + + std::unique_ptr inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "path") return {}; + + auto input = std::make_unique(); + input->path = getStrAttr(attrs, "path"); + + for (auto & [name, value] : attrs) + if (name == "rev") + input->rev = Hash(getStrAttr(attrs, "rev"), htSHA1); + else if (name == "revCount") + input->revCount = getIntAttr(attrs, "revCount"); + else if (name == "lastModified") + input->lastModified = getIntAttr(attrs, "lastModified"); + else if (name == "type" || name == "path") + ; + else + throw Error("unsupported path input attribute '%s'", name); + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc new file mode 100644 index 00000000000..7966da3144b --- /dev/null +++ b/src/libfetchers/tarball.cc @@ -0,0 +1,275 @@ +#include "fetchers.hh" +#include "cache.hh" +#include "filetransfer.hh" +#include "globals.hh" +#include "store-api.hh" +#include "archive.hh" +#include "tarfile.hh" + +namespace nix::fetchers { + +DownloadFileResult downloadFile( + ref store, + const std::string & url, + const std::string & name, + bool immutable) +{ + // FIXME: check store + + Attrs inAttrs({ + {"type", "file"}, + {"url", url}, + {"name", name}, + }); + + auto cached = getCache()->lookupExpired(store, inAttrs); + + auto useCached = [&]() -> DownloadFileResult + { + return { + .storePath = std::move(cached->storePath), + .etag = getStrAttr(cached->infoAttrs, "etag"), + .effectiveUrl = getStrAttr(cached->infoAttrs, "url") + }; + }; + + if (cached && !cached->expired) + return useCached(); + + FileTransferRequest request(url); + if (cached) + request.expectedETag = getStrAttr(cached->infoAttrs, "etag"); + FileTransferResult res; + try { + res = getFileTransfer()->download(request); + } catch (FileTransferError & e) { + if (cached) { + warn("%s; using cached version", e.msg()); + return useCached(); + } else + throw; + } + + // FIXME: write to temporary file. + + Attrs infoAttrs({ + {"etag", res.etag}, + {"url", res.effectiveUri}, + }); + + std::optional storePath; + + if (res.cached) { + assert(cached); + assert(request.expectedETag == res.etag); + storePath = std::move(cached->storePath); + } else { + StringSink sink; + dumpString(*res.data, sink); + auto hash = hashString(htSHA256, *res.data); + ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name)); + info.narHash = hashString(htSHA256, *sink.s); + info.narSize = sink.s->size(); + info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash); + auto source = StringSource { *sink.s }; + store->addToStore(info, source, NoRepair, NoCheckSigs); + storePath = std::move(info.path); + } + + getCache()->add( + store, + inAttrs, + infoAttrs, + *storePath, + immutable); + + if (url != res.effectiveUri) + getCache()->add( + store, + { + {"type", "file"}, + {"url", res.effectiveUri}, + {"name", name}, + }, + infoAttrs, + *storePath, + immutable); + + return { + .storePath = std::move(*storePath), + .etag = res.etag, + .effectiveUrl = res.effectiveUri, + }; +} + +Tree downloadTarball( + ref store, + const std::string & url, + const std::string & name, + bool immutable) +{ + Attrs inAttrs({ + {"type", "tarball"}, + {"url", url}, + {"name", name}, + }); + + auto cached = getCache()->lookupExpired(store, inAttrs); + + if (cached && !cached->expired) + return Tree { + .actualPath = store->toRealPath(cached->storePath), + .storePath = std::move(cached->storePath), + .info = TreeInfo { + .lastModified = getIntAttr(cached->infoAttrs, "lastModified"), + }, + }; + + auto res = downloadFile(store, url, name, immutable); + + std::optional unpackedStorePath; + time_t lastModified; + + if (cached && res.etag != "" && getStrAttr(cached->infoAttrs, "etag") == res.etag) { + unpackedStorePath = std::move(cached->storePath); + lastModified = getIntAttr(cached->infoAttrs, "lastModified"); + } else { + Path tmpDir = createTempDir(); + AutoDelete autoDelete(tmpDir, true); + unpackTarfile(store->toRealPath(res.storePath), tmpDir); + auto members = readDirectory(tmpDir); + if (members.size() != 1) + throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); + auto topDir = tmpDir + "/" + members.begin()->name; + lastModified = lstat(topDir).st_mtime; + unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair); + } + + Attrs infoAttrs({ + {"lastModified", lastModified}, + {"etag", res.etag}, + }); + + getCache()->add( + store, + inAttrs, + infoAttrs, + *unpackedStorePath, + immutable); + + return Tree { + .actualPath = store->toRealPath(*unpackedStorePath), + .storePath = std::move(*unpackedStorePath), + .info = TreeInfo { + .lastModified = lastModified, + }, + }; +} + +struct TarballInput : Input +{ + ParsedURL url; + std::optional hash; + + TarballInput(const ParsedURL & url) : url(url) + { } + + std::string type() const override { return "tarball"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && to_string() == other2->to_string() + && hash == other2->hash; + } + + bool isImmutable() const override + { + return hash || narHash; + } + + ParsedURL toURL() const override + { + auto url2(url); + // NAR hashes are preferred over file hashes since tar/zip files + // don't have a canonical representation. + if (narHash) + url2.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); + else if (hash) + url2.query.insert_or_assign("hash", hash->to_string(SRI, true)); + return url2; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("url", url.to_string()); + if (hash) + attrs.emplace("hash", hash->to_string(SRI, true)); + return attrs; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto tree = downloadTarball(store, url.to_string(), "source", false); + + auto input = std::make_shared(*this); + input->narHash = store->queryPathInfo(tree.storePath)->narHash; + + return {std::move(tree), input}; + } +}; + +struct TarballInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr; + + if (!hasSuffix(url.path, ".zip") + && !hasSuffix(url.path, ".tar") + && !hasSuffix(url.path, ".tar.gz") + && !hasSuffix(url.path, ".tar.xz") + && !hasSuffix(url.path, ".tar.bz2")) + return nullptr; + + auto input = std::make_unique(url); + + auto hash = input->url.query.find("hash"); + if (hash != input->url.query.end()) { + // FIXME: require SRI hash. + input->hash = Hash(hash->second); + input->url.query.erase(hash); + } + + auto narHash = input->url.query.find("narHash"); + if (narHash != input->url.query.end()) { + // FIXME: require SRI hash. + input->narHash = Hash(narHash->second); + input->url.query.erase(narHash); + } + + return input; + } + + std::unique_ptr inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "hash") + throw Error("unsupported tarball input attribute '%s'", name); + + auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); + if (auto hash = maybeGetStrAttr(attrs, "hash")) + input->hash = newHashAllowEmpty(*hash, htUnknown); + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc new file mode 100644 index 00000000000..b2d8cfc8d03 --- /dev/null +++ b/src/libfetchers/tree-info.cc @@ -0,0 +1,14 @@ +#include "tree-info.hh" +#include "store-api.hh" + +#include + +namespace nix::fetchers { + +StorePath TreeInfo::computeStorePath(Store & store) const +{ + assert(narHash); + return store.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "source"); +} + +} diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh new file mode 100644 index 00000000000..2c734728135 --- /dev/null +++ b/src/libfetchers/tree-info.hh @@ -0,0 +1,29 @@ +#pragma once + +#include "path.hh" +#include "hash.hh" + +#include + +namespace nix { class Store; } + +namespace nix::fetchers { + +struct TreeInfo +{ + Hash narHash; + std::optional revCount; + std::optional lastModified; + + bool operator ==(const TreeInfo & other) const + { + return + narHash == other.narHash + && revCount == other.revCount + && lastModified == other.lastModified; + } + + StorePath computeStorePath(Store & store) const; +}; + +} diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 98693d78a7f..051668e53a2 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,29 +1,69 @@ #include "common-args.hh" #include "globals.hh" +#include "loggers.hh" namespace nix { MixCommonArgs::MixCommonArgs(const string & programName) : programName(programName) { - mkFlag('v', "verbose", "increase verbosity level", []() { - verbosity = (Verbosity) (verbosity + 1); + addFlag({ + .longName = "verbose", + .shortName = 'v', + .description = "increase verbosity level", + .handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }}, }); - mkFlag(0, "quiet", "decrease verbosity level", []() { - verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; + addFlag({ + .longName = "quiet", + .description = "decrease verbosity level", + .handler = {[]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }}, }); - mkFlag(0, "debug", "enable debug output", []() { - verbosity = lvlDebug; + addFlag({ + .longName = "debug", + .description = "enable debug output", + .handler = {[]() { verbosity = lvlDebug; }}, }); - mkFlag(0, "option", {"name", "value"}, "set a Nix configuration option (overriding nix.conf)", 2, - [](Strings ss) { - auto name = ss.front(); ss.pop_front(); - auto value = ss.front(); - settings.set(name, value); - }); + addFlag({ + .longName = "option", + .description = "set a Nix configuration option (overriding nix.conf)", + .labels = {"name", "value"}, + .handler = {[](std::string name, std::string value) { + try { + globalConfig.set(name, value); + } catch (UsageError & e) { + warn(e.what()); + } + }}, + }); + + addFlag({ + .longName = "log-format", + .description = "format of log output; \"raw\", \"internal-json\", \"bar\" " + "or \"bar-with-logs\"", + .labels = {"format"}, + .handler = {[](std::string format) { setLogFormat(format); }}, + }); + + addFlag({ + .longName = "max-jobs", + .shortName = 'j', + .description = "maximum number of parallel builds", + .labels = Strings{"jobs"}, + .handler = {[=](std::string s) { + settings.set("max-jobs", s); + }} + }); + + std::string cat = "config"; + globalConfig.convertToArgs(*this, cat); + + // Backward compatibility hack: nix-env already had a --system flag. + if (programName == "nix-env") longFlags.erase("system"); + + hiddenCategories.insert(cat); } } diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh index 2c0d71edd81..a4de3dccf0a 100644 --- a/src/libmain/common-args.hh +++ b/src/libmain/common-args.hh @@ -12,7 +12,7 @@ struct MixCommonArgs : virtual Args struct MixDryRun : virtual Args { - bool dryRun; + bool dryRun = false; MixDryRun() { @@ -20,4 +20,14 @@ struct MixDryRun : virtual Args } }; +struct MixJSON : virtual Args +{ + bool json = false; + + MixJSON() + { + mkFlag(0, "json", "produce JSON output", &json); + } +}; + } diff --git a/src/libmain/local.mk b/src/libmain/local.mk index f1fd3eb7242..a8eed6c65fe 100644 --- a/src/libmain/local.mk +++ b/src/libmain/local.mk @@ -6,9 +6,11 @@ libmain_DIR := $(d) libmain_SOURCES := $(wildcard $(d)/*.cc) +libmain_CXXFLAGS += -I src/libutil -I src/libstore + libmain_LDFLAGS = $(OPENSSL_LIBS) -libmain_LIBS = libstore libutil libformat +libmain_LIBS = libstore libutil libmain_ALLOW_UNDEFINED = 1 diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc new file mode 100644 index 00000000000..c44bb640869 --- /dev/null +++ b/src/libmain/loggers.cc @@ -0,0 +1,52 @@ +#include "loggers.hh" +#include "progress-bar.hh" + +namespace nix { + +LogFormat defaultLogFormat = LogFormat::raw; + +LogFormat parseLogFormat(const std::string & logFormatStr) { + if (logFormatStr == "raw") + return LogFormat::raw; + else if (logFormatStr == "raw-with-logs") + return LogFormat::rawWithLogs; + else if (logFormatStr == "internal-json") + return LogFormat::internalJson; + else if (logFormatStr == "bar") + return LogFormat::bar; + else if (logFormatStr == "bar-with-logs") + return LogFormat::barWithLogs; + throw Error("option 'log-format' has an invalid value '%s'", logFormatStr); +} + +Logger * makeDefaultLogger() { + switch (defaultLogFormat) { + case LogFormat::raw: + return makeSimpleLogger(false); + case LogFormat::rawWithLogs: + return makeSimpleLogger(true); + case LogFormat::internalJson: + return makeJSONLogger(*makeSimpleLogger()); + case LogFormat::bar: + return makeProgressBar(); + case LogFormat::barWithLogs: + return makeProgressBar(true); + default: + abort(); + } +} + +void setLogFormat(const std::string & logFormatStr) { + setLogFormat(parseLogFormat(logFormatStr)); +} + +void setLogFormat(const LogFormat & logFormat) { + defaultLogFormat = logFormat; + createDefaultLogger(); +} + +void createDefaultLogger() { + logger = makeDefaultLogger(); +} + +} diff --git a/src/libmain/loggers.hh b/src/libmain/loggers.hh new file mode 100644 index 00000000000..cada0311049 --- /dev/null +++ b/src/libmain/loggers.hh @@ -0,0 +1,20 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +enum class LogFormat { + raw, + rawWithLogs, + internalJson, + bar, + barWithLogs, +}; + +void setLogFormat(const std::string & logFormatStr); +void setLogFormat(const LogFormat & logFormat); + +void createDefaultLogger(); + +} diff --git a/src/libmain/nix-main.pc.in b/src/libmain/nix-main.pc.in index de1bdf706f7..37b03dcd42c 100644 --- a/src/libmain/nix-main.pc.in +++ b/src/libmain/nix-main.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixmain -Cflags: -I${includedir}/nix +Cflags: -I${includedir}/nix -std=c++17 diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc new file mode 100644 index 00000000000..95a9187de69 --- /dev/null +++ b/src/libmain/progress-bar.cc @@ -0,0 +1,491 @@ +#include "progress-bar.hh" +#include "util.hh" +#include "sync.hh" +#include "store-api.hh" +#include "names.hh" + +#include +#include +#include +#include + +namespace nix { + +static std::string getS(const std::vector & fields, size_t n) +{ + assert(n < fields.size()); + assert(fields[n].type == Logger::Field::tString); + return fields[n].s; +} + +static uint64_t getI(const std::vector & fields, size_t n) +{ + assert(n < fields.size()); + assert(fields[n].type == Logger::Field::tInt); + return fields[n].i; +} + +static std::string_view storePathToName(std::string_view path) +{ + auto base = baseNameOf(path); + auto i = base.find('-'); + return i == std::string::npos ? base.substr(0, 0) : base.substr(i + 1); +} + +class ProgressBar : public Logger +{ +private: + + struct ActInfo + { + std::string s, lastLine, phase; + ActivityType type = actUnknown; + uint64_t done = 0; + uint64_t expected = 0; + uint64_t running = 0; + uint64_t failed = 0; + std::map expectedByType; + bool visible = true; + ActivityId parent; + std::optional name; + }; + + struct ActivitiesByType + { + std::map::iterator> its; + uint64_t done = 0; + uint64_t expected = 0; + uint64_t failed = 0; + }; + + struct State + { + std::list activities; + std::map::iterator> its; + + std::map activitiesByType; + + uint64_t filesLinked = 0, bytesLinked = 0; + + uint64_t corruptedPaths = 0, untrustedPaths = 0; + + bool active = true; + bool haveUpdate = true; + }; + + Sync state_; + + std::thread updateThread; + + std::condition_variable quitCV, updateCV; + + bool printBuildLogs; + bool isTTY; + +public: + + ProgressBar(bool printBuildLogs, bool isTTY) + : printBuildLogs(printBuildLogs) + , isTTY(isTTY) + { + state_.lock()->active = isTTY; + updateThread = std::thread([&]() { + auto state(state_.lock()); + while (state->active) { + if (!state->haveUpdate) + state.wait(updateCV); + draw(*state); + state.wait_for(quitCV, std::chrono::milliseconds(50)); + } + }); + } + + ~ProgressBar() + { + stop(); + updateThread.join(); + } + + void stop() override + { + auto state(state_.lock()); + if (!state->active) return; + state->active = false; + writeToStderr("\r\e[K"); + updateCV.notify_one(); + quitCV.notify_one(); + } + + bool isVerbose() override { + return printBuildLogs; + } + + void log(Verbosity lvl, const FormatOrString & fs) override + { + auto state(state_.lock()); + log(*state, lvl, fs.s); + } + + void logEI(const ErrorInfo &ei) override + { + auto state(state_.lock()); + + std::stringstream oss; + oss << ei; + + log(*state, ei.level, oss.str()); + } + + void log(State & state, Verbosity lvl, const std::string & s) + { + if (state.active) { + writeToStderr("\r\e[K" + filterANSIEscapes(s, !isTTY) + ANSI_NORMAL "\n"); + draw(state); + } else { + auto s2 = s + ANSI_NORMAL "\n"; + if (!isTTY) s2 = filterANSIEscapes(s2, true); + writeToStderr(s2); + } + } + + void startActivity(ActivityId act, Verbosity lvl, ActivityType type, + const std::string & s, const Fields & fields, ActivityId parent) override + { + auto state(state_.lock()); + + if (lvl <= verbosity && !s.empty() && type != actBuildWaiting) + log(*state, lvl, s + "..."); + + state->activities.emplace_back(ActInfo()); + auto i = std::prev(state->activities.end()); + i->s = s; + i->type = type; + i->parent = parent; + state->its.emplace(act, i); + state->activitiesByType[type].its.emplace(act, i); + + if (type == actBuild) { + std::string name(storePathToName(getS(fields, 0))); + if (hasSuffix(name, ".drv")) + name = name.substr(0, name.size() - 4); + i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name); + auto machineName = getS(fields, 1); + if (machineName != "") + i->s += fmt(" on " ANSI_BOLD "%s" ANSI_NORMAL, machineName); + auto curRound = getI(fields, 2); + auto nrRounds = getI(fields, 3); + if (nrRounds != 1) + i->s += fmt(" (round %d/%d)", curRound, nrRounds); + i->name = DrvName(name).name; + } + + if (type == actSubstitute) { + auto name = storePathToName(getS(fields, 0)); + auto sub = getS(fields, 1); + i->s = fmt( + hasPrefix(sub, "local") + ? "copying " ANSI_BOLD "%s" ANSI_NORMAL " from %s" + : "fetching " ANSI_BOLD "%s" ANSI_NORMAL " from %s", + name, sub); + } + + if (type == actPostBuildHook) { + auto name = storePathToName(getS(fields, 0)); + if (hasSuffix(name, ".drv")) + name = name.substr(0, name.size() - 4); + i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name); + i->name = DrvName(name).name; + } + + if (type == actQueryPathInfo) { + auto name = storePathToName(getS(fields, 0)); + i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1)); + } + + if ((type == actFileTransfer && hasAncestor(*state, actCopyPath, parent)) + || (type == actFileTransfer && hasAncestor(*state, actQueryPathInfo, parent)) + || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent))) + i->visible = false; + + update(*state); + } + + /* Check whether an activity has an ancestore with the specified + type. */ + bool hasAncestor(State & state, ActivityType type, ActivityId act) + { + while (act != 0) { + auto i = state.its.find(act); + if (i == state.its.end()) break; + if (i->second->type == type) return true; + act = i->second->parent; + } + return false; + } + + void stopActivity(ActivityId act) override + { + auto state(state_.lock()); + + auto i = state->its.find(act); + if (i != state->its.end()) { + + auto & actByType = state->activitiesByType[i->second->type]; + actByType.done += i->second->done; + actByType.failed += i->second->failed; + + for (auto & j : i->second->expectedByType) + state->activitiesByType[j.first].expected -= j.second; + + actByType.its.erase(act); + state->activities.erase(i->second); + state->its.erase(i); + } + + update(*state); + } + + void result(ActivityId act, ResultType type, const std::vector & fields) override + { + auto state(state_.lock()); + + if (type == resFileLinked) { + state->filesLinked++; + state->bytesLinked += getI(fields, 0); + update(*state); + } + + else if (type == resBuildLogLine || type == resPostBuildLogLine) { + auto lastLine = trim(getS(fields, 0)); + if (!lastLine.empty()) { + auto i = state->its.find(act); + assert(i != state->its.end()); + ActInfo info = *i->second; + if (printBuildLogs) { + auto suffix = "> "; + if (type == resPostBuildLogLine) { + suffix = " (post)> "; + } + log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + suffix + ANSI_NORMAL + lastLine); + } else { + state->activities.erase(i->second); + info.lastLine = lastLine; + state->activities.emplace_back(info); + i->second = std::prev(state->activities.end()); + update(*state); + } + } + } + + else if (type == resUntrustedPath) { + state->untrustedPaths++; + update(*state); + } + + else if (type == resCorruptedPath) { + state->corruptedPaths++; + update(*state); + } + + else if (type == resSetPhase) { + auto i = state->its.find(act); + assert(i != state->its.end()); + i->second->phase = getS(fields, 0); + update(*state); + } + + else if (type == resProgress) { + auto i = state->its.find(act); + assert(i != state->its.end()); + ActInfo & actInfo = *i->second; + actInfo.done = getI(fields, 0); + actInfo.expected = getI(fields, 1); + actInfo.running = getI(fields, 2); + actInfo.failed = getI(fields, 3); + update(*state); + } + + else if (type == resSetExpected) { + auto i = state->its.find(act); + assert(i != state->its.end()); + ActInfo & actInfo = *i->second; + auto type = (ActivityType) getI(fields, 0); + auto & j = actInfo.expectedByType[type]; + state->activitiesByType[type].expected -= j; + j = getI(fields, 1); + state->activitiesByType[type].expected += j; + update(*state); + } + } + + void update(State & state) + { + state.haveUpdate = true; + updateCV.notify_one(); + } + + void draw(State & state) + { + state.haveUpdate = false; + if (!state.active) return; + + std::string line; + + std::string status = getStatus(state); + if (!status.empty()) { + line += '['; + line += status; + line += "]"; + } + + if (!state.activities.empty()) { + if (!status.empty()) line += " "; + auto i = state.activities.rbegin(); + + while (i != state.activities.rend() && (!i->visible || (i->s.empty() && i->lastLine.empty()))) + ++i; + + if (i != state.activities.rend()) { + line += i->s; + if (!i->phase.empty()) { + line += " ("; + line += i->phase; + line += ")"; + } + if (!i->lastLine.empty()) { + if (!i->s.empty()) line += ": "; + line += i->lastLine; + } + } + } + + auto width = getWindowSize().second; + if (width <= 0) width = std::numeric_limits::max(); + + writeToStderr("\r" + filterANSIEscapes(line, false, width) + "\e[K"); + } + + std::string getStatus(State & state) + { + auto MiB = 1024.0 * 1024.0; + + std::string res; + + auto renderActivity = [&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) { + auto & act = state.activitiesByType[type]; + uint64_t done = act.done, expected = act.done, running = 0, failed = act.failed; + for (auto & j : act.its) { + done += j.second->done; + expected += j.second->expected; + running += j.second->running; + failed += j.second->failed; + } + + expected = std::max(expected, act.expected); + + std::string s; + + if (running || done || expected || failed) { + if (running) + if (expected != 0) + s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, + running / unit, done / unit, expected / unit); + else + s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL, + running / unit, done / unit); + else if (expected != done) + if (expected != 0) + s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, + done / unit, expected / unit); + else + s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL, done / unit); + else + s = fmt(done ? ANSI_GREEN + numberFmt + ANSI_NORMAL : numberFmt, done / unit); + s = fmt(itemFmt, s); + + if (failed) + s += fmt(" (" ANSI_RED "%d failed" ANSI_NORMAL ")", failed / unit); + } + + return s; + }; + + auto showActivity = [&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) { + auto s = renderActivity(type, itemFmt, numberFmt, unit); + if (s.empty()) return; + if (!res.empty()) res += ", "; + res += s; + }; + + showActivity(actBuilds, "%s built"); + + auto s1 = renderActivity(actCopyPaths, "%s copied"); + auto s2 = renderActivity(actCopyPath, "%s MiB", "%.1f", MiB); + + if (!s1.empty() || !s2.empty()) { + if (!res.empty()) res += ", "; + if (s1.empty()) res += "0 copied"; else res += s1; + if (!s2.empty()) { res += " ("; res += s2; res += ')'; } + } + + showActivity(actFileTransfer, "%s MiB DL", "%.1f", MiB); + + { + auto s = renderActivity(actOptimiseStore, "%s paths optimised"); + if (s != "") { + s += fmt(", %.1f MiB / %d inodes freed", state.bytesLinked / MiB, state.filesLinked); + if (!res.empty()) res += ", "; + res += s; + } + } + + // FIXME: don't show "done" paths in green. + showActivity(actVerifyPaths, "%s paths verified"); + + if (state.corruptedPaths) { + if (!res.empty()) res += ", "; + res += fmt(ANSI_RED "%d corrupted" ANSI_NORMAL, state.corruptedPaths); + } + + if (state.untrustedPaths) { + if (!res.empty()) res += ", "; + res += fmt(ANSI_RED "%d untrusted" ANSI_NORMAL, state.untrustedPaths); + } + + return res; + } + + void writeToStdout(std::string_view s) override + { + auto state(state_.lock()); + if (state->active) { + std::cerr << "\r\e[K"; + Logger::writeToStdout(s); + draw(*state); + } else { + Logger::writeToStdout(s); + } + } +}; + +Logger * makeProgressBar(bool printBuildLogs) +{ + return new ProgressBar( + printBuildLogs, + isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb" + ); +} + +void startProgressBar(bool printBuildLogs) +{ + logger = makeProgressBar(printBuildLogs); +} + +void stopProgressBar() +{ + auto progressBar = dynamic_cast(logger); + if (progressBar) progressBar->stop(); + +} + +} diff --git a/src/libmain/progress-bar.hh b/src/libmain/progress-bar.hh new file mode 100644 index 00000000000..7f0dafecf8c --- /dev/null +++ b/src/libmain/progress-bar.hh @@ -0,0 +1,13 @@ +#pragma once + +#include "logging.hh" + +namespace nix { + +Logger * makeProgressBar(bool printBuildLogs = false); + +void startProgressBar(bool printBuildLogs = false); + +void stopProgressBar(); + +} diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 12f083c7f79..dc6d5e41394 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -1,10 +1,8 @@ -#include "config.h" - -#include "common-args.hh" #include "globals.hh" #include "shared.hh" #include "store-api.hh" #include "util.hh" +#include "loggers.hh" #include #include @@ -31,45 +29,45 @@ void printGCWarning() if (!gcWarning) return; static bool haveWarned = false; warnOnce(haveWarned, - "you did not specify ‘--add-root’; " + "you did not specify '--add-root'; " "the result might be removed by the garbage collector"); } -void printMissing(ref store, const PathSet & paths) +void printMissing(ref store, const std::vector & paths, Verbosity lvl) { unsigned long long downloadSize, narSize; - PathSet willBuild, willSubstitute, unknown; + StorePathSet willBuild, willSubstitute, unknown; store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); - printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize); + printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize, lvl); } -void printMissing(ref store, const PathSet & willBuild, - const PathSet & willSubstitute, const PathSet & unknown, - unsigned long long downloadSize, unsigned long long narSize) +void printMissing(ref store, const StorePathSet & willBuild, + const StorePathSet & willSubstitute, const StorePathSet & unknown, + unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl) { if (!willBuild.empty()) { - printInfo(format("these derivations will be built:")); - Paths sorted = store->topoSortPaths(willBuild); + printMsg(lvl, "these derivations will be built:"); + auto sorted = store->topoSortPaths(willBuild); reverse(sorted.begin(), sorted.end()); for (auto & i : sorted) - printInfo(format(" %1%") % i); + printMsg(lvl, fmt(" %s", store->printStorePath(i))); } if (!willSubstitute.empty()) { - printInfo(format("these paths will be fetched (%.2f MiB download, %.2f MiB unpacked):") - % (downloadSize / (1024.0 * 1024.0)) - % (narSize / (1024.0 * 1024.0))); + printMsg(lvl, fmt("these paths will be fetched (%.2f MiB download, %.2f MiB unpacked):", + downloadSize / (1024.0 * 1024.0), + narSize / (1024.0 * 1024.0))); for (auto & i : willSubstitute) - printInfo(format(" %1%") % i); + printMsg(lvl, fmt(" %s", store->printStorePath(i))); } if (!unknown.empty()) { - printInfo(format("don't know how to build these paths%1%:") - % (settings.readOnlyMode ? " (may be caused by read-only store access)" : "")); + printMsg(lvl, fmt("don't know how to build these paths%s:", + (settings.readOnlyMode ? " (may be caused by read-only store access)" : ""))); for (auto & i : unknown) - printInfo(format(" %1%") % i); + printMsg(lvl, fmt(" %s", store->printStorePath(i))); } } @@ -78,11 +76,12 @@ string getArg(const string & opt, Strings::iterator & i, const Strings::iterator & end) { ++i; - if (i == end) throw UsageError(format("‘%1%’ requires an argument") % opt); + if (i == end) throw UsageError("'%1%' requires an argument", opt); return *i; } +#if OPENSSL_VERSION_NUMBER < 0x10101000L /* OpenSSL is not thread-safe by default - it will randomly crash unless the user supplies a mutex locking function. So let's do that. */ @@ -95,6 +94,10 @@ static void opensslLockCallback(int mode, int type, const char * file, int line) else opensslLocks[type].unlock(); } +#endif + + +static void sigHandler(int signo) { } void initNix() @@ -105,31 +108,37 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif - logger = makeDefaultLogger(); - +#if OPENSSL_VERSION_NUMBER < 0x10101000L /* Initialise OpenSSL locking. */ opensslLocks = std::vector(CRYPTO_num_locks()); CRYPTO_set_locking_callback(opensslLockCallback); +#endif - settings.processEnvironment(); - settings.loadConfFile(); + loadConfFile(); startSignalHandlerThread(); - /* Ignore SIGPIPE. */ + /* Reset SIGCHLD to its default. */ struct sigaction act; sigemptyset(&act.sa_mask); - act.sa_handler = SIG_IGN; - act.sa_flags = 0; - if (sigaction(SIGPIPE, &act, 0)) - throw SysError("ignoring SIGPIPE"); - - /* Reset SIGCHLD to its default. */ act.sa_handler = SIG_DFL; act.sa_flags = 0; if (sigaction(SIGCHLD, &act, 0)) throw SysError("resetting SIGCHLD"); + /* Install a dummy SIGUSR1 handler for use with pthread_kill(). */ + act.sa_handler = sigHandler; + if (sigaction(SIGUSR1, &act, 0)) throw SysError("handling SIGUSR1"); + +#if __APPLE__ + /* HACK: on darwin, we need can’t use sigprocmask with SIGWINCH. + * Instead, add a dummy sigaction handler, and signalHandlerThread + * can handle the rest. */ + struct sigaction sa; + sa.sa_handler = sigHandler; + if (sigaction(SIGWINCH, &sa, 0)) throw SysError("handling SIGWINCH"); +#endif + /* Register a SIGSEGV handler to detect stack overflows. */ detectStackOverflow(); @@ -143,83 +152,105 @@ void initNix() gettimeofday(&tv, 0); srandom(tv.tv_usec); - if (char *pack = getenv("_NIX_OPTIONS")) - settings.unpack(pack); + /* On macOS, don't use the per-session TMPDIR (as set e.g. by + sshd). This breaks build users because they don't have access + to the TMPDIR, in particular in ‘nix-store --serve’. */ +#if __APPLE__ + if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) + unsetenv("TMPDIR"); +#endif } -struct LegacyArgs : public MixCommonArgs +LegacyArgs::LegacyArgs(const std::string & programName, + std::function parseArg) + : MixCommonArgs(programName), parseArg(parseArg) { - std::function parseArg; + addFlag({ + .longName = "no-build-output", + .shortName = 'Q', + .description = "do not show build output", + .handler = {[&]() {setLogFormat(LogFormat::raw); }}, + }); - LegacyArgs(const std::string & programName, - std::function parseArg) - : MixCommonArgs(programName), parseArg(parseArg) - { - mkFlag('Q', "no-build-output", "do not show build output", - &settings.verboseBuild, false); + addFlag({ + .longName = "keep-failed", + .shortName ='K', + .description = "keep temporary directories of failed builds", + .handler = {&(bool&) settings.keepFailed, true}, + }); - mkFlag('K', "keep-failed", "keep temporary directories of failed builds", - &settings.keepFailed); + addFlag({ + .longName = "keep-going", + .shortName ='k', + .description = "keep going after a build fails", + .handler = {&(bool&) settings.keepGoing, true}, + }); - mkFlag('k', "keep-going", "keep going after a build fails", - &settings.keepGoing); + addFlag({ + .longName = "fallback", + .description = "build from source if substitution fails", + .handler = {&(bool&) settings.tryFallback, true}, + }); - mkFlag(0, "fallback", "build from source if substitution fails", []() { - settings.set("build-fallback", "true"); + auto intSettingAlias = [&](char shortName, const std::string & longName, + const std::string & description, const std::string & dest) { + mkFlag(shortName, longName, description, [=](unsigned int n) { + settings.set(dest, std::to_string(n)); }); + }; - auto intSettingAlias = [&](char shortName, const std::string & longName, - const std::string & description, const std::string & dest) { - mkFlag(shortName, longName, description, [=](unsigned int n) { - settings.set(dest, std::to_string(n)); - }); - }; + intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "cores"); + intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "max-silent-time"); + intSettingAlias(0, "timeout", "number of seconds before a build is killed", "timeout"); - intSettingAlias('j', "max-jobs", "maximum number of parallel builds", "build-max-jobs"); - intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "build-cores"); - intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "build-max-silent-time"); - intSettingAlias(0, "timeout", "number of seconds before a build is killed", "build-timeout"); + mkFlag(0, "readonly-mode", "do not write to the Nix store", + &settings.readOnlyMode); - mkFlag(0, "readonly-mode", "do not write to the Nix store", - &settings.readOnlyMode); + mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'", + &gcWarning, false); - mkFlag(0, "no-build-hook", "disable use of the build hook mechanism", - &settings.useBuildHook, false); + addFlag({ + .longName = "store", + .description = "URI of the Nix store to use", + .labels = {"store-uri"}, + .handler = {&(std::string&) settings.storeUri}, + }); +} - mkFlag(0, "show-trace", "show Nix expression stack trace in evaluation errors", - &settings.showTrace); - mkFlag(0, "no-gc-warning", "disable warning about not using ‘--add-root’", - &gcWarning, false); - } +bool LegacyArgs::processFlag(Strings::iterator & pos, Strings::iterator end) +{ + if (MixCommonArgs::processFlag(pos, end)) return true; + bool res = parseArg(pos, end); + if (res) ++pos; + return res; +} - bool processFlag(Strings::iterator & pos, Strings::iterator end) override - { - if (MixCommonArgs::processFlag(pos, end)) return true; - bool res = parseArg(pos, end); - if (res) ++pos; - return res; - } - bool processArgs(const Strings & args, bool finish) override - { - if (args.empty()) return true; - assert(args.size() == 1); - Strings ss(args); - auto pos = ss.begin(); - if (!parseArg(pos, ss.end())) - throw UsageError(format("unexpected argument ‘%1%’") % args.front()); - return true; - } -}; +bool LegacyArgs::processArgs(const Strings & args, bool finish) +{ + if (args.empty()) return true; + assert(args.size() == 1); + Strings ss(args); + auto pos = ss.begin(); + if (!parseArg(pos, ss.end())) + throw UsageError("unexpected argument '%1%'", args.front()); + return true; +} void parseCmdLine(int argc, char * * argv, std::function parseArg) { - LegacyArgs(baseNameOf(argv[0]), parseArg).parseCmdline(argvToStrings(argc, argv)); - settings.update(); + parseCmdLine(std::string(baseNameOf(argv[0])), argvToStrings(argc, argv), parseArg); +} + + +void parseCmdLine(const string & programName, const Strings & args, + std::function parseArg) +{ + LegacyArgs(programName, parseArg).parseCmdline(args); } @@ -235,7 +266,10 @@ void printVersion(const string & programName) cfg.push_back("signed-caches"); #endif std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; - std::cout << "Configuration file: " << settings.nixConfDir + "/nix.conf" << "\n"; + std::cout << "System configuration file: " << settings.nixConfDir + "/nix.conf" << "\n"; + std::cout << "User configuration files: " << + concatStringsSep(":", settings.nixUserConfFiles) + << "\n"; std::cout << "Store directory: " << settings.nixStore << "\n"; std::cout << "State directory: " << settings.nixStateDir << "\n"; } @@ -245,14 +279,19 @@ void printVersion(const string & programName) void showManPage(const string & name) { - restoreSIGPIPE(); - execlp("man", "man", name.c_str(), NULL); - throw SysError(format("command ‘man %1%’ failed") % name.c_str()); + restoreSignals(); + setenv("MANPATH", settings.nixManDir.c_str(), 1); + execlp("man", "man", name.c_str(), nullptr); + throw SysError("command 'man %1%' failed", name.c_str()); } int handleExceptions(const string & programName, std::function fun) { + ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this + + ErrorInfo::programName = baseNameOf(programName); + string error = ANSI_RED "error:" ANSI_NORMAL " "; try { try { @@ -262,20 +301,21 @@ int handleExceptions(const string & programName, std::function fun) condition is discharged before we reach printMsg() below, since otherwise it will throw an (uncaught) exception. */ - interruptThrown = true; + setInterruptThrown(); throw; } } catch (Exit & e) { return e.status; } catch (UsageError & e) { - printError( - format(error + "%1%\nTry ‘%2% --help’ for more information.") - % e.what() % programName); + logError(e.info()); + printError("Try '%1% --help' for more information.", programName); return 1; } catch (BaseError & e) { - printError(format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); + if (settings.showTrace && e.prefix() != "") + printError(e.prefix()); + logError(e.info()); if (e.prefix() != "" && !settings.showTrace) - printError("(use ‘--show-trace’ to show detailed location information)"); + printError("(use '--show-trace' to show detailed location information)"); return e.status; } catch (std::bad_alloc & e) { printError(error + "out of memory"); @@ -296,16 +336,6 @@ RunPager::RunPager() if (!pager) pager = getenv("PAGER"); if (pager && ((string) pager == "" || (string) pager == "cat")) return; - /* Ignore SIGINT. The pager will handle it (and we'll get - SIGPIPE). */ - struct sigaction act; - act.sa_handler = SIG_IGN; - act.sa_flags = 0; - sigemptyset(&act.sa_mask); - if (sigaction(SIGINT, &act, 0)) throw SysError("ignoring SIGINT"); - - restoreSIGPIPE(); - Pipe toPager; toPager.create(); @@ -314,14 +344,17 @@ RunPager::RunPager() throw SysError("dupping stdin"); if (!getenv("LESS")) setenv("LESS", "FRSXMK", 1); + restoreSignals(); if (pager) - execl("/bin/sh", "sh", "-c", pager, NULL); - execlp("pager", "pager", NULL); - execlp("less", "less", NULL); - execlp("more", "more", NULL); - throw SysError(format("executing ‘%1%’") % pager); + execl("/bin/sh", "sh", "-c", pager, nullptr); + execlp("pager", "pager", nullptr); + execlp("less", "less", nullptr); + execlp("more", "more", nullptr); + throw SysError("executing '%1%'", pager); }); + pid.setKillSignal(SIGINT); + if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("dupping stdout"); } @@ -355,5 +388,6 @@ PrintFreed::~PrintFreed() % showBytes(results.bytesFreed); } +Exit::~Exit() { } } diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 6d94a22f788..f558247c0df 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -2,6 +2,8 @@ #include "util.hh" #include "args.hh" +#include "common-args.hh" +#include "path.hh" #include @@ -16,27 +18,36 @@ public: int status; Exit() : status(0) { } Exit(int status) : status(status) { } + virtual ~Exit(); }; int handleExceptions(const string & programName, std::function fun); +/* Don't forget to call initPlugins() after settings are initialized! */ void initNix(); void parseCmdLine(int argc, char * * argv, std::function parseArg); +void parseCmdLine(const string & programName, const Strings & args, + std::function parseArg); + void printVersion(const string & programName); /* Ugh. No better place to put this. */ void printGCWarning(); class Store; +struct StorePathWithOutputs; -void printMissing(ref store, const PathSet & paths); +void printMissing( + ref store, + const std::vector & paths, + Verbosity lvl = lvlInfo); -void printMissing(ref store, const PathSet & willBuild, - const PathSet & willSubstitute, const PathSet & unknown, - unsigned long long downloadSize, unsigned long long narSize); +void printMissing(ref store, const StorePathSet & willBuild, + const StorePathSet & willSubstitute, const StorePathSet & unknown, + unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl = lvlInfo); string getArg(const string & opt, Strings::iterator & i, const Strings::iterator & end); @@ -45,7 +56,7 @@ template N getIntArg(const string & opt, Strings::iterator & i, const Strings::iterator & end, bool allowUnit) { ++i; - if (i == end) throw UsageError(format("‘%1%’ requires an argument") % opt); + if (i == end) throw UsageError("'%1%' requires an argument", opt); string s = *i; N multiplier = 1; if (allowUnit && !s.empty()) { @@ -55,17 +66,30 @@ template N getIntArg(const string & opt, else if (u == 'M') multiplier = 1ULL << 20; else if (u == 'G') multiplier = 1ULL << 30; else if (u == 'T') multiplier = 1ULL << 40; - else throw UsageError(format("invalid unit specifier ‘%1%’") % u); + else throw UsageError("invalid unit specifier '%1%'", u); s.resize(s.size() - 1); } } N n; if (!string2Int(s, n)) - throw UsageError(format("‘%1%’ requires an integer argument") % opt); + throw UsageError("'%1%' requires an integer argument", opt); return n * multiplier; } +struct LegacyArgs : public MixCommonArgs +{ + std::function parseArg; + + LegacyArgs(const std::string & programName, + std::function parseArg); + + bool processFlag(Strings::iterator & pos, Strings::iterator end) override; + + bool processArgs(const Strings & args, bool finish) override; +}; + + /* Show the manual page for the specified program. */ void showManPage(const string & name); diff --git a/src/libmain/stack.cc b/src/libmain/stack.cc index abf59dc4baa..b0a4a4c5dbe 100644 --- a/src/libmain/stack.cc +++ b/src/libmain/stack.cc @@ -1,6 +1,4 @@ -#include "config.h" - -#include "types.hh" +#include "error.hh" #include #include @@ -20,9 +18,9 @@ static void sigsegvHandler(int signo, siginfo_t * info, void * ctx) bool haveSP = true; char * sp = 0; #if defined(__x86_64__) && defined(REG_RSP) - sp = (char *) ((ucontext *) ctx)->uc_mcontext.gregs[REG_RSP]; + sp = (char *) ((ucontext_t *) ctx)->uc_mcontext.gregs[REG_RSP]; #elif defined(REG_ESP) - sp = (char *) ((ucontext *) ctx)->uc_mcontext.gregs[REG_ESP]; + sp = (char *) ((ucontext_t *) ctx)->uc_mcontext.gregs[REG_ESP]; #else haveSP = false; #endif @@ -32,7 +30,7 @@ static void sigsegvHandler(int signo, siginfo_t * info, void * ctx) if (diff < 0) diff = -diff; if (diff < 4096) { char msg[] = "error: stack overflow (possible infinite recursion)\n"; - [[gnu::unused]] int res = write(2, msg, strlen(msg)); + [[gnu::unused]] auto res = write(2, msg, strlen(msg)); _exit(1); // maybe abort instead? } } @@ -54,7 +52,8 @@ void detectStackOverflow() delivered when we're out of stack space. */ stack_t stack; stack.ss_size = 4096 * 4 + MINSIGSTKSZ; - stack.ss_sp = new char[stack.ss_size]; + static auto stackBuf = std::make_unique>(stack.ss_size); + stack.ss_sp = stackBuf->data(); if (!stack.ss_sp) throw Error("cannot allocate alternative stack"); stack.ss_flags = 0; if (sigaltstack(&stack, 0) == -1) throw SysError("cannot set alternative stack"); @@ -64,7 +63,7 @@ void detectStackOverflow() act.sa_sigaction = sigsegvHandler; act.sa_flags = SA_SIGINFO | SA_ONSTACK; if (sigaction(SIGSEGV, &act, 0)) - throw SysError("resetting SIGCHLD"); + throw SysError("resetting SIGSEGV"); #endif } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 3e07a2aa2b6..64933149560 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -10,79 +10,19 @@ #include "nar-info-disk-cache.hh" #include "nar-accessor.hh" #include "json.hh" +#include "thread-pool.hh" #include - #include +#include -namespace nix { - -/* Given requests for a path /nix/store//, this accessor will - first download the NAR for /nix/store/ from the binary cache, - build a NAR accessor for that NAR, and use that to access . */ -struct BinaryCacheStoreAccessor : public FSAccessor -{ - ref store; - - std::map> nars; - - BinaryCacheStoreAccessor(ref store) - : store(store) - { - } - - std::pair, Path> fetch(const Path & path_) - { - auto path = canonPath(path_); - - auto storePath = store->toStorePath(path); - std::string restPath = std::string(path, storePath.size()); - - if (!store->isValidPath(storePath)) - throw InvalidPath(format("path ‘%1%’ is not a valid store path") % storePath); +#include - auto i = nars.find(storePath); - if (i != nars.end()) return {i->second, restPath}; - - StringSink sink; - store->narFromPath(storePath, sink); - - auto accessor = makeNarAccessor(sink.s); - nars.emplace(storePath, accessor); - return {accessor, restPath}; - } - - Stat stat(const Path & path) override - { - auto res = fetch(path); - return res.first->stat(res.second); - } - - StringSet readDirectory(const Path & path) override - { - auto res = fetch(path); - return res.first->readDirectory(res.second); - } - - std::string readFile(const Path & path) override - { - auto res = fetch(path); - return res.first->readFile(res.second); - } - - std::string readLink(const Path & path) override - { - auto res = fetch(path); - return res.first->readLink(res.second); - } -}; +namespace nix { BinaryCacheStore::BinaryCacheStore(const Params & params) : Store(params) - , compression(get(params, "compression", "xz")) - , writeNARListing(get(params, "write-nar-listing", "0") == "1") { - auto secretKeyFile = get(params, "secret-key", ""); if (secretKeyFile != "") secretKey = std::unique_ptr(new SecretKey(readFile(secretKeyFile))); @@ -97,53 +37,88 @@ void BinaryCacheStore::init() auto cacheInfo = getFile(cacheInfoFile); if (!cacheInfo) { - upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n"); + upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info"); } else { for (auto & line : tokenizeString(*cacheInfo, "\n")) { - size_t colon = line.find(':'); - if (colon == std::string::npos) continue; + size_t colon= line.find(':'); + if (colon ==std::string::npos) continue; auto name = line.substr(0, colon); auto value = trim(line.substr(colon + 1, std::string::npos)); if (name == "StoreDir") { if (value != storeDir) - throw Error(format("binary cache ‘%s’ is for Nix stores with prefix ‘%s’, not ‘%s’") - % getUri() % value % storeDir); + throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'", + getUri(), value, storeDir); } else if (name == "WantMassQuery") { - wantMassQuery_ = value == "1"; + wantMassQuery.setDefault(value == "1" ? "true" : "false"); } else if (name == "Priority") { - string2Int(value, priority); + priority.setDefault(fmt("%d", std::stoi(value))); } } } } -void BinaryCacheStore::notImpl() +void BinaryCacheStore::getFile(const std::string & path, + Callback> callback) noexcept { - throw Error("operation not implemented for binary cache stores"); + try { + callback(getFile(path)); + } catch (...) { callback.rethrow(); } } -std::shared_ptr BinaryCacheStore::getFile(const std::string & path) +void BinaryCacheStore::getFile(const std::string & path, Sink & sink) { std::promise> promise; getFile(path, - [&](std::shared_ptr result) { - promise.set_value(result); - }, - [&](std::exception_ptr exc) { - promise.set_exception(exc); - }); - return promise.get_future().get(); + {[&](std::future> result) { + try { + promise.set_value(result.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }}); + auto data = promise.get_future().get(); + sink((unsigned char *) data->data(), data->size()); +} + +std::shared_ptr BinaryCacheStore::getFile(const std::string & path) +{ + StringSink sink; + try { + getFile(path, sink); + } catch (NoSuchBinaryCacheFile &) { + return nullptr; + } + return sink.s; } -Path BinaryCacheStore::narInfoFileFor(const Path & storePath) +std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath) { - assertStorePath(storePath); - return storePathToHash(storePath) + ".narinfo"; + return storePathToHash(printStorePath(storePath)) + ".narinfo"; +} + +void BinaryCacheStore::writeNarInfo(ref narInfo) +{ + auto narInfoFile = narInfoFileFor(narInfo->path); + + upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo"); + + auto hashPart = storePathToHash(printStorePath(narInfo->path)); + + { + auto state_(state.lock()); + state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = std::shared_ptr(narInfo) }); + } + + if (diskCache) + diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr(narInfo)); } -void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref & nar, - bool repair, bool dontCheckSigs, std::shared_ptr accessor) +void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource, + RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr accessor) { + // FIXME: See if we can use the original source to reduce memory usage. + auto nar = make_ref(narSource.drain()); + if (!repair && isValidPath(info.path)) return; /* Verify that all references are valid. This may do some .narinfo @@ -153,12 +128,10 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const refcompare(0, narMagic.size(), narMagic) == 0); auto narInfo = make_ref(info); @@ -167,9 +140,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const refnarHash = hashString(htSHA256, *nar); if (info.narHash && info.narHash != narInfo->narHash) - throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path); + throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path)); + + auto accessor_ = std::dynamic_pointer_cast(accessor); - auto accessor_ = std::dynamic_pointer_cast(accessor); + auto narAccessor = makeNarAccessor(nar); + + if (accessor_) + accessor_->addToCache(printStorePath(info.path), *nar, narAccessor); /* Optionally write a JSON file containing a listing of the contents of the NAR. */ @@ -180,80 +158,96 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const refnars.emplace(info.path, narAccessor); - - std::function recurse; - - recurse = [&](const Path & path, JSONPlaceholder & res) { - auto st = narAccessor->stat(path); - - auto obj = res.object(); - - switch (st.type) { - case FSAccessor::Type::tRegular: - obj.attr("type", "regular"); - obj.attr("size", st.fileSize); - if (st.isExecutable) - obj.attr("executable", true); - break; - case FSAccessor::Type::tDirectory: - obj.attr("type", "directory"); - { - auto res2 = obj.object("entries"); - for (auto & name : narAccessor->readDirectory(path)) { - auto res3 = res2.placeholder(name); - recurse(path + "/" + name, res3); - } - } - break; - case FSAccessor::Type::tSymlink: - obj.attr("type", "symlink"); - obj.attr("target", narAccessor->readLink(path)); - break; - default: - abort(); - } - }; - { auto res = jsonRoot.placeholder("root"); - recurse("", res); + listNar(res, narAccessor, "", true); } } - upsertFile(storePathToHash(info.path) + ".ls.xz", *compress("xz", jsonOut.str())); - } - - else { - if (accessor_) - accessor_->nars.emplace(info.path, makeNarAccessor(nar)); + upsertFile(storePathToHash(printStorePath(info.path)) + ".ls", jsonOut.str(), "application/json"); } /* Compress the NAR. */ narInfo->compression = compression; auto now1 = std::chrono::steady_clock::now(); - auto narCompressed = compress(compression, *nar); + auto narCompressed = compress(compression, *nar, parallelCompression); auto now2 = std::chrono::steady_clock::now(); narInfo->fileHash = hashString(htSHA256, *narCompressed); narInfo->fileSize = narCompressed->size(); auto duration = std::chrono::duration_cast(now2 - now1).count(); - printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache") - % narInfo->path % narInfo->narSize - % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0) - % duration); + printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", + printStorePath(narInfo->path), narInfo->narSize, + ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0), + duration); - /* Atomically write the NAR file. */ - narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar" + narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : + compression == "br" ? ".br" : ""); + + /* Optionally maintain an index of DWARF debug info files + consisting of JSON files named 'debuginfo/' that + specify the NAR file and member containing the debug info. */ + if (writeDebugInfo) { + + std::string buildIdDir = "/lib/debug/.build-id"; + + if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) { + + ThreadPool threadPool(25); + + auto doFile = [&](std::string member, std::string key, std::string target) { + checkInterrupt(); + + nlohmann::json json; + json["archive"] = target; + json["member"] = member; + + // FIXME: or should we overwrite? The previous link may point + // to a GC'ed file, so overwriting might be useful... + if (fileExists(key)) return; + + printMsg(lvlTalkative, "creating debuginfo link from '%s' to '%s'", key, target); + + upsertFile(key, json.dump(), "application/json"); + }; + + std::regex regex1("^[0-9a-f]{2}$"); + std::regex regex2("^[0-9a-f]{38}\\.debug$"); + + for (auto & s1 : narAccessor->readDirectory(buildIdDir)) { + auto dir = buildIdDir + "/" + s1; + + if (narAccessor->stat(dir).type != FSAccessor::tDirectory + || !std::regex_match(s1, regex1)) + continue; + + for (auto & s2 : narAccessor->readDirectory(dir)) { + auto debugPath = dir + "/" + s2; + + if (narAccessor->stat(debugPath).type != FSAccessor::tRegular + || !std::regex_match(s2, regex2)) + continue; + + auto buildId = s1 + s2; + + std::string key = "debuginfo/" + buildId; + std::string target = "../" + narInfo->url; + + threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target)); + } + } + + threadPool.process(); + } + } + + /* Atomically write the NAR file. */ if (repair || !fileExists(narInfo->url)) { stats.narWrite++; - upsertFile(narInfo->url, *narCompressed); + upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar"); } else stats.narWriteAverted++; @@ -262,24 +256,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const refsign(*secretKey); - - upsertFile(narInfoFile, narInfo->to_string()); - - auto hashPart = storePathToHash(narInfo->path); - - { - auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, std::shared_ptr(narInfo)); - } + if (secretKey) narInfo->sign(*this, *secretKey); - if (diskCache) - diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr(narInfo)); + writeNarInfo(narInfo); stats.narInfoWrite++; } -bool BinaryCacheStore::isValidPathUncached(const Path & storePath) +bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath) { // FIXME: this only checks whether a .narinfo with a matching hash // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even @@ -287,55 +271,66 @@ bool BinaryCacheStore::isValidPathUncached(const Path & storePath) return fileExists(narInfoFileFor(storePath)); } -void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) +void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) { auto info = queryPathInfo(storePath).cast(); - auto nar = getFile(info->url); + uint64_t narSize = 0; - if (!nar) throw Error(format("file ‘%s’ missing from binary cache") % info->url); + LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { + sink(data, len); + narSize += len; + }); - stats.narRead++; - stats.narReadCompressedBytes += nar->size(); + auto decompressor = makeDecompressionSink(info->compression, wrapperSink); - /* Decompress the NAR. FIXME: would be nice to have the remote - side do this. */ try { - nar = decompress(info->compression, *nar); - } catch (UnknownCompressionMethod &) { - throw Error(format("binary cache path ‘%s’ uses unknown compression method ‘%s’") - % storePath % info->compression); + getFile(info->url, *decompressor); + } catch (NoSuchBinaryCacheFile & e) { + throw SubstituteGone(e.info()); } - stats.narReadBytes += nar->size(); + decompressor->finish(); - printMsg(lvlTalkative, format("exporting path ‘%1%’ (%2% bytes)") % storePath % nar->size()); - - assert(nar->size() % 8 == 0); - - sink((unsigned char *) nar->c_str(), nar->size()); + stats.narRead++; + //stats.narReadCompressedBytes += nar->size(); // FIXME + stats.narReadBytes += narSize; } -void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, - std::function)> success, - std::function failure) +void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, + Callback> callback) noexcept { + auto uri = getUri(); + auto storePathS = printStorePath(storePath); + auto act = std::make_shared(*logger, lvlTalkative, actQueryPathInfo, + fmt("querying info about '%s' on '%s'", storePathS, uri), Logger::Fields{storePathS, uri}); + PushActivity pact(act->id); + auto narInfoFile = narInfoFileFor(storePath); + auto callbackPtr = std::make_shared(std::move(callback)); + getFile(narInfoFile, - [=](std::shared_ptr data) { - if (!data) return success(0); + {[=](std::future> fut) { + try { + auto data = fut.get(); + + if (!data) return (*callbackPtr)(nullptr); - stats.narInfoRead++; + stats.narInfoRead++; - callSuccess(success, failure, (std::shared_ptr) - std::make_shared(*this, *data, narInfoFile)); - }, - failure); + (*callbackPtr)((std::shared_ptr) + std::make_shared(*this, *data, narInfoFile)); + + (void) act; // force Activity into this lambda to ensure it stays alive + } catch (...) { + callbackPtr->rethrow(); + } + }}); } -Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) +StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath, + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { // FIXME: some cut&paste from LocalStore::addToStore(). @@ -344,7 +339,7 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, small files. */ StringSink sink; Hash h; - if (recursive) { + if (method == FileIngestionMethod::Recursive) { dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); } else { @@ -353,33 +348,71 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, h = hashString(hashAlgo, s); } - ValidPathInfo info; - info.path = makeFixedOutputPath(recursive, h, name); + ValidPathInfo info(makeFixedOutputPath(method, h, name)); - addToStore(info, sink.s, repair, false, 0); + auto source = StringSource { *sink.s }; + addToStore(info, source, repair, CheckSigs, nullptr); - return info.path; + return std::move(info.path); } -Path BinaryCacheStore::addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair) +StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) { - ValidPathInfo info; - info.path = computeStorePathForText(name, s, references); - info.references = references; + ValidPathInfo info(computeStorePathForText(name, s, references)); + info.references = cloneStorePathSet(references); if (repair || !isValidPath(info.path)) { StringSink sink; dumpString(s, sink); - addToStore(info, sink.s, repair, false, 0); + auto source = StringSource { *sink.s }; + addToStore(info, source, repair, CheckSigs, nullptr); } - return info.path; + return std::move(info.path); } ref BinaryCacheStore::getFSAccessor() { - return make_ref(ref(shared_from_this())); + return make_ref(ref(shared_from_this()), localNarCache); +} + +void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs) +{ + /* Note: this is inherently racy since there is no locking on + binary caches. In particular, with S3 this unreliable, even + when addSignatures() is called sequentially on a path, because + S3 might return an outdated cached version. */ + + auto narInfo = make_ref((NarInfo &) *queryPathInfo(storePath)); + + narInfo->sigs.insert(sigs.begin(), sigs.end()); + + auto narInfoFile = narInfoFileFor(narInfo->path); + + writeNarInfo(narInfo); +} + +std::shared_ptr BinaryCacheStore::getBuildLog(const StorePath & path) +{ + auto drvPath = path.clone(); + + if (!path.isDerivation()) { + try { + auto info = queryPathInfo(path); + // FIXME: add a "Log" field to .narinfo + if (!info->deriver) return nullptr; + drvPath = info->deriver->clone(); + } catch (InvalidPath &) { + return nullptr; + } + } + + auto logPath = "log/" + std::string(baseNameOf(printStorePath(drvPath))); + + debug("fetching build log from binary cache '%s/%s'", getUri(), logPath); + + return getFile(logPath); } } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 31878bbb247..52ef8aa7ac2 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -13,38 +13,44 @@ struct NarInfo; class BinaryCacheStore : public Store { -private: +public: - std::unique_ptr secretKey; + const Setting compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; + const Setting writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; + const Setting writeDebugInfo{this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"}; + const Setting secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; + const Setting localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"}; + const Setting parallelCompression{this, false, "parallel-compression", + "enable multi-threading compression, available for xz only currently"}; - std::string compression; +private: - bool writeNARListing; + std::unique_ptr secretKey; protected: BinaryCacheStore(const Params & params); - [[noreturn]] void notImpl(); - public: virtual bool fileExists(const std::string & path) = 0; - virtual void upsertFile(const std::string & path, const std::string & data) = 0; + virtual void upsertFile(const std::string & path, + const std::string & data, + const std::string & mimeType) = 0; - /* Return the contents of the specified file, or null if it - doesn't exist. */ - virtual void getFile(const std::string & path, - std::function)> success, - std::function failure) = 0; + /* Note: subclasses must implement at least one of the two + following getFile() methods. */ - std::shared_ptr getFile(const std::string & path); + /* Dump the contents of the specified file to a sink. */ + virtual void getFile(const std::string & path, Sink & sink); -protected: + /* Fetch the specified file and call the specified callback with + the result. A subclass may implement this asynchronously. */ + virtual void getFile(const std::string & path, + Callback> callback) noexcept; - bool wantMassQuery_ = false; - int priority = 50; + std::shared_ptr getFile(const std::string & path); public: @@ -54,95 +60,48 @@ private: std::string narMagic; - std::string narInfoFileFor(const Path & storePath); - -public: - - bool isValidPathUncached(const Path & path) override; - - PathSet queryAllValidPaths() override - { notImpl(); } + std::string narInfoFileFor(const StorePath & storePath); - void queryPathInfoUncached(const Path & path, - std::function)> success, - std::function failure) override; + void writeNarInfo(ref narInfo); - void queryReferrers(const Path & path, - PathSet & referrers) override - { notImpl(); } - - PathSet queryValidDerivers(const Path & path) override - { return {}; } - - PathSet queryDerivationOutputs(const Path & path) override - { notImpl(); } - - StringSet queryDerivationOutputNames(const Path & path) override - { notImpl(); } - - Path queryPathFromHashPart(const string & hashPart) override - { notImpl(); } +public: - PathSet querySubstitutablePaths(const PathSet & paths) override - { return {}; } + bool isValidPathUncached(const StorePath & path) override; - void querySubstitutablePathInfos(const PathSet & paths, - SubstitutablePathInfos & infos) override - { } + void queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept override; - bool wantMassQuery() override { return wantMassQuery_; } + std::optional queryPathFromHashPart(const std::string & hashPart) override + { unsupported("queryPathFromHashPart"); } - void addToStore(const ValidPathInfo & info, const ref & nar, - bool repair, bool dontCheckSigs, + void addToStore(const ValidPathInfo & info, Source & narSource, + RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr accessor) override; - Path addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, - PathFilter & filter, bool repair) override; - - Path addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair) override; + StorePath addToStore(const string & name, const Path & srcPath, + FileIngestionMethod method, HashType hashAlgo, + PathFilter & filter, RepairFlag repair) override; - void narFromPath(const Path & path, Sink & sink) override; + StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) override; - void buildPaths(const PathSet & paths, BuildMode buildMode) override - { notImpl(); } + void narFromPath(const StorePath & path, Sink & sink) override; - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override - { notImpl(); } - - void ensurePath(const Path & path) override - { notImpl(); } - - void addTempRoot(const Path & path) override - { notImpl(); } + { unsupported("buildDerivation"); } - void addIndirectRoot(const Path & path) override - { notImpl(); } - - void syncWithGC() override - { } - - Roots findRoots() override - { notImpl(); } - - void collectGarbage(const GCOptions & options, GCResults & results) override - { notImpl(); } - - void optimiseStore() override - { } - - bool verifyStore(bool checkContents, bool repair) override - { return true; } + void ensurePath(const StorePath & path) override + { unsupported("ensurePath"); } ref getFSAccessor() override; -public: + void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - void addSignatures(const Path & storePath, const StringSet & sigs) override - { notImpl(); } + std::shared_ptr getBuildLog(const StorePath & path) override; }; +MakeError(NoSuchBinaryCacheFile, Error); + } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 7fc6ff0df0f..8ad2ca4f32c 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1,5 +1,3 @@ -#include "config.h" - #include "references.hh" #include "pathlocks.hh" #include "globals.hh" @@ -8,8 +6,16 @@ #include "archive.hh" #include "affinity.hh" #include "builtins.hh" +#include "builtins/buildenv.hh" +#include "filetransfer.hh" #include "finally.hh" #include "compression.hh" +#include "json.hh" +#include "nar-info.hh" +#include "parsed-derivations.hh" +#include "machines.hh" +#include "daemon.hh" +#include "worker-protocol.hh" #include #include @@ -18,30 +24,29 @@ #include #include #include +#include +#include +#include -#include #include #include #include #include #include -#include #include +#include +#include #include +#include #include #include #include +#include +#include #include #include -/* chroot-like behavior from Apple's sandbox */ -#if __APPLE__ - #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh" -#else - #define DEFAULT_ALLOWED_IMPURE_PREFIXES "" -#endif - /* Includes required for chroot support. */ #if __linux__ #include @@ -54,6 +59,9 @@ #include #include #include +#if HAVE_SECCOMP +#include +#endif #define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) #endif @@ -61,6 +69,8 @@ #include #endif +#include + namespace nix { @@ -82,7 +92,7 @@ typedef std::shared_ptr GoalPtr; typedef std::weak_ptr WeakGoalPtr; struct CompareGoalPtrs { - bool operator() (const GoalPtr & a, const GoalPtr & b); + bool operator() (const GoalPtr & a, const GoalPtr & b) const; }; /* Set of goals. */ @@ -90,17 +100,14 @@ typedef set Goals; typedef list WeakGoals; /* A map of paths to goals (and the other way around). */ -typedef map WeakGoalMap; +typedef std::map WeakGoalMap; -class Goal : public std::enable_shared_from_this +struct Goal : public std::enable_shared_from_this { -public: typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; -protected: - /* Backlink to the worker. */ Worker & worker; @@ -128,6 +135,9 @@ class Goal : public std::enable_shared_from_this /* Whether the goal is finished. */ ExitCode exitCode; + /* Exception containing an error message, if any. */ + std::optional ex; + Goal(Worker & worker) : worker(worker) { nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; @@ -139,7 +149,6 @@ class Goal : public std::enable_shared_from_this trace("goal destroyed"); } -public: virtual void work() = 0; void addWaitee(GoalPtr waitee); @@ -156,31 +165,25 @@ class Goal : public std::enable_shared_from_this abort(); } - void trace(const format & f); + void trace(const FormatOrString & fs); string getName() { return name; } - ExitCode getExitCode() - { - return exitCode; - } - /* Callback in case of a timeout. It should wake up its waiters, get rid of any running child processes that are being monitored by the worker (important!), etc. */ - virtual void timedOut() = 0; + virtual void timedOut(Error && ex) = 0; virtual string key() = 0; -protected: - void amDone(ExitCode result); + void amDone(ExitCode result, std::optional ex = {}); }; -bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) { +bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { string s1 = a->key(); string s2 = b->key(); return s1 < s2; @@ -244,10 +247,14 @@ class Worker steady_time_point lastWokenUp; /* Cache for pathContentsGood(). */ - std::map pathContentsGoodCache; + std::map pathContentsGoodCache; public: + const Activity act; + const Activity actDerivations; + const Activity actSubstitutions; + /* Set if at least one derivation had a BuildError (i.e. permanent failure). */ bool permanentFailure; @@ -255,18 +262,42 @@ class Worker /* Set if at least one derivation had a timeout. */ bool timedOut; + /* Set if at least one derivation fails with a hash mismatch. */ + bool hashMismatch; + + /* Set if at least one derivation is not deterministic in check mode. */ + bool checkMismatch; + LocalStore & store; std::unique_ptr hook; + uint64_t expectedBuilds = 0; + uint64_t doneBuilds = 0; + uint64_t failedBuilds = 0; + uint64_t runningBuilds = 0; + + uint64_t expectedSubstitutions = 0; + uint64_t doneSubstitutions = 0; + uint64_t failedSubstitutions = 0; + uint64_t runningSubstitutions = 0; + uint64_t expectedDownloadSize = 0; + uint64_t doneDownloadSize = 0; + uint64_t expectedNarSize = 0; + uint64_t doneNarSize = 0; + + /* Whether to ask the build hook if it can build a derivation. If + it answers with "decline-permanently", we don't try again. */ + bool tryBuildHook = true; + Worker(LocalStore & store); ~Worker(); /* Make a goal (with caching). */ - GoalPtr makeDerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - std::shared_ptr makeBasicDerivationGoal(const Path & drvPath, + GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); + std::shared_ptr makeBasicDerivationGoal(StorePath && drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - GoalPtr makeSubstitutionGoal(const Path & storePath, bool repair = false); + GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); @@ -314,9 +345,17 @@ class Worker /* Check whether the given valid path exists and has the right contents. */ - bool pathContentsGood(const Path & path); + bool pathContentsGood(const StorePath & path); + + void markContentsGood(StorePath && path); - void markContentsGood(const Path & path); + void updateProgress() + { + actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); + actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); + act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); + act.setExpected(actCopyPath, expectedNarSize + doneNarSize); + } }; @@ -345,8 +384,7 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result) assert(waitees.find(waitee) != waitees.end()); waitees.erase(waitee); - trace(format("waitee ‘%1%’ done; %2% left") % - waitee->name % waitees.size()); + trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size())); if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed; @@ -371,12 +409,20 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result) } -void Goal::amDone(ExitCode result) +void Goal::amDone(ExitCode result, std::optional ex) { trace("done"); assert(exitCode == ecBusy); assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); exitCode = result; + + if (ex) { + if (!waiters.empty()) + logError(ex->info()); + else + this->ex = std::move(*ex); + } + for (auto & i : waiters) { GoalPtr goal = i.lock(); if (goal) goal->waiteeDone(shared_from_this(), result); @@ -386,9 +432,9 @@ void Goal::amDone(ExitCode result) } -void Goal::trace(const format & f) +void Goal::trace(const FormatOrString & fs) { - debug(format("%1%: %2%") % name % f); + debug("%1%: %2%", name, fs.s); } @@ -399,12 +445,14 @@ void Goal::trace(const format & f) /* Common initialisation performed in child processes. */ static void commonChildInit(Pipe & logPipe) { + restoreSignals(); + /* Put the child in a separate session (and thus a separate process group) so that it has no controlling terminal (meaning that e.g. ssh cannot open /dev/tty) and it doesn't receive terminal signals. */ if (setsid() == -1) - throw SysError(format("creating a new session")); + throw SysError("creating a new session"); /* Dup the write side of the logger pipe into stderr. */ if (dup2(logPipe.writeSide.get(), STDERR_FILENO) == -1) @@ -417,12 +465,43 @@ static void commonChildInit(Pipe & logPipe) /* Reroute stdin to /dev/null. */ int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); if (fdDevNull == -1) - throw SysError(format("cannot open ‘%1%’") % pathNullDevice); + throw SysError("cannot open '%1%'", pathNullDevice); if (dup2(fdDevNull, STDIN_FILENO) == -1) throw SysError("cannot dup null device into stdin"); close(fdDevNull); } +void handleDiffHook( + uid_t uid, uid_t gid, + const Path & tryA, const Path & tryB, + const Path & drvPath, const Path & tmpDir) +{ + auto diffHook = settings.diffHook; + if (diffHook != "" && settings.runDiffHook) { + try { + RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir}); + diffHookOptions.searchPath = true; + diffHookOptions.uid = uid; + diffHookOptions.gid = gid; + diffHookOptions.chdir = "/"; + + auto diffRes = runProgram(diffHookOptions); + if (!statusOk(diffRes.first)) + throw ExecError(diffRes.first, + "diff-hook program '%1%' %2%", + diffHook, + statusToString(diffRes.first)); + + if (diffRes.second != "") + printError(chomp(diffRes.second)); + } catch (Error & error) { + ErrorInfo ei = error.info(); + ei.hint = hintfmt("diff hook execution failed: %s", + (error.info().hint.has_value() ? error.info().hint->str() : "")); + logError(ei); + } + } +} ////////////////////////////////////////////////////////////////////// @@ -430,25 +509,17 @@ static void commonChildInit(Pipe & logPipe) class UserLock { private: - /* POSIX locks suck. If we have a lock on a file, and we open and - close that file again (without closing the original file - descriptor), we lose the lock. So we have to be *very* careful - not to open a lock file on which we are holding a lock. */ - static PathSet lockedPaths; /* !!! not thread-safe */ - Path fnUserLock; AutoCloseFD fdUserLock; + bool isEnabled = false; string user; uid_t uid = 0; gid_t gid = 0; std::vector supplementaryGIDs; public: - ~UserLock(); - - void acquire(); - void release(); + UserLock(); void kill(); @@ -457,76 +528,66 @@ class UserLock uid_t getGID() { assert(gid); return gid; } std::vector getSupplementaryGIDs() { return supplementaryGIDs; } - bool enabled() { return uid != 0; } - -}; + bool findFreeUser(); + bool enabled() { return isEnabled; } -PathSet UserLock::lockedPaths; +}; -UserLock::~UserLock() +UserLock::UserLock() { - release(); + assert(settings.buildUsersGroup != ""); + createDirs(settings.nixStateDir + "/userpool"); } - -void UserLock::acquire() -{ - assert(uid == 0); - - assert(settings.buildUsersGroup != ""); +bool UserLock::findFreeUser() { + if (enabled()) return true; /* Get the members of the build-users-group. */ - struct group * gr = getgrnam(settings.buildUsersGroup.c_str()); + struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) - throw Error(format("the group ‘%1%’ specified in ‘build-users-group’ does not exist") - % settings.buildUsersGroup); + throw Error("the group '%1%' specified in 'build-users-group' does not exist", + settings.buildUsersGroup); gid = gr->gr_gid; /* Copy the result of getgrnam. */ Strings users; for (char * * p = gr->gr_mem; *p; ++p) { - debug(format("found build user ‘%1%’") % *p); + debug("found build user '%1%'", *p); users.push_back(*p); } if (users.empty()) - throw Error(format("the build users group ‘%1%’ has no members") - % settings.buildUsersGroup); + throw Error("the build users group '%1%' has no members", + settings.buildUsersGroup); /* Find a user account that isn't currently in use for another build. */ for (auto & i : users) { - debug(format("trying user ‘%1%’") % i); + debug("trying user '%1%'", i); struct passwd * pw = getpwnam(i.c_str()); if (!pw) - throw Error(format("the user ‘%1%’ in the group ‘%2%’ does not exist") - % i % settings.buildUsersGroup); + throw Error("the user '%1%' in the group '%2%' does not exist", + i, settings.buildUsersGroup); - createDirs(settings.nixStateDir + "/userpool"); fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); - if (lockedPaths.find(fnUserLock) != lockedPaths.end()) - /* We already have a lock on this one. */ - continue; - AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); if (!fd) - throw SysError(format("opening user lock ‘%1%’") % fnUserLock); + throw SysError("opening user lock '%1%'", fnUserLock); if (lockFile(fd.get(), ltWrite, false)) { fdUserLock = std::move(fd); - lockedPaths.insert(fnUserLock); user = i; uid = pw->pw_uid; /* Sanity check... */ if (uid == getuid() || uid == geteuid()) - throw Error(format("the Nix user should not be a member of ‘%1%’") - % settings.buildUsersGroup); + throw Error("the Nix user should not be a member of '%1%'", + settings.buildUsersGroup); #if __linux__ /* Get the list of supplementary groups of this build user. This @@ -536,35 +597,21 @@ void UserLock::acquire() int err = getgrouplist(pw->pw_name, pw->pw_gid, supplementaryGIDs.data(), &ngroups); if (err == -1) - throw Error(format("failed to get list of supplementary groups for ‘%1%’") % pw->pw_name); + throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name); supplementaryGIDs.resize(ngroups); #endif - return; + isEnabled = true; + return true; } } - throw Error(format("all build users are currently in use; " - "consider creating additional users and adding them to the ‘%1%’ group") - % settings.buildUsersGroup); -} - - -void UserLock::release() -{ - if (uid == 0) return; - fdUserLock = -1; /* releases lock */ - assert(lockedPaths.find(fnUserLock) != lockedPaths.end()); - lockedPaths.erase(fnUserLock); - fnUserLock = ""; - uid = 0; + return false; } - void UserLock::kill() { - assert(enabled()); killUser(uid); } @@ -586,6 +633,10 @@ struct HookInstance /* The process ID of the hook. */ Pid pid; + FdSink sink; + + std::map activities; + HookInstance(); ~HookInstance(); @@ -594,11 +645,7 @@ struct HookInstance HookInstance::HookInstance() { - debug("starting build hook"); - - Path buildHook = getEnv("NIX_BUILD_HOOK"); - if (string(buildHook, 0, 1) != "/") buildHook = settings.nixLibexecDir + "/nix/" + buildHook; - buildHook = canonPath(buildHook); + debug("starting build hook '%s'", settings.buildHook); /* Create a pipe to get the output of the child. */ fromHook.create(); @@ -624,21 +671,31 @@ HookInstance::HookInstance() if (dup2(builderOut.writeSide.get(), 4) == -1) throw SysError("dupping builder's stdout/stderr"); + /* Hack: pass the read side of that fd to allow build-remote + to read SSH error messages. */ + if (dup2(builderOut.readSide.get(), 5) == -1) + throw SysError("dupping builder's stdout/stderr"); + Strings args = { - baseNameOf(buildHook), - settings.thisSystem, - (format("%1%") % settings.maxSilentTime).str(), - (format("%1%") % settings.buildTimeout).str() + std::string(baseNameOf(settings.buildHook.get())), + std::to_string(verbosity), }; - execv(buildHook.c_str(), stringsToCharPtrs(args).data()); + execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); - throw SysError(format("executing ‘%1%’") % buildHook); + throw SysError("executing '%s'", settings.buildHook); }); pid.setSeparatePG(true); fromHook.writeSide = -1; toHook.readSide = -1; + + sink = FdSink(toHook.writeSide.get()); + std::map settings; + globalConfig.getSettings(settings); + for (auto & setting : settings) + sink << 1 << setting.first << setting.second.value; + sink << 0; } @@ -646,7 +703,7 @@ HookInstance::~HookInstance() { try { toHook.writeSide = -1; - if (pid != -1) pid.kill(true); + if (pid != -1) pid.kill(); } catch (...) { ignoreException(); } @@ -656,23 +713,6 @@ HookInstance::~HookInstance() ////////////////////////////////////////////////////////////////////// -typedef map StringRewrites; - - -std::string rewriteStrings(std::string s, const StringRewrites & rewrites) -{ - for (auto & i : rewrites) { - size_t j = 0; - while ((j = s.find(i.first, j)) != string::npos) - s.replace(j, i.first.size(), i.second); - } - return s; -} - - -////////////////////////////////////////////////////////////////////// - - typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; class SubstitutionGoal; @@ -684,7 +724,7 @@ class DerivationGoal : public Goal bool useDerivation; /* The path of the derivation. */ - Path drvPath; + StorePath drvPath; /* The specific outputs that we need to build. Empty means all of them. */ @@ -695,11 +735,13 @@ class DerivationGoal : public Goal /* Whether to retry substituting the outputs after building the inputs. */ - bool retrySubstitution = false; + bool retrySubstitution; /* The derivation stored at drvPath. */ std::unique_ptr drv; + std::unique_ptr parsedDrv; + /* The remainder is state held during the build. */ /* Locks on the output paths. */ @@ -707,20 +749,17 @@ class DerivationGoal : public Goal /* All input paths (that is, the union of FS closures of the immediate input paths). */ - PathSet inputPaths; - - /* Referenceable paths (i.e., input and output paths). */ - PathSet allPaths; + StorePathSet inputPaths; /* Outputs that are already valid. If we're repairing, these are the outputs that are valid *and* not corrupt. */ - PathSet validPaths; + StorePathSet validPaths; /* Outputs that are corrupt or not valid. */ - PathSet missingPaths; + StorePathSet missingPaths; /* User selected for running the builder. */ - UserLock buildUser; + std::unique_ptr buildUser; /* The process ID of the builder. */ Pid pid; @@ -744,12 +783,18 @@ class DerivationGoal : public Goal std::string currentLogLine; size_t currentLogLinePos = 0; // to handle carriage return + std::string currentHookLine; + /* Pipe for the builder's standard output/error. */ Pipe builderOut; - /* Pipe for synchronising updates to the builder user namespace. */ + /* Pipe for synchronising updates to the builder namespaces. */ Pipe userNamespaceSync; + /* The mount namespace of the builder, used to add additional + paths to the sandbox as a result of recursive Nix calls. */ + AutoCloseFD sandboxMountNamespace; + /* The build hook. */ std::unique_ptr hook; @@ -780,18 +825,18 @@ class DerivationGoal : public Goal }; typedef map DirsInChroot; // maps target path to source path DirsInChroot dirsInChroot; + typedef map Environment; Environment env; #if __APPLE__ typedef string SandboxProfile; SandboxProfile additionalSandboxProfile; - AutoDelete autoDelSandbox; #endif /* Hash rewriting. */ - StringRewrites inputRewrites, outputRewrites; - typedef map RedirectedOutputs; + StringMap inputRewrites, outputRewrites; + typedef map RedirectedOutputs; RedirectedOutputs redirectedOutputs; BuildMode buildMode; @@ -799,32 +844,72 @@ class DerivationGoal : public Goal /* If we're repairing without a chroot, there may be outputs that are valid but corrupt. So we redirect these outputs to temporary paths. */ - PathSet redirectedBadOutputs; + StorePathSet redirectedBadOutputs; BuildResult result; /* The current round, if we're building multiple times. */ - unsigned int curRound = 1; + size_t curRound = 1; - unsigned int nrRounds; + size_t nrRounds; /* Path registration info from the previous round, if we're building multiple times. Since this contains the hash, it allows us to compare whether two rounds produced the same result. */ - ValidPathInfos prevInfos; + std::map prevInfos; const uid_t sandboxUid = 1000; const gid_t sandboxGid = 100; + const static Path homeDir; + + std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; + + std::unique_ptr act; + + /* Activity that denotes waiting for a lock. */ + std::unique_ptr actLock; + + std::map builderActivities; + + /* The remote machine on which we're building. */ + std::string machineName; + + /* The recursive Nix daemon socket. */ + AutoCloseFD daemonSocket; + + /* The daemon main thread. */ + std::thread daemonThread; + + /* The daemon worker threads. */ + std::vector daemonWorkerThreads; + + /* Paths that were added via recursive Nix calls. */ + StorePathSet addedPaths; + + /* Recursive Nix calls are only allowed to build or realize paths + in the original input closure or added via a recursive Nix call + (so e.g. you can't do 'nix-store -r /nix/store/' where + /nix/store/ is some arbitrary path in a binary cache). */ + bool isAllowed(const StorePath & path) + { + return inputPaths.count(path) || addedPaths.count(path); + } + + friend struct RestrictedStore; + public: - DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, + DerivationGoal(StorePath && drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); - DerivationGoal(const Path & drvPath, const BasicDerivation & drv, + DerivationGoal(StorePath && drvPath, const BasicDerivation & drv, Worker & worker, BuildMode buildMode = bmNormal); ~DerivationGoal(); - void timedOut() override; + /* Whether we need to perform hash rewriting if there are valid output paths. */ + bool needsHashRewrite(); + + void timedOut(Error && ex) override; string key() override { @@ -832,14 +917,14 @@ class DerivationGoal : public Goal i.e. a derivation named "aardvark" always comes before "baboon". And substitution goals always happen before derivation goals (due to "b$"). */ - return "b$" + storePathToName(drvPath) + "$" + drvPath; + return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); } void work() override; - Path getDrvPath() + StorePath getDrvPath() { - return drvPath; + return drvPath.clone(); } /* Add wanted outputs to an already existing derivation goal. */ @@ -856,6 +941,7 @@ class DerivationGoal : public Goal void closureRepaired(); void inputsRealised(); void tryToBuild(); + void tryLocalBuild(); void buildDone(); /* Is the build hook willing to perform the build? */ @@ -864,6 +950,26 @@ class DerivationGoal : public Goal /* Start building a derivation. */ void startBuilder(); + /* Fill in the environment for the builder. */ + void initEnv(); + + /* Setup tmp dir location. */ + void initTmpDir(); + + /* Write a JSON file containing the derivation attributes. */ + void writeStructuredAttrs(); + + void startDaemon(); + + void stopDaemon(); + + /* Add 'path' to the set of paths that may be referenced by the + outputs, and make it appear in the sandbox. */ + void addDependency(const StorePath & path); + + /* Make a file owned by the builder. */ + void chownToBuilder(const Path & path); + /* Run the builder's process. */ void runChild(); @@ -873,6 +979,11 @@ class DerivationGoal : public Goal as valid. */ void registerOutputs(); + /* Check that an output meets the requirements specified by the + 'outputChecks' attribute (or the legacy + '{allowed,disallowed}{References,Requisites}' attributes). */ + void checkOutputs(const std::map & outputs); + /* Open a log file and a pipe to it. */ Path openLogFile(); @@ -888,51 +999,63 @@ class DerivationGoal : public Goal void flushLine(); /* Return the set of (in)valid paths. */ - PathSet checkPathValidity(bool returnValid, bool checkHash); - - /* Abort the goal if `path' failed to build. */ - bool pathFailed(const Path & path); + StorePathSet checkPathValidity(bool returnValid, bool checkHash); /* Forcibly kill the child process, if any. */ void killChild(); - Path addHashRewrite(const Path & path); + void addHashRewrite(const StorePath & path); void repairClosure(); - void done(BuildResult::Status status, const string & msg = ""); + void started(); + + void done( + BuildResult::Status status, + std::optional ex = {}); + + StorePathSet exportReferences(const StorePathSet & storePaths); }; -DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, +const Path DerivationGoal::homeDir = "/homeless-shelter"; + + +DerivationGoal::DerivationGoal(StorePath && drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker) , useDerivation(true) - , drvPath(drvPath) + , drvPath(std::move(drvPath)) , wantedOutputs(wantedOutputs) , buildMode(buildMode) { state = &DerivationGoal::getDerivation; - name = (format("building of ‘%1%’") % drvPath).str(); + name = fmt("building of '%s'", worker.store.printStorePath(this->drvPath)); trace("created"); + + mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); + worker.updateProgress(); } -DerivationGoal::DerivationGoal(const Path & drvPath, const BasicDerivation & drv, +DerivationGoal::DerivationGoal(StorePath && drvPath, const BasicDerivation & drv, Worker & worker, BuildMode buildMode) : Goal(worker) , useDerivation(false) - , drvPath(drvPath) + , drvPath(std::move(drvPath)) , buildMode(buildMode) { - this->drv = std::unique_ptr(new BasicDerivation(drv)); + this->drv = std::make_unique(BasicDerivation(drv)); state = &DerivationGoal::haveDerivation; - name = (format("building of %1%") % showPaths(drv.outputPaths())).str(); + name = fmt("building of %s", worker.store.showPaths(drv.outputPaths())); trace("created"); + mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); + worker.updateProgress(); + /* Prevent the .chroot directory from being garbage-collected. (See isActiveTempFile() in gc.cc.) */ - worker.store.addTempRoot(drvPath); + worker.store.addTempRoot(this->drvPath); } @@ -941,17 +1064,29 @@ DerivationGoal::~DerivationGoal() /* Careful: we should never ever throw an exception from a destructor. */ try { killChild(); } catch (...) { ignoreException(); } + try { stopDaemon(); } catch (...) { ignoreException(); } try { deleteTmpDir(false); } catch (...) { ignoreException(); } try { closeLogFile(); } catch (...) { ignoreException(); } } +inline bool DerivationGoal::needsHashRewrite() +{ +#if __linux__ + return !useChroot; +#else + /* Darwin requires hash rewriting even when sandboxing is enabled. */ + return true; +#endif +} + + void DerivationGoal::killChild() { if (pid != -1) { worker.childTerminated(this); - if (buildUser.enabled()) { + if (buildUser) { /* If we're using a build user, then there is a tricky race condition: if we kill the build user before the child has done its setuid() to the build user uid, then @@ -959,7 +1094,7 @@ void DerivationGoal::killChild() pid.wait(). So also send a conventional kill to the child. */ ::kill(-pid, SIGKILL); /* ignore the result */ - buildUser.kill(); + buildUser->kill(); pid.wait(); } else pid.kill(); @@ -971,10 +1106,10 @@ void DerivationGoal::killChild() } -void DerivationGoal::timedOut() +void DerivationGoal::timedOut(Error && ex) { killChild(); - done(BuildResult::TimedOut); + done(BuildResult::TimedOut, ex); } @@ -994,10 +1129,8 @@ void DerivationGoal::addWantedOutputs(const StringSet & outputs) needRestart = true; } else for (auto & i : outputs) - if (wantedOutputs.find(i) == wantedOutputs.end()) { - wantedOutputs.insert(i); + if (wantedOutputs.insert(i).second) needRestart = true; - } } @@ -1024,8 +1157,7 @@ void DerivationGoal::loadDerivation() trace("loading derivation"); if (nrFailed != 0) { - printError(format("cannot build missing derivation ‘%1%’") % drvPath); - done(BuildResult::MiscFailure); + done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); return; } @@ -1047,11 +1179,13 @@ void DerivationGoal::haveDerivation() { trace("have derivation"); + retrySubstitution = false; + for (auto & i : drv->outputs) worker.store.addTempRoot(i.second.path); /* Check what outputs paths are not already valid. */ - PathSet invalidOutputs = checkPathValidity(false, buildMode == bmRepair); + auto invalidOutputs = checkPathValidity(false, buildMode == bmRepair); /* If they are all valid, then we're done. */ if (invalidOutputs.size() == 0 && buildMode == bmNormal) { @@ -1059,21 +1193,14 @@ void DerivationGoal::haveDerivation() return; } - /* Reject doing a hash build of anything other than a fixed-output - derivation. */ - if (buildMode == bmHash) { - if (drv->outputs.size() != 1 || - drv->outputs.find("out") == drv->outputs.end() || - drv->outputs["out"].hashAlgo == "") - throw Error(format("cannot do a hash build of non-fixed-output derivation ‘%1%’") % drvPath); - } + parsedDrv = std::make_unique(drvPath.clone(), *drv); /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ - if (settings.useSubstitutes && drv->substitutesAllowed()) + if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) for (auto & i : invalidOutputs) - addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair)); + addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ outputsSubstituted(); @@ -1087,14 +1214,16 @@ void DerivationGoal::outputsSubstituted() trace("all outputs substituted (maybe)"); if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { - done(BuildResult::TransientFailure, (format("some substitutes for the outputs of derivation ‘%1%’ failed (usually happens due to networking issues); try ‘--fallback’ to build derivation from source ") % drvPath).str()); + done(BuildResult::TransientFailure, + fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", + worker.store.printStorePath(drvPath))); return; } /* If the substitutes form an incomplete closure, then we should build the dependencies of this derivation, but after that, we can still use the substitutes for this derivation itself. */ - if (nrIncompleteClosure > 0 && !retrySubstitution) retrySubstitution = true; + if (nrIncompleteClosure > 0) retrySubstitution = true; nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; @@ -1104,7 +1233,7 @@ void DerivationGoal::outputsSubstituted() return; } - unsigned int nrInvalid = checkPathValidity(false, buildMode == bmRepair).size(); + auto nrInvalid = checkPathValidity(false, buildMode == bmRepair).size(); if (buildMode == bmNormal && nrInvalid == 0) { done(BuildResult::Substituted); return; @@ -1114,14 +1243,15 @@ void DerivationGoal::outputsSubstituted() return; } if (buildMode == bmCheck && nrInvalid > 0) - throw Error(format("some outputs of ‘%1%’ are not valid, so checking is not possible") % drvPath); + throw Error("some outputs of '%s' are not valid, so checking is not possible", + worker.store.printStorePath(drvPath)); /* Otherwise, at least one of the output paths could not be produced using a substitute. So we have to build instead. */ /* Make sure checkPathValidity() from now on checks all outputs. */ - wantedOutputs = PathSet(); + wantedOutputs.clear(); /* The inputs must be built before we can build this goal. */ if (useDerivation) @@ -1131,8 +1261,8 @@ void DerivationGoal::outputsSubstituted() for (auto & i : drv->inputSrcs) { if (worker.store.isValidPath(i)) continue; if (!settings.useSubstitutes) - throw Error(format("dependency of ‘%1%’ of ‘%2%’ does not exist, and substitution is disabled") - % i % drvPath); + throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", + worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); addWaitee(worker.makeSubstitutionGoal(i)); } @@ -1151,7 +1281,7 @@ void DerivationGoal::repairClosure() that produced those outputs. */ /* Get the output closure. */ - PathSet outputClosure; + StorePathSet outputClosure; for (auto & i : drv->outputs) { if (!wantOutput(i.first, wantedOutputs)) continue; worker.store.computeFSClosure(i.second.path, outputClosure); @@ -1164,26 +1294,30 @@ void DerivationGoal::repairClosure() /* Get all dependencies of this derivation so that we know which derivation is responsible for which path in the output closure. */ - PathSet inputClosure; + StorePathSet inputClosure; if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure); - std::map outputsToDrv; + std::map outputsToDrv; for (auto & i : inputClosure) - if (isDerivation(i)) { + if (i.isDerivation()) { Derivation drv = worker.store.derivationFromPath(i); for (auto & j : drv.outputs) - outputsToDrv[j.second.path] = i; + outputsToDrv.insert_or_assign(j.second.path.clone(), i.clone()); } /* Check each path (slow!). */ - PathSet broken; for (auto & i : outputClosure) { if (worker.pathContentsGood(i)) continue; - printError(format("found corrupted or missing path ‘%1%’ in the output closure of ‘%2%’") % i % drvPath); - Path drvPath2 = outputsToDrv[i]; - if (drvPath2 == "") - addWaitee(worker.makeSubstitutionGoal(i, true)); + logError({ + .name = "Corrupt path in closure", + .hint = hintfmt( + "found corrupted or missing path '%s' in the output closure of '%s'", + worker.store.printStorePath(i), worker.store.printStorePath(drvPath)) + }); + auto drvPath2 = outputsToDrv.find(i); + if (drvPath2 == outputsToDrv.end()) + addWaitee(worker.makeSubstitutionGoal(i, Repair)); else - addWaitee(worker.makeDerivationGoal(drvPath2, PathSet(), bmRepair)); + addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair)); } if (waitees.empty()) { @@ -1199,7 +1333,8 @@ void DerivationGoal::closureRepaired() { trace("closure repaired"); if (nrFailed > 0) - throw Error(format("some paths in the output closure of derivation ‘%1%’ could not be repaired") % drvPath); + throw Error("some paths in the output closure of derivation '%s' could not be repaired", + worker.store.printStorePath(drvPath)); done(BuildResult::AlreadyValid); } @@ -1210,11 +1345,10 @@ void DerivationGoal::inputsRealised() if (nrFailed != 0) { if (!useDerivation) - throw Error(format("some dependencies of ‘%1%’ are missing") % drvPath); - printError( - format("cannot build derivation ‘%1%’: %2% dependencies couldn't be built") - % drvPath % nrFailed); - done(BuildResult::DependencyFailed); + throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); + done(BuildResult::DependencyFailed, Error( + "%s dependencies of derivation '%s' failed to build", + nrFailed, worker.store.printStorePath(drvPath))); return; } @@ -1226,12 +1360,6 @@ void DerivationGoal::inputsRealised() /* Gather information necessary for computing the closure and/or running the build hook. */ - /* The outputs are referenceable paths. */ - for (auto & i : drv->outputs) { - debug(format("building path ‘%1%’") % i.second.path); - allPaths.insert(i.second.path); - } - /* Determine the full set of input paths. */ /* First, the input derivations. */ @@ -1242,30 +1370,28 @@ void DerivationGoal::inputsRealised() that are specified as inputs. */ assert(worker.store.isValidPath(i.first)); Derivation inDrv = worker.store.derivationFromPath(i.first); - for (auto & j : i.second) - if (inDrv.outputs.find(j) != inDrv.outputs.end()) - worker.store.computeFSClosure(inDrv.outputs[j].path, inputPaths); + for (auto & j : i.second) { + auto k = inDrv.outputs.find(j); + if (k != inDrv.outputs.end()) + worker.store.computeFSClosure(k->second.path, inputPaths); else throw Error( - format("derivation ‘%1%’ requires non-existent output ‘%2%’ from input derivation ‘%3%’") - % drvPath % j % i.first); + "derivation '%s' requires non-existent output '%s' from input derivation '%s'", + worker.store.printStorePath(drvPath), j, worker.store.printStorePath(i.first)); + } } /* Second, the input sources. */ worker.store.computeFSClosure(drv->inputSrcs, inputPaths); - debug(format("added input paths %1%") % showPaths(inputPaths)); - - allPaths.insert(inputPaths.begin(), inputPaths.end()); + debug("added input paths %s", worker.store.showPaths(inputPaths)); /* Is this a fixed-output derivation? */ - fixedOutput = true; - for (auto & i : drv->outputs) - if (i.second.hash == "") fixedOutput = false; + fixedOutput = drv->isFixedOutput(); /* Don't repeat fixed-output derivations since they're already verified by their output hash.*/ - nrRounds = fixedOutput ? 1 : settings.get("build-repeat", 0) + 1; + nrRounds = fixedOutput ? 1 : settings.buildRepeat + 1; /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a @@ -1276,24 +1402,24 @@ void DerivationGoal::inputsRealised() result = BuildResult(); } +void DerivationGoal::started() { + auto msg = fmt( + buildMode == bmRepair ? "repairing outputs of '%s'" : + buildMode == bmCheck ? "checking outputs of '%s'" : + nrRounds > 1 ? "building '%s' (round %d/%d)" : + "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); + fmt("building '%s'", worker.store.printStorePath(drvPath)); + if (hook) msg += fmt(" on '%s'", machineName); + act = std::make_unique(*logger, lvlInfo, actBuild, msg, + Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); + mcRunningBuilds = std::make_unique>(worker.runningBuilds); + worker.updateProgress(); +} void DerivationGoal::tryToBuild() { trace("trying to build"); - /* Check for the possibility that some other goal in this process - has locked the output since we checked in haveDerivation(). - (It can't happen between here and the lockPaths() call below - because we're not allowing multi-threading.) If so, put this - goal to sleep until another goal finishes, then try again. */ - for (auto & i : drv->outputs) - if (pathIsLockedByMe(worker.store.toRealPath(i.second.path))) { - debug(format("putting derivation ‘%1%’ to sleep because ‘%2%’ is locked by another goal") - % drvPath % i.second.path); - worker.waitForAnyGoal(shared_from_this()); - return; - } - /* Obtain locks on all output paths. The locks are automatically released when we exit this function or Nix crashes. If we can't acquire the lock, then continue; hopefully some other @@ -1301,13 +1427,18 @@ void DerivationGoal::tryToBuild() few seconds and then retry this goal. */ PathSet lockFiles; for (auto & outPath : drv->outputPaths()) - lockFiles.insert(worker.store.toRealPath(outPath)); + lockFiles.insert(worker.store.Store::toRealPath(outPath)); if (!outputLocks.lockPaths(lockFiles, "", false)) { + if (!actLock) + actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles)))); worker.waitForAWhile(shared_from_this()); return; } + actLock.reset(); + /* Now check again whether the outputs are valid. This is because another process may have started building in parallel. After it has finished and released the locks, we can (and should) @@ -1317,29 +1448,28 @@ void DerivationGoal::tryToBuild() build this derivation, so no further checks are necessary. */ validPaths = checkPathValidity(true, buildMode == bmRepair); if (buildMode != bmCheck && validPaths.size() == drv->outputs.size()) { - debug(format("skipping build of derivation ‘%1%’, someone beat us to it") % drvPath); + debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); outputLocks.setDeletion(true); done(BuildResult::AlreadyValid); return; } - missingPaths = drv->outputPaths(); + missingPaths = cloneStorePathSet(drv->outputPaths()); if (buildMode != bmCheck) for (auto & i : validPaths) missingPaths.erase(i); /* If any of the outputs already exist but are not valid, delete them. */ for (auto & i : drv->outputs) { - Path path = i.second.path; - if (worker.store.isValidPath(path)) continue; - debug(format("removing invalid path ‘%1%’") % path); - deletePath(worker.store.toRealPath(path)); + if (worker.store.isValidPath(i.second.path)) continue; + debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path)); + deletePath(worker.store.Store::toRealPath(i.second.path)); } /* Don't do a remote build if the derivation has the attribute `preferLocalBuild' set. Also, check and repair modes are only supported for local builds. */ - bool buildLocally = buildMode != bmNormal || drv->willBuildLocally(); + bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(); /* Is the build hook willing to accept this job? */ if (!buildLocally) { @@ -1347,12 +1477,17 @@ void DerivationGoal::tryToBuild() case rpAccept: /* Yes, it has started doing so. Wait until we get EOF from the hook. */ + actLock.reset(); result.startTime = time(0); // inexact state = &DerivationGoal::buildDone; + started(); return; case rpPostpone: /* Not now; wait until at least one child finishes or the wake-up timeout expires. */ + if (!actLock) + actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); outputLocks.unlock(); return; @@ -1362,6 +1497,8 @@ void DerivationGoal::tryToBuild() } } + actLock.reset(); + /* Make sure that we are allowed to start a build. If this derivation prefers to be done locally, do it even if maxBuildJobs is 0. */ @@ -1372,23 +1509,56 @@ void DerivationGoal::tryToBuild() return; } + state = &DerivationGoal::tryLocalBuild; + worker.wakeUp(shared_from_this()); +} + +void DerivationGoal::tryLocalBuild() { + + /* If `build-users-group' is not empty, then we have to build as + one of the members of that group. */ + if (settings.buildUsersGroup != "" && getuid() == 0) { +#if defined(__linux__) || defined(__APPLE__) + if (!buildUser) buildUser = std::make_unique(); + + if (buildUser->findFreeUser()) { + /* Make sure that no other processes are executing under this + uid. */ + buildUser->kill(); + } else { + if (!actLock) + actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); + worker.waitForAWhile(shared_from_this()); + return; + } +#else + /* Don't know how to block the creation of setuid/setgid + binaries on this platform. */ + throw Error("build users are not supported on this platform for security reasons"); +#endif + } + + actLock.reset(); + try { /* Okay, we have to build. */ startBuilder(); } catch (BuildError & e) { - printError(e.msg()); outputLocks.unlock(); - buildUser.release(); + buildUser.reset(); worker.permanentFailure = true; - done(BuildResult::InputRejected, e.msg()); + done(BuildResult::InputRejected, e); return; } /* This state will be reached when we get EOF on the child's log pipe. */ state = &DerivationGoal::buildDone; + + started(); } @@ -1398,29 +1568,38 @@ void replaceValidPath(const Path & storePath, const Path tmpPath) tmpPath (the replacement), so we have to move it out of the way first. We'd better not be interrupted here, because if we're repairing (say) Glibc, we end up with a broken system. */ - Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % rand()).str(); + Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); if (pathExists(storePath)) rename(storePath.c_str(), oldPath.c_str()); - if (rename(tmpPath.c_str(), storePath.c_str()) == -1) - throw SysError(format("moving ‘%1%’ to ‘%2%’") % tmpPath % storePath); + if (rename(tmpPath.c_str(), storePath.c_str()) == -1) { + rename(oldPath.c_str(), storePath.c_str()); // attempt to recover + throw SysError("moving '%s' to '%s'", tmpPath, storePath); + } deletePath(oldPath); } -MakeError(NotDeterministic, BuildError) +MakeError(NotDeterministic, BuildError); void DerivationGoal::buildDone() { trace("build done"); + /* Release the build user at the end of this function. We don't do + it right away because we don't want another build grabbing this + uid and then messing around with our output. */ + Finally releaseBuildUser([&]() { buildUser.reset(); }); + + sandboxMountNamespace = -1; + /* Since we got an EOF on the logger pipe, the builder is presumed to have terminated. In fact, the builder could also have simply have closed its end of the pipe, so just to be sure, kill it. */ - int status = hook ? hook->pid.kill(true) : pid.kill(true); + int status = hook ? hook->pid.kill() : pid.kill(); - debug(format("builder process for ‘%1%’ finished") % drvPath); + debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); result.timesBuilt++; result.stopTime = time(0); @@ -1432,8 +1611,8 @@ void DerivationGoal::buildDone() if (hook) { hook->builderOut.readSide = -1; hook->fromHook.readSide = -1; - } - else builderOut.readSide = -1; + } else + builderOut.readSide = -1; /* Close the log file. */ closeLogFile(); @@ -1443,7 +1622,10 @@ void DerivationGoal::buildDone() malicious user from leaving behind a process that keeps files open and modifies them after they have been chown'ed to root. */ - if (buildUser.enabled()) buildUser.kill(); + if (buildUser) buildUser->kill(); + + /* Terminate the recursive Nix daemon. */ + stopDaemon(); bool diskFull = false; @@ -1473,14 +1655,17 @@ void DerivationGoal::buildDone() /* Move paths out of the chroot for easier debugging of build failures. */ if (useChroot && buildMode == bmNormal) - for (auto & i : missingPaths) - if (pathExists(chrootRootDir + i)) - rename((chrootRootDir + i).c_str(), i.c_str()); + for (auto & i : missingPaths) { + auto p = worker.store.printStorePath(i); + if (pathExists(chrootRootDir + p)) + rename((chrootRootDir + p).c_str(), p.c_str()); + } - std::string msg = (format("builder for ‘%1%’ %2%") - % drvPath % statusToString(status)).str(); + auto msg = fmt("builder for '%s' %s", + yellowtxt(worker.store.printStorePath(drvPath)), + statusToString(status)); - if (!settings.verboseBuild && !logTail.empty()) { + if (!logger->isVerbose() && !logTail.empty()) { msg += (format("; last %d log lines:") % logTail.size()).str(); for (auto & line : logTail) msg += "\n " + line; @@ -1496,14 +1681,66 @@ void DerivationGoal::buildDone() being valid. */ registerOutputs(); + if (settings.postBuildHook != "") { + Activity act(*logger, lvlInfo, actPostBuildHook, + fmt("running post-build-hook '%s'", settings.postBuildHook), + Logger::Fields{worker.store.printStorePath(drvPath)}); + PushActivity pact(act.id); + auto outputPaths = drv->outputPaths(); + std::map hookEnvironment = getEnv(); + + hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); + hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths)))); + + RunOptions opts(settings.postBuildHook, {}); + opts.environment = hookEnvironment; + + struct LogSink : Sink { + Activity & act; + std::string currentLine; + + LogSink(Activity & act) : act(act) { } + + void operator() (const unsigned char * data, size_t len) override { + for (size_t i = 0; i < len; i++) { + auto c = data[i]; + + if (c == '\n') { + flushLine(); + } else { + currentLine += c; + } + } + } + + void flushLine() { + act.result(resPostBuildLogLine, currentLine); + currentLine.clear(); + } + + ~LogSink() { + if (currentLine != "") { + currentLine += '\n'; + flushLine(); + } + } + }; + LogSink sink(act); + + opts.standardOut = &sink; + opts.mergeStderrToStdout = true; + runProgram2(opts); + } + if (buildMode == bmCheck) { + deleteTmpDir(true); done(BuildResult::Built); return; } /* Delete unused redirected outputs (when doing hash rewriting). */ for (auto & i : redirectedOutputs) - deletePath(i.second); + deletePath(worker.store.Store::toRealPath(i.second)); /* Delete the chroot (if we were using one). */ autoDelChroot.reset(); /* this runs the destructor */ @@ -1513,7 +1750,6 @@ void DerivationGoal::buildDone() /* Repeat the build if necessary. */ if (curRound++ < nrRounds) { outputLocks.unlock(); - buildUser.release(); state = &DerivationGoal::tryToBuild; worker.wakeUp(shared_from_this()); return; @@ -1527,10 +1763,7 @@ void DerivationGoal::buildDone() outputLocks.unlock(); } catch (BuildError & e) { - if (!hook) - printError(e.msg()); outputLocks.unlock(); - buildUser.release(); BuildResult::Status st = BuildResult::MiscFailure; @@ -1548,78 +1781,90 @@ void DerivationGoal::buildDone() BuildResult::PermanentFailure; } - done(st, e.msg()); + done(st, e); return; } - /* Release the build user, if applicable. */ - buildUser.release(); - done(BuildResult::Built); } HookReply DerivationGoal::tryBuildHook() { - if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "" || !useDerivation) return rpDecline; + if (!worker.tryBuildHook || !useDerivation) return rpDecline; if (!worker.hook) worker.hook = std::make_unique(); - /* Tell the hook about system features (beyond the system type) - required from the build machine. (The hook could parse the - drv file itself, but this is easier.) */ - Strings features = tokenizeString(get(drv->env, "requiredSystemFeatures")); - for (auto & i : features) checkStoreName(i); /* !!! abuse */ - - /* Send the request to the hook. */ - writeLine(worker.hook->toHook.writeSide.get(), (format("%1% %2% %3% %4%") - % (worker.getNrLocalBuilds() < settings.maxBuildJobs ? "1" : "0") - % drv->platform % drvPath % concatStringsSep(",", features)).str()); + try { - /* Read the first line of input, which should be a word indicating - whether the hook wishes to perform the build. */ - string reply; - while (true) { - string s = readLine(worker.hook->fromHook.readSide.get()); - if (string(s, 0, 2) == "# ") { - reply = string(s, 2); - break; + /* Send the request to the hook. */ + worker.hook->sink + << "try" + << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) + << drv->platform + << worker.store.printStorePath(drvPath) + << parsedDrv->getRequiredSystemFeatures(); + worker.hook->sink.flush(); + + /* Read the first line of input, which should be a word indicating + whether the hook wishes to perform the build. */ + string reply; + while (true) { + string s = readLine(worker.hook->fromHook.readSide.get()); + if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) + ; + else if (string(s, 0, 2) == "# ") { + reply = string(s, 2); + break; + } + else { + s += "\n"; + writeToStderr(s); + } } - s += "\n"; - writeToStderr(s); - } - debug(format("hook reply is ‘%1%’") % reply); + debug("hook reply is '%1%'", reply); - if (reply == "decline" || reply == "postpone") - return reply == "decline" ? rpDecline : rpPostpone; - else if (reply != "accept") - throw Error(format("bad hook reply ‘%1%’") % reply); - - printMsg(lvlTalkative, format("using hook to build path(s) %1%") % showPaths(missingPaths)); + if (reply == "decline") + return rpDecline; + else if (reply == "decline-permanently") { + worker.tryBuildHook = false; + worker.hook = 0; + return rpDecline; + } + else if (reply == "postpone") + return rpPostpone; + else if (reply != "accept") + throw Error("bad hook reply '%s'", reply); + + } catch (SysError & e) { + if (e.errNo == EPIPE) { + logError({ + .name = "Build hook died", + .hint = hintfmt( + "build hook died unexpectedly: %s", + chomp(drainFD(worker.hook->fromHook.readSide.get()))) + }); + worker.hook = 0; + return rpDecline; + } else + throw; + } hook = std::move(worker.hook); + machineName = readLine(hook->fromHook.readSide.get()); + /* Tell the hook all the inputs that have to be copied to the - remote system. This unfortunately has to contain the entire - derivation closure to ensure that the validity invariant holds - on the remote system. (I.e., it's unfortunate that we have to - list it since the remote system *probably* already has it.) */ - PathSet allInputs; - allInputs.insert(inputPaths.begin(), inputPaths.end()); - worker.store.computeFSClosure(drvPath, allInputs); - - string s; - for (auto & i : allInputs) { s += i; s += ' '; } - writeLine(hook->toHook.writeSide.get(), s); + remote system. */ + writeStorePaths(worker.store, hook->sink, inputPaths); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ - s = ""; - for (auto & i : missingPaths) { s += i; s += ' '; } - writeLine(hook->toHook.writeSide.get(), s); + writeStorePaths(worker.store, hook->sink, missingPaths); + hook->sink = FdSink(); hook->toHook.writeSide = -1; /* Create the log file and pipe. */ @@ -1634,10 +1879,10 @@ HookReply DerivationGoal::tryBuildHook() } -void chmod_(const Path & path, mode_t mode) +static void chmod_(const Path & path, mode_t mode) { if (chmod(path.c_str(), mode) == -1) - throw SysError(format("setting permissions on ‘%1%’") % path); + throw SysError("setting permissions on '%s'", path); } @@ -1648,232 +1893,161 @@ int childEntry(void * arg) } -void DerivationGoal::startBuilder() +StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) { - auto f = format( - buildMode == bmRepair ? "repairing path(s) %1%" : - buildMode == bmCheck ? "checking path(s) %1%" : - nrRounds > 1 ? "building path(s) %1% (round %2%/%3%)" : - "building path(s) %1%"); - f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); - printInfo(f % showPaths(missingPaths) % curRound % nrRounds); + StorePathSet paths; - /* Right platform? */ - if (!drv->canBuildLocally()) { - throw Error( - format("a ‘%1%’ is required to build ‘%3%’, but I am a ‘%2%’") - % drv->platform % settings.thisSystem % drvPath); + for (auto & storePath : storePaths) { + if (!inputPaths.count(storePath)) + throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath)); + + worker.store.computeFSClosure(singleton(storePath), paths); + } + + /* If there are derivations in the graph, then include their + outputs as well. This is useful if you want to do things + like passing all build-time dependencies of some path to a + derivation that builds a NixOS DVD image. */ + auto paths2 = cloneStorePathSet(paths); + + for (auto & j : paths2) { + if (j.isDerivation()) { + Derivation drv = worker.store.derivationFromPath(j); + for (auto & k : drv.outputs) + worker.store.computeFSClosure(k.second.path, paths); + } + } + + return paths; +} + +static std::once_flag dns_resolve_flag; + +static void preloadNSS() { + /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of + one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already + been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to + load its lookup libraries in the parent before any child gets a chance to. */ + std::call_once(dns_resolve_flag, []() { + struct addrinfo *res = NULL; + + if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) { + if (res) freeaddrinfo(res); + } + }); +} + + +void linkOrCopy(const Path & from, const Path & to) +{ + if (link(from.c_str(), to.c_str()) == -1) { + /* Hard-linking fails if we exceed the maximum link count on a + file (e.g. 32000 of ext3), which is quite possible after a + 'nix-store --optimise'. FIXME: actually, why don't we just + bind-mount in this case? */ + if (errno != EMLINK) + throw SysError("linking '%s' to '%s'", to, from); + copyPath(from, to); } +} + + +void DerivationGoal::startBuilder() +{ + /* Right platform? */ + if (!parsedDrv->canBuildLocally()) + throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", + drv->platform, + concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), + worker.store.printStorePath(drvPath), + settings.thisSystem, + concatStringsSep(", ", settings.systemFeatures)); + + if (drv->isBuiltin()) + preloadNSS(); #if __APPLE__ - additionalSandboxProfile = get(drv->env, "__sandboxProfile"); + additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); #endif - /* Are we doing a chroot build? Note that fixed-output - derivations are never done in a chroot, mainly so that - functions like fetchurl (which needs a proper /etc/resolv.conf) - work properly. Purity checking for fixed-output derivations - is somewhat pointless anyway. */ + /* Are we doing a chroot build? */ { - string x = settings.get("build-use-sandbox", - /* deprecated alias */ - settings.get("build-use-chroot", string("false"))); - if (x != "true" && x != "false" && x != "relaxed") - throw Error("option ‘build-use-sandbox’ must be set to one of ‘true’, ‘false’ or ‘relaxed’"); - if (x == "true") { - if (get(drv->env, "__noChroot") == "1") - throw Error(format("derivation ‘%1%’ has ‘__noChroot’ set, " - "but that's not allowed when ‘build-use-sandbox’ is ‘true’") % drvPath); + auto noChroot = parsedDrv->getBoolAttr("__noChroot"); + if (settings.sandboxMode == smEnabled) { + if (noChroot) + throw Error("derivation '%s' has '__noChroot' set, " + "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); #if __APPLE__ if (additionalSandboxProfile != "") - throw Error(format("derivation ‘%1%’ specifies a sandbox profile, " - "but this is only allowed when ‘build-use-sandbox’ is ‘relaxed’") % drvPath); + throw Error("derivation '%s' specifies a sandbox profile, " + "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); #endif useChroot = true; } - else if (x == "false") + else if (settings.sandboxMode == smDisabled) useChroot = false; - else if (x == "relaxed") - useChroot = !fixedOutput && get(drv->env, "__noChroot") != "1"; + else if (settings.sandboxMode == smRelaxed) + useChroot = !fixedOutput && !noChroot; } - if (worker.store.storeDir != worker.store.realStoreDir) - useChroot = true; - - /* Construct the environment passed to the builder. */ - env.clear(); - - /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when - PATH is not set. We don't want this, so we fill it in with some dummy - value. */ - env["PATH"] = "/path-not-set"; - - /* Set HOME to a non-existing path to prevent certain programs from using - /etc/passwd (or NIS, or whatever) to locate the home directory (for - example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd - if HOME is not set, but they will just assume that the settings file - they are looking for does not exist if HOME is set but points to some - non-existing path. */ - Path homeDir = "/homeless-shelter"; - env["HOME"] = homeDir; - - /* Tell the builder where the Nix store is. Usually they - shouldn't care, but this is useful for purity checking (e.g., - the compiler or linker might only want to accept paths to files - in the store or in the build directory). */ - env["NIX_STORE"] = worker.store.storeDir; - - /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); + if (worker.store.storeDir != worker.store.realStoreDir) { + #if __linux__ + useChroot = true; + #else + throw Error("building using a diverted store is not supported on this platform"); + #endif + } /* Create a temporary directory where the build will take place. */ - auto drvName = storePathToName(drvPath); - tmpDir = createTempDir("", "nix-build-" + drvName, false, false, 0700); + tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); - /* In a sandbox, for determinism, always use the same temporary - directory. */ - tmpDirInSandbox = useChroot ? canonPath("/tmp", true) + "/nix-build-" + drvName + "-0" : tmpDir; - - /* Add all bindings specified in the derivation via the - environments, except those listed in the passAsFile - attribute. Those are passed as file names pointing to - temporary files containing the contents. */ - PathSet filesToChown; - StringSet passAsFile = tokenizeString(get(drv->env, "passAsFile")); - int fileNr = 0; - for (auto & i : drv->env) { - if (passAsFile.find(i.first) == passAsFile.end()) { - env[i.first] = i.second; - } else { - string fn = ".attr-" + std::to_string(fileNr++); - Path p = tmpDir + "/" + fn; - writeFile(p, i.second); - filesToChown.insert(p); - env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; + chownToBuilder(tmpDir); + + /* Substitute output placeholders with the actual output paths. */ + for (auto & output : drv->outputs) + inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path); + + /* Construct the environment passed to the builder. */ + initEnv(); + + writeStructuredAttrs(); + + /* Handle exportReferencesGraph(), if set. */ + if (!parsedDrv->getStructuredAttrs()) { + /* The `exportReferencesGraph' feature allows the references graph + to be passed to a builder. This attribute should be a list of + pairs [name1 path1 name2 path2 ...]. The references graph of + each `pathN' will be stored in a text file `nameN' in the + temporary build directory. The text files have the format used + by `nix-store --register-validity'. However, the deriver + fields are left empty. */ + string s = get(drv->env, "exportReferencesGraph").value_or(""); + Strings ss = tokenizeString(s); + if (ss.size() % 2 != 0) + throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); + for (Strings::iterator i = ss.begin(); i != ss.end(); ) { + string fileName = *i++; + static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); + if (!std::regex_match(fileName, regex)) + throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); + + auto storePath = worker.store.parseStorePath(*i++); + + /* Write closure info to . */ + writeFile(tmpDir + "/" + fileName, + worker.store.makeValidityRegistration( + exportReferences(singleton(storePath)), false, false)); } } - /* For convenience, set an environment pointing to the top build - directory. */ - env["NIX_BUILD_TOP"] = tmpDirInSandbox; + if (useChroot) { - /* Also set TMPDIR and variants to point to this directory. */ - env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; - - /* Explicitly set PWD to prevent problems with chroot builds. In - particular, dietlibc cannot figure out the cwd because the - inode of the current directory doesn't appear in .. (because - getdents returns the inode of the mount point). */ - env["PWD"] = tmpDirInSandbox; - - /* Compatibility hack with Nix <= 0.7: if this is a fixed-output - derivation, tell the builder, so that for instance `fetchurl' - can skip checking the output. On older Nixes, this environment - variable won't be set, so `fetchurl' will do the check. */ - if (fixedOutput) env["NIX_OUTPUT_CHECKED"] = "1"; - - /* *Only* if this is a fixed-output derivation, propagate the - values of the environment variables specified in the - `impureEnvVars' attribute to the builder. This allows for - instance environment variables for proxy configuration such as - `http_proxy' to be easily passed to downloaders like - `fetchurl'. Passing such environment variables from the caller - to the builder is generally impure, but the output of - fixed-output derivations is by definition pure (since we - already know the cryptographic hash of the output). */ - if (fixedOutput) { - Strings varNames = tokenizeString(get(drv->env, "impureEnvVars")); - for (auto & i : varNames) env[i] = getEnv(i); - } - - /* Substitute output placeholders with the actual output paths. */ - for (auto & output : drv->outputs) - inputRewrites[hashPlaceholder(output.first)] = output.second.path; - - /* The `exportReferencesGraph' feature allows the references graph - to be passed to a builder. This attribute should be a list of - pairs [name1 path1 name2 path2 ...]. The references graph of - each `pathN' will be stored in a text file `nameN' in the - temporary build directory. The text files have the format used - by `nix-store --register-validity'. However, the deriver - fields are left empty. */ - string s = get(drv->env, "exportReferencesGraph"); - Strings ss = tokenizeString(s); - if (ss.size() % 2 != 0) - throw BuildError(format("odd number of tokens in ‘exportReferencesGraph’: ‘%1%’") % s); - for (Strings::iterator i = ss.begin(); i != ss.end(); ) { - string fileName = *i++; - checkStoreName(fileName); /* !!! abuse of this function */ - - /* Check that the store path is valid. */ - Path storePath = *i++; - if (!worker.store.isInStore(storePath)) - throw BuildError(format("‘exportReferencesGraph’ contains a non-store path ‘%1%’") - % storePath); - storePath = worker.store.toStorePath(storePath); - if (!worker.store.isValidPath(storePath)) - throw BuildError(format("‘exportReferencesGraph’ contains an invalid path ‘%1%’") - % storePath); - - /* If there are derivations in the graph, then include their - outputs as well. This is useful if you want to do things - like passing all build-time dependencies of some path to a - derivation that builds a NixOS DVD image. */ - PathSet paths, paths2; - worker.store.computeFSClosure(storePath, paths); - paths2 = paths; - - for (auto & j : paths2) { - if (isDerivation(j)) { - Derivation drv = worker.store.derivationFromPath(j); - for (auto & k : drv.outputs) - worker.store.computeFSClosure(k.second.path, paths); - } - } - - /* Write closure info to `fileName'. */ - writeFile(tmpDir + "/" + fileName, - worker.store.makeValidityRegistration(paths, false, false)); - } - - - /* If `build-users-group' is not empty, then we have to build as - one of the members of that group. */ - if (settings.buildUsersGroup != "" && getuid() == 0) { - buildUser.acquire(); - - /* Make sure that no other processes are executing under this - uid. */ - buildUser.kill(); - - /* Change ownership of the temporary build directory. */ - filesToChown.insert(tmpDir); - - for (auto & p : filesToChown) - if (chown(p.c_str(), buildUser.getUID(), buildUser.getGID()) == -1) - throw SysError(format("cannot change ownership of ‘%1%’") % p); - } - - - if (useChroot) { - - string defaultChrootDirs; -#if __linux__ - if (worker.store.isInStore(BASH_PATH)) - defaultChrootDirs = "/bin/sh=" BASH_PATH; -#endif - - /* Allow a user-configurable set of directories from the - host file system. */ - PathSet dirs = tokenizeString( - settings.get("build-sandbox-paths", - /* deprecated alias with lower priority */ - settings.get("build-chroot-dirs", defaultChrootDirs))); - PathSet dirs2 = tokenizeString( - settings.get("build-extra-chroot-dirs", - settings.get("build-extra-sandbox-paths", string("")))); - dirs.insert(dirs2.begin(), dirs2.end()); + /* Allow a user-configurable set of directories from the + host file system. */ + PathSet dirs = settings.sandboxPaths; + PathSet dirs2 = settings.extraSandboxPaths; + dirs.insert(dirs2.begin(), dirs2.end()); dirsInChroot.clear(); @@ -1893,22 +2067,24 @@ void DerivationGoal::startBuilder() dirsInChroot[tmpDirInSandbox] = tmpDir; /* Add the closure of store paths to the chroot. */ - PathSet closure; + StorePathSet closure; for (auto & i : dirsInChroot) try { if (worker.store.isInStore(i.second.source)) - worker.store.computeFSClosure(worker.store.toStorePath(i.second.source), closure); + worker.store.computeFSClosure(worker.store.parseStorePath(worker.store.toStorePath(i.second.source)), closure); + } catch (InvalidPath & e) { } catch (Error & e) { - throw Error(format("while processing ‘build-sandbox-paths’: %s") % e.what()); + throw Error("while processing 'sandbox-paths': %s", e.what()); } - for (auto & i : closure) - dirsInChroot[i] = i; + for (auto & i : closure) { + auto p = worker.store.printStorePath(i); + dirsInChroot.insert_or_assign(p, p); + } - string allowed = settings.get("allowed-impure-host-deps", string(DEFAULT_ALLOWED_IMPURE_PREFIXES)); - PathSet allowedPaths = tokenizeString(allowed); + PathSet allowedPaths = settings.allowedImpureHostPrefixes; /* This works like the above, except on a per-derivation level */ - Strings impurePaths = tokenizeString(get(drv->env, "__impureHostDeps")); + auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings()); for (auto & i : impurePaths) { bool found = false; @@ -1925,7 +2101,8 @@ void DerivationGoal::startBuilder() } } if (!found) - throw Error(format("derivation ‘%1%’ requested impure path ‘%2%’, but it was not in allowed-impure-host-deps (‘%3%’)") % drvPath % i % allowed); + throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", + worker.store.printStorePath(drvPath), i); dirsInChroot[i] = i; } @@ -1935,19 +2112,19 @@ void DerivationGoal::startBuilder() environment using bind-mounts. We put it in the Nix store to ensure that we can create hard-links to non-directory inputs in the fake Nix store in the chroot (see below). */ - chrootRootDir = worker.store.toRealPath(drvPath) + ".chroot"; + chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; deletePath(chrootRootDir); /* Clean up the chroot directory automatically. */ autoDelChroot = std::make_shared(chrootRootDir); - printMsg(lvlChatty, format("setting up chroot environment in ‘%1%’") % chrootRootDir); + printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); if (mkdir(chrootRootDir.c_str(), 0750) == -1) - throw SysError(format("cannot create ‘%1%’") % chrootRootDir); + throw SysError("cannot create '%1%'", chrootRootDir); - if (buildUser.enabled() && chown(chrootRootDir.c_str(), 0, buildUser.getGID()) == -1) - throw SysError(format("cannot change ownership of ‘%1%’") % chrootRootDir); + if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) + throw SysError("cannot change ownership of '%1%'", chrootRootDir); /* Create a writable /tmp in the chroot. Many builders need this. (Of course they should really respect $TMPDIR @@ -1961,11 +2138,11 @@ void DerivationGoal::startBuilder() Samba-in-QEMU. */ createDirs(chrootRootDir + "/etc"); - writeFile(chrootRootDir + "/etc/passwd", - (format( - "root:x:0:0:Nix build user:/:/noshell\n" - "nixbld:x:%1%:%2%:Nix build user:/:/noshell\n" - "nobody:x:65534:65534:Nobody:/:/noshell\n") % sandboxUid % sandboxGid).str()); + writeFile(chrootRootDir + "/etc/passwd", fmt( + "root:x:0:0:Nix build user:%3%:/noshell\n" + "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" + "nobody:x:65534:65534:Nobody:/:/noshell\n", + sandboxUid, sandboxGid, settings.sandboxBuildDir)); /* Declare the build user's group so that programs get a consistent view of the system (e.g., "id -gn"). */ @@ -1977,7 +2154,7 @@ void DerivationGoal::startBuilder() /* Create /etc/hosts with localhost entry. */ if (!fixedOutput) - writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n"); + writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); /* Make the closure of the inputs available in the chroot, rather than the whole Nix store. This prevents any access @@ -1990,31 +2167,19 @@ void DerivationGoal::startBuilder() createDirs(chrootStoreDir); chmod_(chrootStoreDir, 01775); - if (buildUser.enabled() && chown(chrootStoreDir.c_str(), 0, buildUser.getGID()) == -1) - throw SysError(format("cannot change ownership of ‘%1%’") % chrootStoreDir); + if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) + throw SysError("cannot change ownership of '%1%'", chrootStoreDir); for (auto & i : inputPaths) { - Path r = worker.store.toRealPath(i); + auto p = worker.store.printStorePath(i); + Path r = worker.store.toRealPath(p); struct stat st; if (lstat(r.c_str(), &st)) - throw SysError(format("getting attributes of path ‘%1%’") % i); + throw SysError("getting attributes of path '%s'", p); if (S_ISDIR(st.st_mode)) - dirsInChroot[i] = r; - else { - Path p = chrootRootDir + i; - if (link(r.c_str(), p.c_str()) == -1) { - /* Hard-linking fails if we exceed the maximum - link count on a file (e.g. 32000 of ext3), - which is quite possible after a `nix-store - --optimise'. */ - if (errno != EMLINK) - throw SysError(format("linking ‘%1%’ to ‘%2%’") % p % i); - StringSink sink; - dumpPath(r, sink); - StringSource source(*sink.s); - restorePath(p, source); - } - } + dirsInChroot.insert_or_assign(p, r); + else + linkOrCopy(r, chrootRootDir + p); } /* If we're repairing, checking or rebuilding part of a @@ -2023,7 +2188,7 @@ void DerivationGoal::startBuilder() (typically the dependencies of /bin/sh). Throw them out. */ for (auto & i : drv->outputs) - dirsInChroot.erase(i.second.path); + dirsInChroot.erase(worker.store.printStorePath(i.second.path)); #elif __APPLE__ /* We don't really have any parent prep work to do (yet?) @@ -2033,10 +2198,10 @@ void DerivationGoal::startBuilder() #endif } - else { + if (needsHashRewrite()) { if (pathExists(homeDir)) - throw Error(format("directory ‘%1%’ exists; please remove it") % homeDir); + throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); /* We're not doing a chroot build, but we have some valid output paths. Since we can't just overwrite or delete @@ -2055,17 +2220,17 @@ void DerivationGoal::startBuilder() corrupt outputs in advance. So rewrite them as well. */ if (buildMode == bmRepair) for (auto & i : missingPaths) - if (worker.store.isValidPath(i) && pathExists(i)) { + if (worker.store.isValidPath(i) && pathExists(worker.store.printStorePath(i))) { addHashRewrite(i); - redirectedBadOutputs.insert(i); + redirectedBadOutputs.insert(i.clone()); } } - if (settings.preBuildHook != "") { - printMsg(lvlChatty, format("executing pre-build hook ‘%1%’") + if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { + printMsg(lvlChatty, format("executing pre-build hook '%1%'") % settings.preBuildHook); - auto args = useChroot ? Strings({drvPath, chrootRootDir}) : - Strings({ drvPath }); + auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) : + Strings({ worker.store.printStorePath(drvPath) }); enum BuildHookState { stBegin, stExtraChrootDirs @@ -2081,8 +2246,7 @@ void DerivationGoal::startBuilder() if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { state = stExtraChrootDirs; } else { - throw Error(format("unknown pre-build hook command ‘%1%’") - % line); + throw Error("unknown pre-build hook command '%1%'", line); } } else if (state == stExtraChrootDirs) { if (line == "") { @@ -2098,14 +2262,63 @@ void DerivationGoal::startBuilder() } } + /* Fire up a Nix daemon to process recursive Nix calls from the + builder. */ + if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix")) + startDaemon(); + /* Run the builder. */ - printMsg(lvlChatty, format("executing builder ‘%1%’") % drv->builder); + printMsg(lvlChatty, "executing builder '%1%'", drv->builder); /* Create the log file. */ Path logFile = openLogFile(); /* Create a pipe to get the output of the builder. */ - builderOut.create(); + //builderOut.create(); + + builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); + if (!builderOut.readSide) + throw SysError("opening pseudoterminal master"); + + std::string slaveName(ptsname(builderOut.readSide.get())); + + if (buildUser) { + if (chmod(slaveName.c_str(), 0600)) + throw SysError("changing mode of pseudoterminal slave"); + + if (chown(slaveName.c_str(), buildUser->getUID(), 0)) + throw SysError("changing owner of pseudoterminal slave"); + } +#if __APPLE__ + else { + if (grantpt(builderOut.readSide.get())) + throw SysError("granting access to pseudoterminal slave"); + } +#endif + + #if 0 + // Mount the pt in the sandbox so that the "tty" command works. + // FIXME: this doesn't work with the new devpts in the sandbox. + if (useChroot) + dirsInChroot[slaveName] = {slaveName, false}; + #endif + + if (unlockpt(builderOut.readSide.get())) + throw SysError("unlocking pseudoterminal"); + + builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); + if (!builderOut.writeSide) + throw SysError("opening pseudoterminal slave"); + + // Put the pt into raw mode to prevent \n -> \r\n translation. + struct termios term; + if (tcgetattr(builderOut.writeSide.get(), &term)) + throw SysError("getting pseudoterminal attributes"); + + cfmakeraw(&term); + + if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) + throw SysError("putting pseudoterminal into raw mode"); result.startTime = time(0); @@ -2178,17 +2391,37 @@ void DerivationGoal::startBuilder() flags |= CLONE_NEWNET; pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) + if (child == -1 && errno == EINVAL) { /* Fallback for Linux < 2.13 where CLONE_NEWPID and CLONE_PARENT are not allowed together. */ - child = clone(childEntry, stack + stackSize, flags & ~CLONE_NEWPID, this); + flags &= ~CLONE_NEWPID; + child = clone(childEntry, stack + stackSize, flags, this); + } + if (child == -1 && (errno == EPERM || errno == EINVAL)) { + /* Some distros patch Linux to not allow unprivileged + * user namespaces. If we get EPERM or EINVAL, try + * without CLONE_NEWUSER and see if that works. + */ + flags &= ~CLONE_NEWUSER; + child = clone(childEntry, stack + stackSize, flags, this); + } + /* Otherwise exit with EPERM so we can handle this in the + parent. This is only done when sandbox-fallback is set + to true (the default). */ + if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) + _exit(1); if (child == -1) throw SysError("cloning builder process"); writeFull(builderOut.writeSide.get(), std::to_string(child) + "\n"); _exit(0); }, options); - if (helper.wait() != 0) + int res = helper.wait(); + if (res != 0 && settings.sandboxFallback) { + useChroot = false; + initTmpDir(); + goto fallback; + } else if (res != 0) throw Error("unable to start build process"); userNamespaceSync.readSide = -1; @@ -2200,45 +2433,643 @@ void DerivationGoal::startBuilder() /* Set the UID/GID mapping of the builder's user namespace such that the sandbox user maps to the build user, or to the calling user (if build users are disabled). */ - uid_t hostUid = buildUser.enabled() ? buildUser.getUID() : getuid(); - uid_t hostGid = buildUser.enabled() ? buildUser.getGID() : getgid(); + uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); + uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); writeFile("/proc/" + std::to_string(pid) + "/uid_map", (format("%d %d 1") % sandboxUid % hostUid).str()); - writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); + writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); + + writeFile("/proc/" + std::to_string(pid) + "/gid_map", + (format("%d %d 1") % sandboxGid % hostGid).str()); + + /* Save the mount namespace of the child. We have to do this + *before* the child does a chroot. */ + sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); + if (sandboxMountNamespace.get() == -1) + throw SysError("getting sandbox mount namespace"); + + /* Signal the builder that we've updated its user namespace. */ + writeFull(userNamespaceSync.writeSide.get(), "1"); + userNamespaceSync.writeSide = -1; + + } else +#endif + { + fallback: + options.allowVfork = !buildUser && !drv->isBuiltin(); + pid = startProcess([&]() { + runChild(); + }, options); + } + + /* parent */ + pid.setSeparatePG(true); + builderOut.writeSide = -1; + worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true); + + /* Check if setting up the build environment failed. */ + while (true) { + string msg = readLine(builderOut.readSide.get()); + if (string(msg, 0, 1) == "\1") { + if (msg.size() == 1) break; + throw Error(string(msg, 1)); + } + debug(msg); + } +} + + +void DerivationGoal::initTmpDir() { + /* In a sandbox, for determinism, always use the same temporary + directory. */ +#if __linux__ + tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; +#else + tmpDirInSandbox = tmpDir; +#endif + + /* In non-structured mode, add all bindings specified in the + derivation via the environment, except those listed in the + passAsFile attribute. Those are passed as file names pointing + to temporary files containing the contents. Note that + passAsFile is ignored in structure mode because it's not + needed (attributes are not passed through the environment, so + there is no size constraint). */ + if (!parsedDrv->getStructuredAttrs()) { + + StringSet passAsFile = tokenizeString(get(drv->env, "passAsFile").value_or("")); + for (auto & i : drv->env) { + if (passAsFile.find(i.first) == passAsFile.end()) { + env[i.first] = i.second; + } else { + auto hash = hashString(htSHA256, i.first); + string fn = ".attr-" + hash.to_string(Base32, false); + Path p = tmpDir + "/" + fn; + writeFile(p, rewriteStrings(i.second, inputRewrites)); + chownToBuilder(p); + env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; + } + } + + } + + /* For convenience, set an environment pointing to the top build + directory. */ + env["NIX_BUILD_TOP"] = tmpDirInSandbox; + + /* Also set TMPDIR and variants to point to this directory. */ + env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; + + /* Explicitly set PWD to prevent problems with chroot builds. In + particular, dietlibc cannot figure out the cwd because the + inode of the current directory doesn't appear in .. (because + getdents returns the inode of the mount point). */ + env["PWD"] = tmpDirInSandbox; +} + + +void DerivationGoal::initEnv() +{ + env.clear(); + + /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when + PATH is not set. We don't want this, so we fill it in with some dummy + value. */ + env["PATH"] = "/path-not-set"; + + /* Set HOME to a non-existing path to prevent certain programs from using + /etc/passwd (or NIS, or whatever) to locate the home directory (for + example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd + if HOME is not set, but they will just assume that the settings file + they are looking for does not exist if HOME is set but points to some + non-existing path. */ + env["HOME"] = homeDir; + + /* Tell the builder where the Nix store is. Usually they + shouldn't care, but this is useful for purity checking (e.g., + the compiler or linker might only want to accept paths to files + in the store or in the build directory). */ + env["NIX_STORE"] = worker.store.storeDir; + + /* The maximum number of cores to utilize for parallel building. */ + env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); + + initTmpDir(); + + /* Compatibility hack with Nix <= 0.7: if this is a fixed-output + derivation, tell the builder, so that for instance `fetchurl' + can skip checking the output. On older Nixes, this environment + variable won't be set, so `fetchurl' will do the check. */ + if (fixedOutput) env["NIX_OUTPUT_CHECKED"] = "1"; + + /* *Only* if this is a fixed-output derivation, propagate the + values of the environment variables specified in the + `impureEnvVars' attribute to the builder. This allows for + instance environment variables for proxy configuration such as + `http_proxy' to be easily passed to downloaders like + `fetchurl'. Passing such environment variables from the caller + to the builder is generally impure, but the output of + fixed-output derivations is by definition pure (since we + already know the cryptographic hash of the output). */ + if (fixedOutput) { + for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) + env[i] = getEnv(i).value_or(""); + } + + /* Currently structured log messages piggyback on stderr, but we + may change that in the future. So tell the builder which file + descriptor to use for that. */ + env["NIX_LOG_FD"] = "2"; + + /* Trigger colored output in various tools. */ + env["TERM"] = "xterm-256color"; +} + + +static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); + + +void DerivationGoal::writeStructuredAttrs() +{ + auto structuredAttrs = parsedDrv->getStructuredAttrs(); + if (!structuredAttrs) return; + + auto json = *structuredAttrs; + + /* Add an "outputs" object containing the output paths. */ + nlohmann::json outputs; + for (auto & i : drv->outputs) + outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path), inputRewrites); + json["outputs"] = outputs; + + /* Handle exportReferencesGraph. */ + auto e = json.find("exportReferencesGraph"); + if (e != json.end() && e->is_object()) { + for (auto i = e->begin(); i != e->end(); ++i) { + std::ostringstream str; + { + JSONPlaceholder jsonRoot(str, true); + StorePathSet storePaths; + for (auto & p : *i) + storePaths.insert(worker.store.parseStorePath(p.get())); + worker.store.pathInfoToJSON(jsonRoot, + exportReferences(storePaths), false, true); + } + json[i.key()] = nlohmann::json::parse(str.str()); // urgh + } + } + + writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.json"); + + /* As a convenience to bash scripts, write a shell file that + maps all attributes that are representable in bash - + namely, strings, integers, nulls, Booleans, and arrays and + objects consisting entirely of those values. (So nested + arrays or objects are not supported.) */ + + auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { + if (value.is_string()) + return shellEscape(value); + + if (value.is_number()) { + auto f = value.get(); + if (std::ceil(f) == f) + return std::to_string(value.get()); + } + + if (value.is_null()) + return std::string("''"); + + if (value.is_boolean()) + return value.get() ? std::string("1") : std::string(""); + + return {}; + }; + + std::string jsonSh; + + for (auto i = json.begin(); i != json.end(); ++i) { + + if (!std::regex_match(i.key(), shVarName)) continue; + + auto & value = i.value(); + + auto s = handleSimpleType(value); + if (s) + jsonSh += fmt("declare %s=%s\n", i.key(), *s); + + else if (value.is_array()) { + std::string s2; + bool good = true; + + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += *s3; s2 += ' '; + } + + if (good) + jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); + } + + else if (value.is_object()) { + std::string s2; + bool good = true; + + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); + } + + if (good) + jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); + } + } + + writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.sh"); +} + + +/* A wrapper around LocalStore that only allows building/querying of + paths that are in the input closures of the build or were added via + recursive Nix calls. */ +struct RestrictedStore : public LocalFSStore +{ + ref next; + + DerivationGoal & goal; + + RestrictedStore(const Params & params, ref next, DerivationGoal & goal) + : Store(params), LocalFSStore(params), next(next), goal(goal) + { } + + Path getRealStoreDir() override + { return next->realStoreDir; } + + std::string getUri() override + { return next->getUri(); } + + StorePathSet queryAllValidPaths() override + { + StorePathSet paths; + for (auto & p : goal.inputPaths) paths.insert(p.clone()); + for (auto & p : goal.addedPaths) paths.insert(p.clone()); + return paths; + } + + void queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept override + { + if (goal.isAllowed(path)) { + try { + /* Censor impure information. */ + auto info = std::make_shared(*next->queryPathInfo(path)); + info->deriver.reset(); + info->registrationTime = 0; + info->ultimate = false; + info->sigs.clear(); + callback(info); + } catch (InvalidPath &) { + callback(nullptr); + } + } else + callback(nullptr); + }; + + void queryReferrers(const StorePath & path, StorePathSet & referrers) override + { } + + StorePathSet queryDerivationOutputs(const StorePath & path) override + { throw Error("queryDerivationOutputs"); } + + std::optional queryPathFromHashPart(const std::string & hashPart) override + { throw Error("queryPathFromHashPart"); } + + StorePath addToStore(const string & name, const Path & srcPath, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override + { throw Error("addToStore"); } + + void addToStore(const ValidPathInfo & info, Source & narSource, + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, + std::shared_ptr accessor = 0) override + { + next->addToStore(info, narSource, repair, checkSigs, accessor); + goal.addDependency(info.path); + } + + StorePath addToStoreFromDump(const string & dump, const string & name, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override + { + auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); + goal.addDependency(path); + return path; + } + + StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair = NoRepair) override + { + auto path = next->addTextToStore(name, s, references, repair); + goal.addDependency(path); + return path; + } + + void narFromPath(const StorePath & path, Sink & sink) override + { + if (!goal.isAllowed(path)) + throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); + LocalFSStore::narFromPath(path, sink); + } + + void ensurePath(const StorePath & path) override + { + if (!goal.isAllowed(path)) + throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); + /* Nothing to be done; 'path' must already be valid. */ + } + + void buildPaths(const std::vector & paths, BuildMode buildMode) override + { + if (buildMode != bmNormal) throw Error("unsupported build mode"); + + StorePathSet newPaths; + + for (auto & path : paths) { + if (path.path.isDerivation()) { + if (!goal.isAllowed(path.path)) + throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); + auto drv = derivationFromPath(path.path); + for (auto & output : drv.outputs) + if (wantOutput(output.first, path.outputs)) + newPaths.insert(output.second.path.clone()); + } else if (!goal.isAllowed(path.path)) + throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); + } + + next->buildPaths(paths, buildMode); + + StorePathSet closure; + next->computeFSClosure(newPaths, closure); + for (auto & path : closure) + goal.addDependency(path); + } + + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode = bmNormal) override + { unsupported("buildDerivation"); } + + void addTempRoot(const StorePath & path) override + { } + + void addIndirectRoot(const Path & path) override + { } + + Roots findRoots(bool censor) override + { return Roots(); } + + void collectGarbage(const GCOptions & options, GCResults & results) override + { } + + void addSignatures(const StorePath & storePath, const StringSet & sigs) override + { unsupported("addSignatures"); } + + void queryMissing(const std::vector & targets, + StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, + unsigned long long & downloadSize, unsigned long long & narSize) override + { + /* This is slightly impure since it leaks information to the + client about what paths will be built/substituted or are + already present. Probably not a big deal. */ + + std::vector allowed; + for (auto & path : targets) { + if (goal.isAllowed(path.path)) + allowed.emplace_back(path); + else + unknown.insert(path.path.clone()); + } + + next->queryMissing(allowed, willBuild, willSubstitute, + unknown, downloadSize, narSize); + } +}; + + +void DerivationGoal::startDaemon() +{ + settings.requireExperimentalFeature("recursive-nix"); + + Store::Params params; + params["path-info-cache-size"] = "0"; + params["store"] = worker.store.storeDir; + params["root"] = worker.store.rootDir; + params["state"] = "/no-such-path"; + params["log"] = "/no-such-path"; + auto store = make_ref(params, + ref(std::dynamic_pointer_cast(worker.store.shared_from_this())), + *this); + + addedPaths.clear(); + + auto socketName = ".nix-socket"; + Path socketPath = tmpDir + "/" + socketName; + env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName; + + daemonSocket = createUnixDomainSocket(socketPath, 0600); + + chownToBuilder(socketPath); + + daemonThread = std::thread([this, store]() { + + while (true) { + + /* Accept a connection. */ + struct sockaddr_un remoteAddr; + socklen_t remoteAddrLen = sizeof(remoteAddr); + + AutoCloseFD remote = accept(daemonSocket.get(), + (struct sockaddr *) &remoteAddr, &remoteAddrLen); + if (!remote) { + if (errno == EINTR) continue; + if (errno == EINVAL) break; + throw SysError("accepting connection"); + } + + closeOnExec(remote.get()); + + debug("received daemon connection"); + + auto workerThread = std::thread([store, remote{std::move(remote)}]() { + FdSource from(remote.get()); + FdSink to(remote.get()); + try { + daemon::processConnection(store, from, to, + daemon::NotTrusted, daemon::Recursive, "nobody", 65535); + debug("terminated daemon connection"); + } catch (SysError &) { + ignoreException(); + } + }); + + daemonWorkerThreads.push_back(std::move(workerThread)); + } + + debug("daemon shutting down"); + }); +} + + +void DerivationGoal::stopDaemon() +{ + if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) + throw SysError("shutting down daemon socket"); + + if (daemonThread.joinable()) + daemonThread.join(); + + // FIXME: should prune worker threads more quickly. + // FIXME: shutdown the client socket to speed up worker termination. + for (auto & thread : daemonWorkerThreads) + thread.join(); + daemonWorkerThreads.clear(); + + daemonSocket = -1; +} + + +void DerivationGoal::addDependency(const StorePath & path) +{ + if (isAllowed(path)) return; + + addedPaths.insert(path.clone()); + + /* If we're doing a sandbox build, then we have to make the path + appear in the sandbox. */ + if (useChroot) { + + debug("materialising '%s' in the sandbox", worker.store.printStorePath(path)); + + #if __linux__ + + Path source = worker.store.Store::toRealPath(path); + Path target = chrootRootDir + worker.store.printStorePath(path); + debug("bind-mounting %s -> %s", target, source); + + if (pathExists(target)) + throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); + + struct stat st; + if (lstat(source.c_str(), &st)) + throw SysError("getting attributes of path '%s'", source); + + if (S_ISDIR(st.st_mode)) { + + /* Bind-mount the path into the sandbox. This requires + entering its mount namespace, which is not possible + in multithreaded programs. So we do this in a + child process.*/ + Pid child(startProcess([&]() { + + if (setns(sandboxMountNamespace.get(), 0) == -1) + throw SysError("entering sandbox mount namespace"); + + createDirs(target); + + if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) + throw SysError("bind mount from '%s' to '%s' failed", source, target); + + _exit(0); + })); + + int status = child.wait(); + if (status != 0) + throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); + + } else + linkOrCopy(source, target); + + #else + throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", + worker.store.printStorePath(path)); + #endif + + } +} + + +void DerivationGoal::chownToBuilder(const Path & path) +{ + if (!buildUser) return; + if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) + throw SysError("cannot change ownership of '%1%'", path); +} + + +void setupSeccomp() +{ +#if __linux__ + if (!settings.filterSyscalls) return; +#if HAVE_SECCOMP + scmp_filter_ctx ctx; + + if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))) + throw SysError("unable to initialize seccomp mode 2"); + + Finally cleanup([&]() { + seccomp_release(ctx); + }); + + if (nativeSystem == "x86_64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) + throw SysError("unable to add 32-bit seccomp architecture"); + + if (nativeSystem == "x86_64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) + throw SysError("unable to add X32 seccomp architecture"); - writeFile("/proc/" + std::to_string(pid) + "/gid_map", - (format("%d %d 1") % sandboxGid % hostGid).str()); + if (nativeSystem == "aarch64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) + printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); - /* Signal the builder that we've updated its user - namespace. */ - writeFull(userNamespaceSync.writeSide.get(), "1"); - userNamespaceSync.writeSide = -1; + /* Prevent builders from creating setuid/setgid binaries. */ + for (int perm : { S_ISUID, S_ISGID }) { + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, + SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) + throw SysError("unable to add seccomp rule"); - } else -#endif - { - options.allowVfork = !buildUser.enabled() && !drv->isBuiltin(); - pid = startProcess([&]() { - runChild(); - }, options); + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1, + SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) + throw SysError("unable to add seccomp rule"); + + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1, + SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) + throw SysError("unable to add seccomp rule"); } - /* parent */ - pid.setSeparatePG(true); - builderOut.writeSide = -1; - worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true); + /* Prevent builders from creating EAs or ACLs. Not all filesystems + support these, and they're not allowed in the Nix store because + they're not representable in the NAR serialisation. */ + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) + throw SysError("unable to add seccomp rule"); - /* Check if setting up the build environment failed. */ - while (true) { - string msg = readLine(builderOut.readSide.get()); - if (string(msg, 0, 1) == "\1") { - if (msg.size() == 1) break; - throw Error(string(msg, 1)); - } - debug(msg); - } + if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0) + throw SysError("unable to set 'no new privileges' seccomp attribute"); + + if (seccomp_load(ctx) != 0) + throw SysError("unable to load seccomp BPF program"); +#else + throw Error( + "seccomp is not supported on this platform; " + "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!"); +#endif +#endif } @@ -2251,8 +3082,22 @@ void DerivationGoal::runChild() commonChildInit(builderOut); + try { + setupSeccomp(); + } catch (...) { + if (buildUser) throw; + } + bool setUser = true; + /* Make the contents of netrc available to builtin:fetchurl + (which may run under a different uid and/or in a sandbox). */ + std::string netrcData; + try { + if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") + netrcData = readFile(settings.netrcFile); + } catch (SysError &) { } + #if __linux__ if (useChroot) { @@ -2292,14 +3137,29 @@ void DerivationGoal::runChild() outside of the namespace. Making a subtree private is local to the namespace, though, so setting MS_PRIVATE does not affect the outside world. */ - if (mount(0, "/", 0, MS_REC|MS_PRIVATE, 0) == -1) { - throw SysError("unable to make ‘/’ private mount"); - } + if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) + throw SysError("unable to make '/' private"); /* Bind-mount chroot directory to itself, to treat it as a different filesystem from /, as needed for pivot_root. */ if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError(format("unable to bind mount ‘%1%’") % chrootRootDir); + throw SysError("unable to bind mount '%1%'", chrootRootDir); + + /* Bind-mount the sandbox's Nix store onto itself so that + we can mark it as a "shared" subtree, allowing bind + mounts made in *this* mount namespace to be propagated + into the child namespace created by the + unshare(CLONE_NEWNS) call below. + + Marking chrootRootDir as MS_SHARED causes pivot_root() + to fail with EINVAL. Don't know why. */ + Path chrootStoreDir = chrootRootDir + worker.store.storeDir; + + if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1) + throw SysError("unable to bind mount the Nix store", chrootStoreDir); + + if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1) + throw SysError("unable to make '%s' shared", chrootStoreDir); /* Set up a nearly empty /dev, unless the user asked to bind-mount the host /dev. */ @@ -2308,17 +3168,13 @@ void DerivationGoal::runChild() createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/pts"); ss.push_back("/dev/full"); -#ifdef __linux__ - if (pathExists("/dev/kvm")) + if (settings.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) ss.push_back("/dev/kvm"); -#endif ss.push_back("/dev/null"); ss.push_back("/dev/random"); ss.push_back("/dev/tty"); ss.push_back("/dev/urandom"); ss.push_back("/dev/zero"); - ss.push_back("/dev/ptmx"); - ss.push_back("/dev/pts"); createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd"); createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin"); createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout"); @@ -2330,28 +3186,32 @@ void DerivationGoal::runChild() on. */ if (fixedOutput) { ss.push_back("/etc/resolv.conf"); - ss.push_back("/etc/nsswitch.conf"); + + // Only use nss functions to resolve hosts and + // services. Don’t use it for anything else that may + // be configured for this system. This limits the + // potential impurities introduced in fixed-outputs. + writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); + ss.push_back("/etc/services"); ss.push_back("/etc/hosts"); - ss.push_back("/var/run/nscd/socket"); + if (pathExists("/var/run/nscd/socket")) + ss.push_back("/var/run/nscd/socket"); } - for (auto & i : ss) dirsInChroot[i] = i; + for (auto & i : ss) dirsInChroot.emplace(i, i); /* Bind-mount all the directories from the "host" filesystem that we want in the chroot environment. */ - for (auto & i : dirsInChroot) { + auto doBind = [&](const Path & source, const Path & target, bool optional = false) { + debug("bind mounting '%1%' to '%2%'", source, target); struct stat st; - Path source = i.second.source; - Path target = chrootRootDir + i.first; - if (source == "/proc") continue; // backwards compatibility - debug(format("bind mounting ‘%1%’ to ‘%2%’") % source % target); if (stat(source.c_str(), &st) == -1) { - if (i.second.optional && errno == ENOENT) - continue; + if (optional && errno == ENOENT) + return; else - throw SysError(format("getting attributes of path ‘%1%’") % source); + throw SysError("getting attributes of path '%1%'", source); } if (S_ISDIR(st.st_mode)) createDirs(target); @@ -2360,7 +3220,12 @@ void DerivationGoal::runChild() writeFile(target, ""); } if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError(format("bind mount from ‘%1%’ to ‘%2%’ failed") % source % target); + throw SysError("bind mount from '%1%' to '%2%' failed", source, target); + }; + + for (auto & i : dirsInChroot) { + if (i.second.source == "/proc") continue; // backwards compatibility + doBind(i.second.source, chrootRootDir + i.first, i.second.optional); } /* Bind a new instance of procfs on /proc. */ @@ -2371,43 +3236,57 @@ void DerivationGoal::runChild() /* Mount a new tmpfs on /dev/shm to ensure that whatever the builder puts in /dev/shm is cleaned up automatically. */ if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, - fmt("size=%s", settings.get("sandbox-dev-shm-size", std::string("50%"))).c_str()) == -1) + fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) throw SysError("mounting /dev/shm"); -#if 0 - // FIXME: can't figure out how to do this in a user - // namespace. - /* Mount a new devpts on /dev/pts. Note that this requires the kernel to be compiled with CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case if /dev/ptx/ptmx exists). */ if (pathExists("/dev/pts/ptmx") && !pathExists(chrootRootDir + "/dev/ptmx") - && dirsInChroot.find("/dev/pts") == dirsInChroot.end()) + && !dirsInChroot.count("/dev/pts")) { - if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == -1) - throw SysError("mounting /dev/pts"); - createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); + if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) + { + createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); - /* Make sure /dev/pts/ptmx is world-writable. With some - Linux versions, it is created with permissions 0. */ - chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); + /* Make sure /dev/pts/ptmx is world-writable. With some + Linux versions, it is created with permissions 0. */ + chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); + } else { + if (errno != EINVAL) + throw SysError("mounting /dev/pts"); + doBind("/dev/pts", chrootRootDir + "/dev/pts"); + doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); + } } -#endif + + /* Unshare this mount namespace. This is necessary because + pivot_root() below changes the root of the mount + namespace. This means that the call to setns() in + addDependency() would hide the host's filesystem, + making it impossible to bind-mount paths from the host + Nix store into the sandbox. Therefore, we save the + pre-pivot_root namespace in + sandboxMountNamespace. Since we made /nix/store a + shared subtree above, this allows addDependency() to + make paths appear in the sandbox. */ + if (unshare(CLONE_NEWNS) == -1) + throw SysError("unsharing mount namespace"); /* Do the chroot(). */ if (chdir(chrootRootDir.c_str()) == -1) - throw SysError(format("cannot change directory to ‘%1%’") % chrootRootDir); + throw SysError("cannot change directory to '%1%'", chrootRootDir); if (mkdir("real-root", 0) == -1) throw SysError("cannot create real-root directory"); if (pivot_root(".", "real-root") == -1) - throw SysError(format("cannot pivot old root directory onto ‘%1%’") % (chrootRootDir + "/real-root")); + throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root")); if (chroot(".") == -1) - throw SysError(format("cannot change root directory to ‘%1%’") % chrootRootDir); + throw SysError("cannot change root directory to '%1%'", chrootRootDir); if (umount2("real-root", MNT_DETACH) == -1) throw SysError("cannot unmount real root filesystem"); @@ -2428,10 +3307,10 @@ void DerivationGoal::runChild() #endif if (chdir(tmpDirInSandbox.c_str()) == -1) - throw SysError(format("changing into ‘%1%’") % tmpDir); + throw SysError("changing into '%1%'", tmpDir); /* Close all other file descriptors. */ - closeMostFDs(set()); + closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); #if __linux__ /* Change the personality to 32-bit if we're doing an @@ -2475,22 +3354,22 @@ void DerivationGoal::runChild() descriptors except std*, so that's safe. Also note that setuid() when run as root sets the real, effective and saved UIDs. */ - if (setUser && buildUser.enabled()) { + if (setUser && buildUser) { /* Preserve supplementary groups of the build user, to allow admins to specify groups such as "kvm". */ - if (!buildUser.getSupplementaryGIDs().empty() && - setgroups(buildUser.getSupplementaryGIDs().size(), - buildUser.getSupplementaryGIDs().data()) == -1) + if (!buildUser->getSupplementaryGIDs().empty() && + setgroups(buildUser->getSupplementaryGIDs().size(), + buildUser->getSupplementaryGIDs().data()) == -1) throw SysError("cannot set supplementary groups of build user"); - if (setgid(buildUser.getGID()) == -1 || - getgid() != buildUser.getGID() || - getegid() != buildUser.getGID()) + if (setgid(buildUser->getGID()) == -1 || + getgid() != buildUser->getGID() || + getegid() != buildUser->getGID()) throw SysError("setgid failed"); - if (setuid(buildUser.getUID()) == -1 || - getuid() != buildUser.getUID() || - geteuid() != buildUser.getUID()) + if (setuid(buildUser->getUID()) == -1 || + getuid() != buildUser->getUID() || + geteuid() != buildUser->getUID()) throw SysError("setuid failed"); } @@ -2499,133 +3378,168 @@ void DerivationGoal::runChild() const char *builder = "invalid"; - string sandboxProfile; if (drv->isBuiltin()) { ; + } #if __APPLE__ - } else if (useChroot) { - /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ - PathSet ancestry; + else { + /* This has to appear before import statements. */ + std::string sandboxProfile = "(version 1)\n"; + + if (useChroot) { + + /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ + PathSet ancestry; + + /* We build the ancestry before adding all inputPaths to the store because we know they'll + all have the same parents (the store), and there might be lots of inputs. This isn't + particularly efficient... I doubt it'll be a bottleneck in practice */ + for (auto & i : dirsInChroot) { + Path cur = i.first; + while (cur.compare("/") != 0) { + cur = dirOf(cur); + ancestry.insert(cur); + } + } - /* We build the ancestry before adding all inputPaths to the store because we know they'll - all have the same parents (the store), and there might be lots of inputs. This isn't - particularly efficient... I doubt it'll be a bottleneck in practice */ - for (auto & i : dirsInChroot) { - Path cur = i.first; + /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost + path component this time, since it's typically /nix/store and we care about that. */ + Path cur = worker.store.storeDir; while (cur.compare("/") != 0) { - cur = dirOf(cur); ancestry.insert(cur); + cur = dirOf(cur); } - } - - /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost - path component this time, since it's typically /nix/store and we care about that. */ - Path cur = worker.store.storeDir; - while (cur.compare("/") != 0) { - ancestry.insert(cur); - cur = dirOf(cur); - } - - /* Add all our input paths to the chroot */ - for (auto & i : inputPaths) - dirsInChroot[i] = i; - - /* This has to appear before import statements */ - sandboxProfile += "(version 1)\n"; - /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ - if (settings.get("darwin-log-sandbox-violations", false)) { - sandboxProfile += "(deny default)\n"; - } else { - sandboxProfile += "(deny default (with no-log))\n"; - } - - /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms - to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ - Path globalTmpDir = canonPath(getEnv("TMPDIR", "/tmp"), true); - - /* They don't like trailing slashes on subpath directives */ - if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); + /* Add all our input paths to the chroot */ + for (auto & i : inputPaths) { + auto p = worker.store.printStorePath(i); + dirsInChroot[p] = p; + } - /* Our rwx outputs */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : missingPaths) { - sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.c_str()).str(); - } - sandboxProfile += ")\n"; + /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ + if (settings.darwinLogSandboxViolations) { + sandboxProfile += "(deny default)\n"; + } else { + sandboxProfile += "(deny default (with no-log))\n"; + } - /* Our inputs (transitive dependencies and any impurities computed above) + sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; - without file-write* allowed, access() incorrectly returns EPERM - */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : dirsInChroot) { - if (i.first != i.second.source) - throw Error(format( - "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin") - % i.first % i.second.source); + if (fixedOutput) + sandboxProfile += "(import \"sandbox-network.sb\")\n"; - string path = i.first; - struct stat st; - if (lstat(path.c_str(), &st)) { - if (i.second.optional && errno == ENOENT) - continue; - throw SysError(format("getting attributes of path ‘%1%’") % path); + /* Our rwx outputs */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + for (auto & i : missingPaths) + sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(i)); + + /* Also add redirected outputs to the chroot */ + for (auto & i : redirectedOutputs) + sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(i.second)); + + sandboxProfile += ")\n"; + + /* Our inputs (transitive dependencies and any impurities computed above) + + without file-write* allowed, access() incorrectly returns EPERM + */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + for (auto & i : dirsInChroot) { + if (i.first != i.second.source) + throw Error( + "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", + i.first, i.second.source); + + string path = i.first; + struct stat st; + if (lstat(path.c_str(), &st)) { + if (i.second.optional && errno == ENOENT) + continue; + throw SysError("getting attributes of path '%s", path); + } + if (S_ISDIR(st.st_mode)) + sandboxProfile += fmt("\t(subpath \"%s\")\n", path); + else + sandboxProfile += fmt("\t(literal \"%s\")\n", path); } - if (S_ISDIR(st.st_mode)) - sandboxProfile += (format("\t(subpath \"%1%\")\n") % path).str(); - else - sandboxProfile += (format("\t(literal \"%1%\")\n") % path).str(); - } - sandboxProfile += ")\n"; + sandboxProfile += ")\n"; - /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ - sandboxProfile += "(allow file-read*\n"; - for (auto & i : ancestry) { - sandboxProfile += (format("\t(literal \"%1%\")\n") % i.c_str()).str(); - } - sandboxProfile += ")\n"; + /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ + sandboxProfile += "(allow file-read*\n"; + for (auto & i : ancestry) { + sandboxProfile += fmt("\t(literal \"%s\")\n", i); + } + sandboxProfile += ")\n"; - sandboxProfile += additionalSandboxProfile; + sandboxProfile += additionalSandboxProfile; + } else + sandboxProfile += "(import \"sandbox-minimal.sb\")\n"; debug("Generated sandbox profile:"); debug(sandboxProfile); - Path sandboxFile = drvPath + ".sb"; - deletePath(sandboxFile); - autoDelSandbox.reset(sandboxFile, false); + Path sandboxFile = tmpDir + "/.sandbox.sb"; writeFile(sandboxFile, sandboxProfile); - builder = "/usr/bin/sandbox-exec"; - args.push_back("sandbox-exec"); - args.push_back("-f"); - args.push_back(sandboxFile); - args.push_back("-D"); - args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); - args.push_back(drv->builder); -#endif - } else { + bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); + + /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms + to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ + Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true); + + /* They don't like trailing slashes on subpath directives */ + if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); + + if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { + builder = "/usr/bin/sandbox-exec"; + args.push_back("sandbox-exec"); + args.push_back("-f"); + args.push_back(sandboxFile); + args.push_back("-D"); + args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); + args.push_back("-D"); + args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); + if (allowLocalNetworking) { + args.push_back("-D"); + args.push_back(string("_ALLOW_LOCAL_NETWORKING=1")); + } + args.push_back(drv->builder); + } else { + builder = drv->builder.c_str(); + args.push_back(std::string(baseNameOf(drv->builder))); + } + } +#else + else { builder = drv->builder.c_str(); - string builderBasename = baseNameOf(drv->builder); - args.push_back(builderBasename); + args.push_back(std::string(baseNameOf(drv->builder))); } +#endif for (auto & i : drv->args) args.push_back(rewriteStrings(i, inputRewrites)); - restoreSIGPIPE(); - /* Indicate that we managed to set up the build environment. */ writeFull(STDERR_FILENO, string("\1\n")); /* Execute the program. This should not return. */ if (drv->isBuiltin()) { try { + logger = makeJSONLogger(*logger); + + BasicDerivation & drv2(*drv); + for (auto & e : drv2.env) + e.second = rewriteStrings(e.second, inputRewrites); + if (drv->builder == "builtin:fetchurl") - builtinFetchurl(*drv); + builtinFetchurl(drv2, netrcData); + else if (drv->builder == "builtin:buildenv") + builtinBuildenv(drv2); + else if (drv->builder == "builtin:unpack-channel") + builtinUnpackChannel(drv2); else - throw Error(format("unsupported builtin function ‘%1%’") % string(drv->builder, 8)); + throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); _exit(0); } catch (std::exception & e) { writeFull(STDERR_FILENO, "error: " + string(e.what()) + "\n"); @@ -2635,7 +3549,7 @@ void DerivationGoal::runChild() execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); - throw SysError(format("executing ‘%1%’") % drv->builder); + throw SysError("executing '%1%'", drv->builder); } catch (std::exception & e) { writeFull(STDERR_FILENO, "\1while setting up the build environment: " + string(e.what()) + "\n"); @@ -2647,22 +3561,43 @@ void DerivationGoal::runChild() /* Parse a list of reference specifiers. Each element must either be a store path, or the symbolic name of the output of the derivation (such as `out'). */ -PathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, string attr) +StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, const Strings & paths) { - PathSet result; - Paths paths = tokenizeString(attr); + StorePathSet result; for (auto & i : paths) { if (store.isStorePath(i)) - result.insert(i); - else if (drv.outputs.find(i) != drv.outputs.end()) - result.insert(drv.outputs.find(i)->second.path); - else throw BuildError( - format("derivation contains an illegal reference specifier ‘%1%’") % i); + result.insert(store.parseStorePath(i)); + else if (drv.outputs.count(i)) + result.insert(drv.outputs.find(i)->second.path.clone()); + else throw BuildError("derivation contains an illegal reference specifier '%s'", i); } return result; } +static void moveCheckToStore(const Path & src, const Path & dst) +{ + /* For the rename of directory to succeed, we must be running as root or + the directory must be made temporarily writable (to update the + directory's parent link ".."). */ + struct stat st; + if (lstat(src.c_str(), &st) == -1) { + throw SysError("getting attributes of path '%1%'", src); + } + + bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); + + if (changePerm) + chmod_(src, st.st_mode | S_IWUSR); + + if (rename(src.c_str(), dst.c_str())) + throw SysError("renaming '%1%' to '%2%'", src, dst); + + if (changePerm) + chmod_(dst, st.st_mode); +} + + void DerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output @@ -2675,7 +3610,7 @@ void DerivationGoal::registerOutputs() if (allValid) return; } - ValidPathInfos infos; + std::map infos; /* Set of inodes seen during calls to canonicalisePathMetaData() for this build's outputs. This needs to be shared between @@ -2683,18 +3618,38 @@ void DerivationGoal::registerOutputs() InodesSeen inodesSeen; Path checkSuffix = ".check"; - bool runDiffHook = settings.get("run-diff-hook", false); - bool keepPreviousRound = settings.keepFailed || runDiffHook; + bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; + + std::exception_ptr delayedException; + + /* The paths that can be referenced are the input closures, the + output paths, and any paths that have been built via recursive + Nix calls. */ + StorePathSet referenceablePaths; + for (auto & p : inputPaths) referenceablePaths.insert(p.clone()); + for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path.clone()); + for (auto & p : addedPaths) referenceablePaths.insert(p.clone()); /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all output paths read-only. */ for (auto & i : drv->outputs) { - Path path = i.second.path; - if (missingPaths.find(path) == missingPaths.end()) continue; + auto path = worker.store.printStorePath(i.second.path); + if (!missingPaths.count(i.second.path)) continue; Path actualPath = path; - if (useChroot) { + if (needsHashRewrite()) { + auto r = redirectedOutputs.find(i.second.path); + if (r != redirectedOutputs.end()) { + auto redirected = worker.store.Store::toRealPath(r->second); + if (buildMode == bmRepair + && redirectedBadOutputs.count(i.second.path) + && pathExists(redirected)) + replaceValidPath(path, redirected); + if (buildMode == bmCheck) + actualPath = redirected; + } + } else if (useChroot) { actualPath = chrootRootDir + path; if (pathExists(actualPath)) { /* Move output paths from the chroot to the Nix store. */ @@ -2702,26 +3657,18 @@ void DerivationGoal::registerOutputs() replaceValidPath(path, actualPath); else if (buildMode != bmCheck && rename(actualPath.c_str(), worker.store.toRealPath(path).c_str()) == -1) - throw SysError(format("moving build output ‘%1%’ from the sandbox to the Nix store") % path); + throw SysError("moving build output '%1%' from the sandbox to the Nix store", path); } if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path); - } else { - Path redirected = redirectedOutputs[path]; - if (buildMode == bmRepair - && redirectedBadOutputs.find(path) != redirectedBadOutputs.end() - && pathExists(redirected)) - replaceValidPath(path, redirected); - if (buildMode == bmCheck && redirected != "") - actualPath = redirected; } struct stat st; if (lstat(actualPath.c_str(), &st) == -1) { if (errno == ENOENT) throw BuildError( - format("builder for ‘%1%’ failed to produce output path ‘%2%’") - % drvPath % path); - throw SysError(format("getting attributes of path ‘%1%’") % actualPath); + "builder for '%s' failed to produce output path '%s'", + worker.store.printStorePath(drvPath), path); + throw SysError("getting attributes of path '%s'", actualPath); } #ifndef __CYGWIN__ @@ -2730,19 +3677,22 @@ void DerivationGoal::registerOutputs() build. Also, the output should be owned by the build user. */ if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || - (buildUser.enabled() && st.st_uid != buildUser.getUID())) - throw BuildError(format("suspicious ownership or permission on ‘%1%’; rejecting this build output") % path); + (buildUser && st.st_uid != buildUser->getUID())) + throw BuildError("suspicious ownership or permission on '%1%'; rejecting this build output", path); #endif /* Apply hash rewriting if necessary. */ bool rewritten = false; if (!outputRewrites.empty()) { - printError(format("warning: rewriting hashes in ‘%1%’; cross fingers") % path); + logWarning({ + .name = "Rewriting hashes", + .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", path) + }); /* Canonicalise first. This ensures that the path we're rewriting doesn't contain a hard link to /etc/shadow or something like that. */ - canonicalisePathMetaData(actualPath, buildUser.enabled() ? buildUser.getUID() : -1, inodesSeen); + canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); /* FIXME: this is in-memory. */ StringSink sink; @@ -2758,156 +3708,140 @@ void DerivationGoal::registerOutputs() /* Check that fixed-output derivations produced the right outputs (i.e., the content hash should match the specified hash). */ - if (i.second.hash != "") { + std::string ca; + + if (fixedOutput) { - bool recursive; Hash h; - i.second.parseHashInfo(recursive, h); + FileIngestionMethod outputHashMode; Hash h; + i.second.parseHashInfo(outputHashMode, h); - if (!recursive) { - /* The output path should be a regular file without - execute permission. */ + if (outputHashMode == FileIngestionMethod::Flat) { + /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( - format("output path ‘%1%’ should be a non-executable regular file") % path); + "output path '%1%' should be a non-executable regular file " + "since recursive hashing is not enabled (outputHashMode=flat)", + path); } /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ - Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath); - if (buildMode == bmHash) { - Path dest = worker.store.makeFixedOutputPath(recursive, h2, drv->env["name"]); - printError(format("build produced path ‘%1%’ with %2% hash ‘%3%’") - % dest % printHashType(h.type) % printHash16or32(h2)); + Hash h2 = outputHashMode == FileIngestionMethod::Recursive + ? hashPath(h.type, actualPath).first + : hashFile(h.type, actualPath); + + auto dest = worker.store.makeFixedOutputPath(outputHashMode, h2, i.second.path.name()); + + if (h != h2) { + + /* Throw an error after registering the path as + valid. */ + worker.hashMismatch = true; + delayedException = std::make_exception_ptr( + BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", + worker.store.printStorePath(dest), h.to_string(SRI, true), h2.to_string(SRI, true))); + + Path actualDest = worker.store.Store::toRealPath(dest); + if (worker.store.isValidPath(dest)) - return; - Path actualDest = worker.store.toRealPath(dest); + std::rethrow_exception(delayedException); + if (actualPath != actualDest) { PathLocks outputLocks({actualDest}); deletePath(actualDest); if (rename(actualPath.c_str(), actualDest.c_str()) == -1) - throw SysError(format("moving ‘%1%’ to ‘%2%’") % actualPath % dest); + throw SysError("moving '%s' to '%s'", actualPath, worker.store.printStorePath(dest)); } - path = dest; + + path = worker.store.printStorePath(dest); actualPath = actualDest; - } else { - if (h != h2) - throw BuildError( - format("output path ‘%1%’ has %2% hash ‘%3%’ when ‘%4%’ was expected") - % path % i.second.hashAlgo % printHash16or32(h2) % printHash16or32(h)); } + else + assert(worker.store.parseStorePath(path) == dest); + + ca = makeFixedOutputCA(outputHashMode, h2); } /* Get rid of all weird permissions. This also checks that all files are owned by the build user, if applicable. */ canonicalisePathMetaData(actualPath, - buildUser.enabled() && !rewritten ? buildUser.getUID() : -1, inodesSeen); + buildUser && !rewritten ? buildUser->getUID() : -1, inodesSeen); /* For this output path, find the references to other paths contained in it. Compute the SHA-256 NAR hash at the same time. The hash is stored in the database so that we can verify later on whether nobody has messed with the store. */ - Activity act(*logger, lvlTalkative, format("scanning for references inside ‘%1%’") % path); + debug("scanning for references inside '%1%'", path); HashResult hash; - PathSet references = scanForReferences(actualPath, allPaths, hash); + auto references = worker.store.parseStorePathSet(scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths), hash)); if (buildMode == bmCheck) { - if (!worker.store.isValidPath(path)) continue; - auto info = *worker.store.queryPathInfo(path); + if (!worker.store.isValidPath(worker.store.parseStorePath(path))) continue; + ValidPathInfo info(*worker.store.queryPathInfo(worker.store.parseStorePath(path))); if (hash.first != info.narHash) { - if (settings.keepFailed) { + worker.checkMismatch = true; + if (settings.runDiffHook || settings.keepFailed) { Path dst = worker.store.toRealPath(path + checkSuffix); deletePath(dst); - if (rename(actualPath.c_str(), dst.c_str())) - throw SysError(format("renaming ‘%1%’ to ‘%2%’") % actualPath % dst); - throw Error(format("derivation ‘%1%’ may not be deterministic: output ‘%2%’ differs from ‘%3%’") - % drvPath % path % dst); + moveCheckToStore(actualPath, dst); + + handleDiffHook( + buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), + path, dst, worker.store.printStorePath(drvPath), tmpDir); + + throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", + worker.store.printStorePath(drvPath), worker.store.toRealPath(path), dst); } else - throw Error(format("derivation ‘%1%’ may not be deterministic: output ‘%2%’ differs") - % drvPath % path); + throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", + worker.store.printStorePath(drvPath), worker.store.toRealPath(path)); } - /* Since we verified the build, it's now ultimately - trusted. */ + /* Since we verified the build, it's now ultimately trusted. */ if (!info.ultimate) { info.ultimate = true; worker.store.signPathInfo(info); - worker.store.registerValidPaths({info}); + ValidPathInfos infos; + infos.push_back(std::move(info)); + worker.store.registerValidPaths(infos); } continue; } - /* For debugging, print out the referenced and unreferenced - paths. */ + /* For debugging, print out the referenced and unreferenced paths. */ for (auto & i : inputPaths) { - PathSet::iterator j = references.find(i); + auto j = references.find(i); if (j == references.end()) - debug(format("unreferenced input: ‘%1%’") % i); + debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); else - debug(format("referenced input: ‘%1%’") % i); + debug("referenced input: '%1%'", worker.store.printStorePath(i)); } - /* Enforce `allowedReferences' and friends. */ - auto checkRefs = [&](const string & attrName, bool allowed, bool recursive) { - if (drv->env.find(attrName) == drv->env.end()) return; - - PathSet spec = parseReferenceSpecifiers(worker.store, *drv, get(drv->env, attrName)); - - PathSet used; - if (recursive) { - /* Our requisites are the union of the closures of our references. */ - for (auto & i : references) - /* Don't call computeFSClosure on ourselves. */ - if (path != i) - worker.store.computeFSClosure(i, used); - } else - used = references; - - PathSet badPaths; - - for (auto & i : used) - if (allowed) { - if (spec.find(i) == spec.end()) - badPaths.insert(i); - } else { - if (spec.find(i) != spec.end()) - badPaths.insert(i); - } - - if (!badPaths.empty()) { - string badPathsStr; - for (auto & i : badPaths) { - badPathsStr += "\n\t"; - badPathsStr += i; - } - throw BuildError(format("output ‘%1%’ is not allowed to refer to the following paths:%2%") % actualPath % badPathsStr); - } - }; - - checkRefs("allowedReferences", true, false); - checkRefs("allowedRequisites", true, true); - checkRefs("disallowedReferences", false, false); - checkRefs("disallowedRequisites", false, true); - if (curRound == nrRounds) { worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() - worker.markContentsGood(path); + worker.markContentsGood(worker.store.parseStorePath(path)); } - ValidPathInfo info; - info.path = path; + ValidPathInfo info(worker.store.parseStorePath(path)); info.narHash = hash.first; info.narSize = hash.second; - info.references = references; - info.deriver = drvPath; + info.references = std::move(references); + info.deriver = drvPath.clone(); info.ultimate = true; + info.ca = ca; worker.store.signPathInfo(info); - infos.push_back(info); + if (!info.references.empty()) info.ca.clear(); + + infos.emplace(i.first, std::move(info)); } if (buildMode == bmCheck) return; + /* Apply output checks. */ + checkOutputs(infos); + /* Compare the result with the previous round, and report which path is different, if any.*/ if (curRound > 1 && prevInfos != infos) { @@ -2915,45 +3849,46 @@ void DerivationGoal::registerOutputs() for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) if (!(*i == *j)) { result.isNonDeterministic = true; - Path prev = i->path + checkSuffix; + Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; bool prevExists = keepPreviousRound && pathExists(prev); - auto msg = prevExists - ? fmt("output ‘%1%’ of ‘%2%’ differs from ‘%3%’ from previous round", i->path, drvPath, prev) - : fmt("output ‘%1%’ of ‘%2%’ differs from previous round", i->path, drvPath); - - auto diffHook = settings.get("diff-hook", std::string("")); - if (prevExists && diffHook != "" && runDiffHook) { - try { - auto diff = runProgram(diffHook, true, {prev, i->path}); - if (diff != "") - printError(chomp(diff)); - } catch (Error & error) { - printError("diff hook execution failed: %s", error.what()); - } - } + hintformat hint = prevExists + ? hintfmt("output '%s' of '%s' differs from '%s' from previous round", + worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev) + : hintfmt("output '%s' of '%s' differs from previous round", + worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath)); + + handleDiffHook( + buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), + prev, worker.store.printStorePath(i->second.path), + worker.store.printStorePath(drvPath), tmpDir); + + if (settings.enforceDeterminism) + throw NotDeterministic(hint); + + logError({ + .name = "Output determinism error", + .hint = hint + }); - if (settings.get("enforce-determinism", true)) - throw NotDeterministic(msg); - - printError(msg); curRound = nrRounds; // we know enough, bail out early } } - /* If this is the first round of several, then move the output out - of the way. */ + /* If this is the first round of several, then move the output out of the way. */ if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { for (auto & i : drv->outputs) { - Path prev = i.second.path + checkSuffix; + auto path = worker.store.printStorePath(i.second.path); + Path prev = path + checkSuffix; deletePath(prev); - Path dst = i.second.path + checkSuffix; - if (rename(i.second.path.c_str(), dst.c_str())) - throw SysError(format("renaming ‘%1%’ to ‘%2%’") % i.second.path % dst); + Path dst = path + checkSuffix; + if (rename(path.c_str(), dst.c_str())) + throw SysError("renaming '%s' to '%s'", path, dst); } } if (curRound < nrRounds) { - prevInfos = infos; + prevInfos = std::move(infos); return; } @@ -2961,7 +3896,7 @@ void DerivationGoal::registerOutputs() if the result was not determistic? */ if (curRound == nrRounds) { for (auto & i : drv->outputs) { - Path prev = i.second.path + checkSuffix; + Path prev = worker.store.printStorePath(i.second.path) + checkSuffix; deletePath(prev); } } @@ -2969,11 +3904,172 @@ void DerivationGoal::registerOutputs() /* Register each output path as valid, and register the sets of paths referenced by each of them. If there are cycles in the outputs, this will fail. */ - worker.store.registerValidPaths(infos); + { + ValidPathInfos infos2; + for (auto & i : infos) infos2.push_back(i.second); + worker.store.registerValidPaths(infos2); + } + + /* In case of a fixed-output derivation hash mismatch, throw an + exception now that we have registered the output as valid. */ + if (delayedException) + std::rethrow_exception(delayedException); } -string drvsLogDir = "drvs"; +void DerivationGoal::checkOutputs(const std::map & outputs) +{ + std::map outputsByPath; + for (auto & output : outputs) + outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second); + + for (auto & output : outputs) { + auto & outputName = output.first; + auto & info = output.second; + + struct Checks + { + bool ignoreSelfRefs = false; + std::optional maxSize, maxClosureSize; + std::optional allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; + }; + + /* Compute the closure and closure size of some output. This + is slightly tricky because some of its references (namely + other outputs) may not be valid yet. */ + auto getClosure = [&](const StorePath & path) + { + uint64_t closureSize = 0; + StorePathSet pathsDone; + std::queue pathsLeft; + pathsLeft.push(path.clone()); + + while (!pathsLeft.empty()) { + auto path = pathsLeft.front().clone(); + pathsLeft.pop(); + if (!pathsDone.insert(path.clone()).second) continue; + + auto i = outputsByPath.find(worker.store.printStorePath(path)); + if (i != outputsByPath.end()) { + closureSize += i->second.narSize; + for (auto & ref : i->second.references) + pathsLeft.push(ref.clone()); + } else { + auto info = worker.store.queryPathInfo(path); + closureSize += info->narSize; + for (auto & ref : info->references) + pathsLeft.push(ref.clone()); + } + } + + return std::make_pair(std::move(pathsDone), closureSize); + }; + + auto applyChecks = [&](const Checks & checks) + { + if (checks.maxSize && info.narSize > *checks.maxSize) + throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", + worker.store.printStorePath(info.path), info.narSize, *checks.maxSize); + + if (checks.maxClosureSize) { + uint64_t closureSize = getClosure(info.path).second; + if (closureSize > *checks.maxClosureSize) + throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", + worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize); + } + + auto checkRefs = [&](const std::optional & value, bool allowed, bool recursive) + { + if (!value) return; + + auto spec = parseReferenceSpecifiers(worker.store, *drv, *value); + + auto used = recursive + ? cloneStorePathSet(getClosure(info.path).first) + : cloneStorePathSet(info.references); + + if (recursive && checks.ignoreSelfRefs) + used.erase(info.path); + + StorePathSet badPaths; + + for (auto & i : used) + if (allowed) { + if (!spec.count(i)) + badPaths.insert(i.clone()); + } else { + if (spec.count(i)) + badPaths.insert(i.clone()); + } + + if (!badPaths.empty()) { + string badPathsStr; + for (auto & i : badPaths) { + badPathsStr += "\n "; + badPathsStr += worker.store.printStorePath(i); + } + throw BuildError("output '%s' is not allowed to refer to the following paths:%s", + worker.store.printStorePath(info.path), badPathsStr); + } + }; + + checkRefs(checks.allowedReferences, true, false); + checkRefs(checks.allowedRequisites, true, true); + checkRefs(checks.disallowedReferences, false, false); + checkRefs(checks.disallowedRequisites, false, true); + }; + + if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { + auto outputChecks = structuredAttrs->find("outputChecks"); + if (outputChecks != structuredAttrs->end()) { + auto output = outputChecks->find(outputName); + + if (output != outputChecks->end()) { + Checks checks; + + auto maxSize = output->find("maxSize"); + if (maxSize != output->end()) + checks.maxSize = maxSize->get(); + + auto maxClosureSize = output->find("maxClosureSize"); + if (maxClosureSize != output->end()) + checks.maxClosureSize = maxClosureSize->get(); + + auto get = [&](const std::string & name) -> std::optional { + auto i = output->find(name); + if (i != output->end()) { + Strings res; + for (auto j = i->begin(); j != i->end(); ++j) { + if (!j->is_string()) + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath)); + res.push_back(j->get()); + } + checks.disallowedRequisites = res; + return res; + } + return {}; + }; + + checks.allowedReferences = get("allowedReferences"); + checks.allowedRequisites = get("allowedRequisites"); + checks.disallowedReferences = get("disallowedReferences"); + checks.disallowedRequisites = get("disallowedRequisites"); + + applyChecks(checks); + } + } + } else { + // legacy non-structured-attributes case + Checks checks; + checks.ignoreSelfRefs = true; + checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences"); + checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites"); + checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences"); + checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites"); + applyChecks(checks); + } + } +} Path DerivationGoal::openLogFile() @@ -2982,19 +4078,17 @@ Path DerivationGoal::openLogFile() if (!settings.keepLog) return ""; - string baseName = baseNameOf(drvPath); + auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath))); /* Create a log file. */ - Path dir = (format("%1%/%2%/%3%/") % worker.store.logDir % drvsLogDir % string(baseName, 0, 2)).str(); + Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2)); createDirs(dir); - Path logFileName = (format("%1%/%2%%3%") - % dir - % string(baseName, 2) - % (settings.compressLog ? ".bz2" : "")).str(); + Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), + settings.compressLog ? ".bz2" : ""); fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); - if (!fdLogFile) throw SysError(format("creating log file ‘%1%’") % logFileName); + if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName); logFileSink = std::make_shared(fdLogFile.get()); @@ -3020,10 +4114,10 @@ void DerivationGoal::closeLogFile() void DerivationGoal::deleteTmpDir(bool force) { if (tmpDir != "") { - if (settings.keepFailed && !force) { - printError( - format("note: keeping build directory ‘%2%’") - % drvPath % tmpDir); + /* Don't keep temporary directories for builtins because they + might have privileged stuff (like a copy of netrc). */ + if (settings.keepFailed && !force && !drv->isBuiltin()) { + printError("note: keeping build directory '%s'", tmpDir); chmod(tmpDir.c_str(), 0755); } else @@ -3040,11 +4134,11 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) { logSize += data.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { - printError( - format("%1% killed after writing more than %2% bytes of log output") - % getName() % settings.maxLogSize); killChild(); - done(BuildResult::LogLimitExceeded); + done( + BuildResult::LogLimitExceeded, + Error("%s killed after writing more than %d bytes of log output", + getName(), settings.maxLogSize)); return; } @@ -3062,8 +4156,14 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) if (logSink) (*logSink)(data); } - if (hook && fd == hook->fromHook.readSide.get()) - printError(data); // FIXME? + if (hook && fd == hook->fromHook.readSide.get()) { + for (auto c : data) + if (c == '\n') { + handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true); + currentHookLine.clear(); + } else + currentHookLine += c; + } } @@ -3076,55 +4176,72 @@ void DerivationGoal::handleEOF(int fd) void DerivationGoal::flushLine() { - if (settings.verboseBuild && - (settings.printRepeatedBuilds || curRound == 1)) - printError(filterANSIEscapes(currentLogLine, true)); + if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) + ; + else { logTail.push_back(currentLogLine); if (logTail.size() > settings.logLines) logTail.pop_front(); + + act->result(resBuildLogLine, currentLogLine); } + currentLogLine = ""; currentLogLinePos = 0; } -PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) +StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) { - PathSet result; + StorePathSet result; for (auto & i : drv->outputs) { if (!wantOutput(i.first, wantedOutputs)) continue; bool good = worker.store.isValidPath(i.second.path) && (!checkHash || worker.pathContentsGood(i.second.path)); - if (good == returnValid) result.insert(i.second.path); + if (good == returnValid) result.insert(i.second.path.clone()); } return result; } -Path DerivationGoal::addHashRewrite(const Path & path) +void DerivationGoal::addHashRewrite(const StorePath & path) { - string h1 = string(path, worker.store.storeDir.size() + 1, 32); - string h2 = string(printHash32(hashString(htSHA256, "rewrite:" + drvPath + ":" + path)), 0, 32); - Path p = worker.store.storeDir + "/" + h2 + string(path, worker.store.storeDir.size() + 33); - deletePath(p); - assert(path.size() == p.size()); + auto h1 = std::string(((std::string_view) path.to_string()).substr(0, 32)); + auto p = worker.store.makeStorePath( + "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), + Hash(htSHA256), path.name()); + auto h2 = std::string(((std::string_view) p.to_string()).substr(0, 32)); + deletePath(worker.store.printStorePath(p)); inputRewrites[h1] = h2; outputRewrites[h2] = h1; - redirectedOutputs[path] = p; - return p; + redirectedOutputs.insert_or_assign(path.clone(), std::move(p)); } -void DerivationGoal::done(BuildResult::Status status, const string & msg) +void DerivationGoal::done(BuildResult::Status status, std::optional ex) { result.status = status; - result.errorMsg = msg; - amDone(result.success() ? ecSuccess : ecFailed); + if (ex) + result.errorMsg = ex->what(); + amDone(result.success() ? ecSuccess : ecFailed, ex); if (result.status == BuildResult::TimedOut) worker.timedOut = true; if (result.status == BuildResult::PermanentFailure) worker.permanentFailure = true; + + mcExpectedBuilds.reset(); + mcRunningBuilds.reset(); + + if (result.success()) { + if (status == BuildResult::Built) + worker.doneBuilds++; + } else { + if (status != BuildResult::DependencyFailed) + worker.failedBuilds++; + } + + worker.updateProgress(); } @@ -3137,7 +4254,7 @@ class SubstitutionGoal : public Goal private: /* The store path that should be realised through a substitute. */ - Path storePath; + StorePath storePath; /* The remaining substituters. */ std::list> subs; @@ -3145,8 +4262,8 @@ class SubstitutionGoal : public Goal /* The current substituter. */ std::shared_ptr sub; - /* Whether any substituter can realise this path. */ - bool hasSubstitute; + /* Whether a substituter failed. */ + bool substituterFailed = false; /* Path info returned by the substituter's query info operation. */ std::shared_ptr info; @@ -3160,29 +4277,32 @@ class SubstitutionGoal : public Goal std::promise promise; /* Whether to try to repair a valid path. */ - bool repair; + RepairFlag repair; /* Location where we're downloading the substitute. Differs from storePath when doing a repair. */ Path destPath; + std::unique_ptr> maintainExpectedSubstitutions, + maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; + typedef void (SubstitutionGoal::*GoalState)(); GoalState state; public: - SubstitutionGoal(const Path & storePath, Worker & worker, bool repair = false); + SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair = NoRepair); ~SubstitutionGoal(); - void timedOut() { abort(); }; + void timedOut(Error && ex) override { abort(); }; - string key() + string key() override { /* "a$" ensures substitution goals happen before derivation goals. */ - return "a$" + storePathToName(storePath) + "$" + storePath; + return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); } - void work(); + void work() override; /* The states. */ void init(); @@ -3193,22 +4313,22 @@ class SubstitutionGoal : public Goal void finished(); /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data); - void handleEOF(int fd); + void handleChildOutput(int fd, const string & data) override; + void handleEOF(int fd) override; - Path getStorePath() { return storePath; } + StorePath getStorePath() { return storePath.clone(); } }; -SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, bool repair) +SubstitutionGoal::SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair) : Goal(worker) - , hasSubstitute(false) + , storePath(std::move(storePath)) , repair(repair) { - this->storePath = storePath; state = &SubstitutionGoal::init; - name = (format("substitution of ‘%1%’") % storePath).str(); + name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); trace("created"); + maintainExpectedSubstitutions = std::make_unique>(worker.expectedSubstitutions); } @@ -3245,7 +4365,7 @@ void SubstitutionGoal::init() } if (settings.readOnlyMode) - throw Error(format("cannot substitute path ‘%1%’ - no write access to the Nix store") % storePath); + throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath)); subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); @@ -3260,12 +4380,18 @@ void SubstitutionGoal::tryNext() if (subs.size() == 0) { /* None left. Terminate this goal and let someone else deal with it. */ - debug(format("path ‘%1%’ is required, but there is no substituter that can build it") % storePath); + debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)); /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ - amDone(hasSubstitute ? ecFailed : ecNoSubstituters); + amDone(substituterFailed ? ecFailed : ecNoSubstituters); + + if (substituterFailed) { + worker.failedSubstitutions++; + worker.updateProgress(); + } + return; } @@ -3283,16 +4409,45 @@ void SubstitutionGoal::tryNext() } catch (InvalidPath &) { tryNext(); return; + } catch (SubstituterDisabled &) { + if (settings.tryFallback) { + tryNext(); + return; + } + throw; + } catch (Error & e) { + if (settings.tryFallback) { + logError(e.info()); + tryNext(); + return; + } + throw; } - hasSubstitute = true; + /* Update the total expected download size. */ + auto narInfo = std::dynamic_pointer_cast(info); + + maintainExpectedNar = std::make_unique>(worker.expectedNarSize, info->narSize); + + maintainExpectedDownload = + narInfo && narInfo->fileSize + ? std::make_unique>(worker.expectedDownloadSize, narInfo->fileSize) + : nullptr; + + worker.updateProgress(); /* Bail out early if this substituter lacks a valid signature. LocalStore::addToStore() also checks for this, but only after we've downloaded the path. */ - if (worker.store.requireSigs && !info->checkSignatures(worker.store, worker.store.publicKeys)) { - printInfo(format("warning: substituter ‘%s’ does not have a valid signature for path ‘%s’") - % sub->getUri() % storePath); + if (worker.store.requireSigs + && !sub->isTrusted + && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) + { + logWarning({ + .name = "Invalid path signature", + .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'", + sub->getUri(), worker.store.printStorePath(storePath)) + }); tryNext(); return; } @@ -3315,7 +4470,7 @@ void SubstitutionGoal::referencesValid() trace("all references realised"); if (nrFailed > 0) { - debug(format("some references of path ‘%1%’ could not be realised") % storePath); + debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)); amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); return; } @@ -3334,15 +4489,16 @@ void SubstitutionGoal::tryToRun() trace("trying to run"); /* Make sure that we are allowed to start a build. Note that even - is maxBuildJobs == 0 (no local builds allowed), we still allow + if maxBuildJobs == 0 (no local builds allowed), we still allow a substituter to run. This is because substitutions cannot be distributed to another machine via the build hook. */ - if (worker.getNrLocalBuilds() >= (settings.maxBuildJobs == 0 ? 1 : settings.maxBuildJobs)) { + if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) { worker.waitForBuildSlot(shared_from_this()); return; } - printInfo(format("fetching path ‘%1%’...") % storePath); + maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); + worker.updateProgress(); outPipe.create(); @@ -3353,8 +4509,11 @@ void SubstitutionGoal::tryToRun() /* Wake up the worker loop when we're done. */ Finally updateStats([this]() { outPipe.writeSide = -1; }); + Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); + PushActivity pact(act.id); + copyStorePath(ref(sub), ref(worker.store.shared_from_this()), - storePath, repair); + storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { @@ -3377,8 +4536,19 @@ void SubstitutionGoal::finished() try { promise.get_future().get(); - } catch (Error & e) { - printInfo(e.msg()); + } catch (std::exception & e) { + printError(e.what()); + + /* Cause the parent build to fail unless --fallback is given, + or the substitute has disappeared. The latter case behaves + the same as the substitute never having existed in the + first place. */ + try { + throw; + } catch (SubstituteGone &) { + } catch (...) { + substituterFailed = true; + } /* Try the next substitute. */ state = &SubstitutionGoal::tryNext; @@ -3386,10 +4556,25 @@ void SubstitutionGoal::finished() return; } - worker.markContentsGood(storePath); + worker.markContentsGood(storePath.clone()); + + printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath)); + + maintainRunningSubstitutions.reset(); + + maintainExpectedSubstitutions.reset(); + worker.doneSubstitutions++; + + if (maintainExpectedDownload) { + auto fileSize = maintainExpectedDownload->delta; + maintainExpectedDownload.reset(); + worker.doneDownloadSize += fileSize; + } + + worker.doneNarSize += maintainExpectedNar->delta; + maintainExpectedNar.reset(); - printMsg(lvlChatty, - format("substitution of path ‘%1%’ succeeded") % storePath); + worker.updateProgress(); amDone(ecSuccess); } @@ -3405,45 +4590,46 @@ void SubstitutionGoal::handleEOF(int fd) if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); } - ////////////////////////////////////////////////////////////////////// -static bool working = false; - - Worker::Worker(LocalStore & store) - : store(store) + : act(*logger, actRealise) + , actDerivations(*logger, actBuilds) + , actSubstitutions(*logger, actCopyPaths) + , store(store) { /* Debugging: prevent recursive workers. */ - if (working) abort(); - working = true; nrLocalBuilds = 0; lastWokenUp = steady_time_point::min(); permanentFailure = false; timedOut = false; + hashMismatch = false; + checkMismatch = false; } Worker::~Worker() { - working = false; - /* Explicitly get rid of all strong pointers now. After this all goals that refer to this worker should be gone. (Otherwise we are in trouble, since goals may call childTerminated() etc. in their destructors). */ topGoals.clear(); + + assert(expectedSubstitutions == 0); + assert(expectedDownloadSize == 0); + assert(expectedNarSize == 0); } -GoalPtr Worker::makeDerivationGoal(const Path & path, +GoalPtr Worker::makeDerivationGoal(const StorePath & path, const StringSet & wantedOutputs, BuildMode buildMode) { - GoalPtr goal = derivationGoals[path].lock(); + GoalPtr goal = derivationGoals[path.clone()].lock(); // FIXME if (!goal) { - goal = std::make_shared(path, wantedOutputs, *this, buildMode); - derivationGoals[path] = goal; + goal = std::make_shared(path.clone(), wantedOutputs, *this, buildMode); + derivationGoals.insert_or_assign(path.clone(), goal); wakeUp(goal); } else (dynamic_cast(goal.get()))->addWantedOutputs(wantedOutputs); @@ -3451,21 +4637,21 @@ GoalPtr Worker::makeDerivationGoal(const Path & path, } -std::shared_ptr Worker::makeBasicDerivationGoal(const Path & drvPath, +std::shared_ptr Worker::makeBasicDerivationGoal(StorePath && drvPath, const BasicDerivation & drv, BuildMode buildMode) { - auto goal = std::make_shared(drvPath, drv, *this, buildMode); + auto goal = std::make_shared(std::move(drvPath), drv, *this, buildMode); wakeUp(goal); return goal; } -GoalPtr Worker::makeSubstitutionGoal(const Path & path, bool repair) +GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair) { - GoalPtr goal = substitutionGoals[path].lock(); + GoalPtr goal = substitutionGoals[path.clone()].lock(); // FIXME if (!goal) { - goal = std::make_shared(path, *this, repair); - substitutionGoals[path] = goal; + goal = std::make_shared(path.clone(), *this, repair); + substitutionGoals.insert_or_assign(path.clone(), goal); wakeUp(goal); } return goal; @@ -3494,7 +4680,7 @@ void Worker::removeGoal(GoalPtr goal) topGoals.erase(goal); /* If a top-level goal failed, then kill all other goals (unless keepGoing was set). */ - if (goal->getExitCode() == Goal::ecFailed && !settings.keepGoing) + if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) topGoals.clear(); } @@ -3590,12 +4776,14 @@ void Worker::run(const Goals & _topGoals) { for (auto & i : _topGoals) topGoals.insert(i); - Activity act(*logger, lvlDebug, "entered goal loop"); + debug("entered goal loop"); while (1) { checkInterrupt(); + store.autoGC(false); + /* Call every wake goal (in the ordering established by CompareGoalPtrs). */ while (!awake.empty() && !topGoals.empty()) { @@ -3618,9 +4806,9 @@ void Worker::run(const Goals & _topGoals) if (!children.empty() || !waitingForAWhile.empty()) waitForInput(); else { - if (awake.empty() && settings.maxBuildJobs == 0) throw Error( - "unable to start any build; either increase ‘--max-jobs’ " - "or enable distributed builds"); + if (awake.empty() && 0 == settings.maxBuildJobs) + throw Error("unable to start any build; either increase '--max-jobs' " + "or enable remote builds"); assert(!awake.empty()); } } @@ -3633,7 +4821,6 @@ void Worker::run(const Goals & _topGoals) assert(!settings.keepGoing || children.empty()); } - void Worker::waitForInput() { printMsg(lvlVomit, "waiting for children"); @@ -3645,23 +4832,25 @@ void Worker::waitForInput() terminated. */ bool useTimeout = false; - struct timeval timeout; - timeout.tv_usec = 0; + long timeout = 0; auto before = steady_time_point::clock::now(); /* If we're monitoring for silence on stdout/stderr, or if there is a build timeout, then wait for input until the first deadline for any child. */ auto nearest = steady_time_point::max(); // nearest deadline + if (settings.minFree.get() != 0) + // Periodicallty wake up to see if we need to run the garbage collector. + nearest = before + std::chrono::seconds(10); for (auto & i : children) { if (!i.respectTimeouts) continue; - if (settings.maxSilentTime != 0) + if (0 != settings.maxSilentTime) nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime)); - if (settings.buildTimeout != 0) + if (0 != settings.buildTimeout) nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); } if (nearest != steady_time_point::max()) { - timeout.tv_sec = std::max(1L, (long) std::chrono::duration_cast(nearest - before).count()); + timeout = std::max(1L, (long) std::chrono::duration_cast(nearest - before).count()); useTimeout = true; } @@ -3669,31 +4858,29 @@ void Worker::waitForInput() up after a few seconds at most. */ if (!waitingForAWhile.empty()) { useTimeout = true; - if (lastWokenUp == steady_time_point::min()) - printError("waiting for locks or build slots..."); if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; - timeout.tv_sec = std::max(1L, + timeout = std::max(1L, (long) std::chrono::duration_cast( lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count()); } else lastWokenUp = steady_time_point::min(); if (useTimeout) - vomit("sleeping %d seconds", timeout.tv_sec); + vomit("sleeping %d seconds", timeout); /* Use select() to wait for the input side of any logger pipe to become `available'. Note that `available' (i.e., non-blocking) includes EOF. */ - fd_set fds; - FD_ZERO(&fds); - int fdMax = 0; + std::vector pollStatus; + std::map fdToPollStatus; for (auto & i : children) { for (auto & j : i.fds) { - FD_SET(j, &fds); - if (j >= fdMax) fdMax = j + 1; + pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); + fdToPollStatus[j] = pollStatus.size() - 1; } } - if (select(fdMax, &fds, 0, 0, useTimeout ? &timeout : 0) == -1) { + if (poll(pollStatus.data(), pollStatus.size(), + useTimeout ? timeout * 1000 : -1) == -1) { if (errno == EINTR) return; throw SysError("waiting for input"); } @@ -3712,48 +4899,47 @@ void Worker::waitForInput() assert(goal); set fds2(j->fds); + std::vector buffer(4096); for (auto & k : fds2) { - if (FD_ISSET(k, &fds)) { - unsigned char buffer[4096]; - ssize_t rd = read(k, buffer, sizeof(buffer)); - if (rd == -1) { - if (errno != EINTR) - throw SysError(format("reading from %1%") - % goal->getName()); - } else if (rd == 0) { - debug(format("%1%: got EOF") % goal->getName()); + if (pollStatus.at(fdToPollStatus.at(k)).revents) { + ssize_t rd = read(k, buffer.data(), buffer.size()); + // FIXME: is there a cleaner way to handle pt close + // than EIO? Is this even standard? + if (rd == 0 || (rd == -1 && errno == EIO)) { + debug("%1%: got EOF", goal->getName()); goal->handleEOF(k); j->fds.erase(k); + } else if (rd == -1) { + if (errno != EINTR) + throw SysError("%s: read failed", goal->getName()); } else { - printMsg(lvlVomit, format("%1%: read %2% bytes") - % goal->getName() % rd); - string data((char *) buffer, rd); + printMsg(lvlVomit, "%1%: read %2% bytes", + goal->getName(), rd); + string data((char *) buffer.data(), rd); j->lastOutput = after; goal->handleChildOutput(k, data); } } } - if (goal->getExitCode() == Goal::ecBusy && - settings.maxSilentTime != 0 && + if (goal->exitCode == Goal::ecBusy && + 0 != settings.maxSilentTime && j->respectTimeouts && after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) { - printError( - format("%1% timed out after %2% seconds of silence") - % goal->getName() % settings.maxSilentTime); - goal->timedOut(); + goal->timedOut(Error( + "%1% timed out after %2% seconds of silence", + goal->getName(), settings.maxSilentTime)); } - else if (goal->getExitCode() == Goal::ecBusy && - settings.buildTimeout != 0 && + else if (goal->exitCode == Goal::ecBusy && + 0 != settings.buildTimeout && j->respectTimeouts && after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) { - printError( - format("%1% timed out after %2% seconds") - % goal->getName() % settings.buildTimeout); - goal->timedOut(); + goal->timedOut(Error( + "%1% timed out after %2% seconds", + goal->getName(), settings.buildTimeout)); } } @@ -3770,73 +4956,124 @@ void Worker::waitForInput() unsigned int Worker::exitStatus() { - return timedOut ? 101 : (permanentFailure ? 100 : 1); + /* + * 1100100 + * ^^^^ + * |||`- timeout + * ||`-- output hash mismatch + * |`--- build failure + * `---- not deterministic + */ + unsigned int mask = 0; + bool buildFailure = permanentFailure || timedOut || hashMismatch; + if (buildFailure) + mask |= 0x04; // 100 + if (timedOut) + mask |= 0x01; // 101 + if (hashMismatch) + mask |= 0x02; // 102 + if (checkMismatch) { + mask |= 0x08; // 104 + } + + if (mask) + mask |= 0x60; + return mask ? mask : 1; } -bool Worker::pathContentsGood(const Path & path) +bool Worker::pathContentsGood(const StorePath & path) { - std::map::iterator i = pathContentsGoodCache.find(path); + auto i = pathContentsGoodCache.find(path); if (i != pathContentsGoodCache.end()) return i->second; - printInfo(format("checking path ‘%1%’...") % path); + printInfo("checking path '%s'...", store.printStorePath(path)); auto info = store.queryPathInfo(path); bool res; - if (!pathExists(path)) + if (!pathExists(store.printStorePath(path))) res = false; else { - HashResult current = hashPath(info->narHash.type, path); + HashResult current = hashPath(info->narHash.type, store.printStorePath(path)); Hash nullHash(htSHA256); res = info->narHash == nullHash || info->narHash == current.first; } - pathContentsGoodCache[path] = res; - if (!res) printError(format("path ‘%1%’ is corrupted or missing!") % path); + pathContentsGoodCache.insert_or_assign(path.clone(), res); + if (!res) + logError({ + .name = "Corrupted path", + .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path)) + }); return res; } -void Worker::markContentsGood(const Path & path) +void Worker::markContentsGood(StorePath && path) { - pathContentsGoodCache[path] = true; + pathContentsGoodCache.insert_or_assign(std::move(path), true); } ////////////////////////////////////////////////////////////////////// -void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) +static void primeCache(Store & store, const std::vector & paths) +{ + StorePathSet willBuild, willSubstitute, unknown; + unsigned long long downloadSize, narSize; + store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); + + if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) + throw Error( + "%d derivations need to be built, but neither local builds ('--max-jobs') " + "nor remote builds ('--builders') are enabled", willBuild.size()); +} + + +void LocalStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) { Worker worker(*this); + primeCache(*this, drvPaths); + Goals goals; - for (auto & i : drvPaths) { - DrvPathWithOutputs i2 = parseDrvPathWithOutputs(i); - if (isDerivation(i2.first)) - goals.insert(worker.makeDerivationGoal(i2.first, i2.second, buildMode)); + for (auto & path : drvPaths) { + if (path.path.isDerivation()) + goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode)); else - goals.insert(worker.makeSubstitutionGoal(i, buildMode)); + goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair)); } worker.run(goals); - PathSet failed; + StorePathSet failed; + std::optional ex; for (auto & i : goals) { - if (i->getExitCode() != Goal::ecSuccess) { + if (i->ex) { + if (ex) + logError(i->ex->info()); + else + ex = i->ex; + } + if (i->exitCode != Goal::ecSuccess) { DerivationGoal * i2 = dynamic_cast(i.get()); if (i2) failed.insert(i2->getDrvPath()); else failed.insert(dynamic_cast(i.get())->getStorePath()); } } - if (!failed.empty()) + if (failed.size() == 1 && ex) { + ex->status = worker.exitStatus(); + throw *ex; + } else if (!failed.empty()) { + if (ex) logError(ex->info()); throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); + } } - -BuildResult LocalStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv, +BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { Worker worker(*this); - auto goal = worker.makeBasicDerivationGoal(drvPath, drv, buildMode); + auto goal = worker.makeBasicDerivationGoal(drvPath.clone(), drv, buildMode); BuildResult result; @@ -3852,40 +5089,47 @@ BuildResult LocalStore::buildDerivation(const Path & drvPath, const BasicDerivat } -void LocalStore::ensurePath(const Path & path) +void LocalStore::ensurePath(const StorePath & path) { /* If the path is already valid, we're done. */ if (isValidPath(path)) return; + primeCache(*this, {StorePathWithOutputs(path)}); + Worker worker(*this); GoalPtr goal = worker.makeSubstitutionGoal(path); Goals goals = {goal}; worker.run(goals); - if (goal->getExitCode() != Goal::ecSuccess) - throw Error(worker.exitStatus(), "path ‘%s’ does not exist and cannot be created", path); + if (goal->exitCode != Goal::ecSuccess) { + if (goal->ex) { + goal->ex->status = worker.exitStatus(); + throw *goal->ex; + } else + throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); + } } -void LocalStore::repairPath(const Path & path) +void LocalStore::repairPath(const StorePath & path) { Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path, true); + GoalPtr goal = worker.makeSubstitutionGoal(path, Repair); Goals goals = {goal}; worker.run(goals); - if (goal->getExitCode() != Goal::ecSuccess) { + if (goal->exitCode != Goal::ecSuccess) { /* Since substituting the path didn't work, if we have a valid deriver, then rebuild the deriver. */ - auto deriver = queryPathInfo(path)->deriver; - if (deriver != "" && isValidPath(deriver)) { + auto info = queryPathInfo(path); + if (info->deriver && isValidPath(*info->deriver)) { goals.clear(); - goals.insert(worker.makeDerivationGoal(deriver, StringSet(), bmRepair)); + goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair)); worker.run(goals); } else - throw Error(worker.exitStatus(), "cannot repair path ‘%s’", path); + throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path)); } } diff --git a/src/libstore/builtins.cc b/src/libstore/builtins.cc deleted file mode 100644 index a30f30906f0..00000000000 --- a/src/libstore/builtins.cc +++ /dev/null @@ -1,63 +0,0 @@ -#include "builtins.hh" -#include "download.hh" -#include "store-api.hh" -#include "archive.hh" -#include "compression.hh" - -namespace nix { - -void builtinFetchurl(const BasicDerivation & drv) -{ - auto getAttr = [&](const string & name) { - auto i = drv.env.find(name); - if (i == drv.env.end()) throw Error(format("attribute ‘%s’ missing") % name); - return i->second; - }; - - auto fetch = [&](const string & url) { - /* No need to do TLS verification, because we check the hash of - the result anyway. */ - DownloadRequest request(url); - request.verifyTLS = false; - - /* Show a progress indicator, even though stderr is not a tty. */ - request.showProgress = DownloadRequest::yes; - - /* Note: have to use a fresh downloader here because we're in - a forked process. */ - auto data = makeDownloader()->download(request); - assert(data.data); - - return data.data; - }; - - std::shared_ptr data; - - try { - if (getAttr("outputHashMode") == "flat") - data = fetch("http://tarballs.nixos.org/" + getAttr("outputHashAlgo") + "/" + getAttr("outputHash")); - } catch (Error & e) { - debug(e.what()); - } - - if (!data) data = fetch(getAttr("url")); - - Path storePath = getAttr("out"); - - auto unpack = drv.env.find("unpack"); - if (unpack != drv.env.end() && unpack->second == "1") { - if (string(*data, 0, 6) == string("\xfd" "7zXZ\0", 6)) - data = decompress("xz", *data); - StringSource source(*data); - restorePath(storePath, source); - } else - writeFile(storePath, *data); - - auto executable = drv.env.find("executable"); - if (executable != drv.env.end() && executable->second == "1") { - if (chmod(storePath.c_str(), 0755) == -1) - throw SysError(format("making ‘%1%’ executable") % storePath); - } -} - -} diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh index 4b2431aa08c..66597e456c4 100644 --- a/src/libstore/builtins.hh +++ b/src/libstore/builtins.hh @@ -4,6 +4,8 @@ namespace nix { -void builtinFetchurl(const BasicDerivation & drv); +// TODO: make pluggable. +void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData); +void builtinUnpackChannel(const BasicDerivation & drv); } diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc new file mode 100644 index 00000000000..802fb87bceb --- /dev/null +++ b/src/libstore/builtins/buildenv.cc @@ -0,0 +1,199 @@ +#include "buildenv.hh" + +#include +#include +#include +#include + +namespace nix { + +struct State +{ + std::map priorities; + unsigned long symlinks = 0; +}; + +/* For each activated package, create symlinks */ +static void createLinks(State & state, const Path & srcDir, const Path & dstDir, int priority) +{ + DirEntries srcFiles; + + try { + srcFiles = readDirectory(srcDir); + } catch (SysError & e) { + if (e.errNo == ENOTDIR) { + logWarning({ + .name = "Create links - directory", + .hint = hintfmt("not including '%s' in the user environment because it's not a directory", srcDir) + }); + return; + } + throw; + } + + for (const auto & ent : srcFiles) { + if (ent.name[0] == '.') + /* not matched by glob */ + continue; + auto srcFile = srcDir + "/" + ent.name; + auto dstFile = dstDir + "/" + ent.name; + + struct stat srcSt; + try { + if (stat(srcFile.c_str(), &srcSt) == -1) + throw SysError("getting status of '%1%'", srcFile); + } catch (SysError & e) { + if (e.errNo == ENOENT || e.errNo == ENOTDIR) { + logWarning({ + .name = "Create links - skipping symlink", + .hint = hintfmt("skipping dangling symlink '%s'", dstFile) + }); + continue; + } + throw; + } + + /* The files below are special-cased to that they don't show up + * in user profiles, either because they are useless, or + * because they would cauase pointless collisions (e.g., each + * Python package brings its own + * `$out/lib/pythonX.Y/site-packages/easy-install.pth'.) + */ + if (hasSuffix(srcFile, "/propagated-build-inputs") || + hasSuffix(srcFile, "/nix-support") || + hasSuffix(srcFile, "/perllocal.pod") || + hasSuffix(srcFile, "/info/dir") || + hasSuffix(srcFile, "/log")) + continue; + + else if (S_ISDIR(srcSt.st_mode)) { + struct stat dstSt; + auto res = lstat(dstFile.c_str(), &dstSt); + if (res == 0) { + if (S_ISDIR(dstSt.st_mode)) { + createLinks(state, srcFile, dstFile, priority); + continue; + } else if (S_ISLNK(dstSt.st_mode)) { + auto target = canonPath(dstFile, true); + if (!S_ISDIR(lstat(target).st_mode)) + throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target); + if (unlink(dstFile.c_str()) == -1) + throw SysError("unlinking '%1%'", dstFile); + if (mkdir(dstFile.c_str(), 0755) == -1) + throw SysError("creating directory '%1%'", dstFile); + createLinks(state, target, dstFile, state.priorities[dstFile]); + createLinks(state, srcFile, dstFile, priority); + continue; + } + } else if (errno != ENOENT) + throw SysError("getting status of '%1%'", dstFile); + } + + else { + struct stat dstSt; + auto res = lstat(dstFile.c_str(), &dstSt); + if (res == 0) { + if (S_ISLNK(dstSt.st_mode)) { + auto prevPriority = state.priorities[dstFile]; + if (prevPriority == priority) + throw Error( + "packages '%1%' and '%2%' have the same priority %3%; " + "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' " + "to change the priority of one of the conflicting packages" + " (0 being the highest priority)", + srcFile, readLink(dstFile), priority); + if (prevPriority < priority) + continue; + if (unlink(dstFile.c_str()) == -1) + throw SysError("unlinking '%1%'", dstFile); + } else if (S_ISDIR(dstSt.st_mode)) + throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile); + } else if (errno != ENOENT) + throw SysError("getting status of '%1%'", dstFile); + } + + createSymlink(srcFile, dstFile); + state.priorities[dstFile] = priority; + state.symlinks++; + } +} + +void buildProfile(const Path & out, Packages && pkgs) +{ + State state; + + std::set done, postponed; + + auto addPkg = [&](const Path & pkgDir, int priority) { + if (!done.insert(pkgDir).second) return; + createLinks(state, pkgDir, out, priority); + + try { + for (const auto & p : tokenizeString>( + readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n")) + if (!done.count(p)) + postponed.insert(p); + } catch (SysError & e) { + if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw; + } + }; + + /* Symlink to the packages that have been installed explicitly by the + * user. Process in priority order to reduce unnecessary + * symlink/unlink steps. + */ + std::sort(pkgs.begin(), pkgs.end(), [](const Package & a, const Package & b) { + return a.priority < b.priority || (a.priority == b.priority && a.path < b.path); + }); + for (const auto & pkg : pkgs) + if (pkg.active) + addPkg(pkg.path, pkg.priority); + + /* Symlink to the packages that have been "propagated" by packages + * installed by the user (i.e., package X declares that it wants Y + * installed as well). We do these later because they have a lower + * priority in case of collisions. + */ + auto priorityCounter = 1000; + while (!postponed.empty()) { + std::set pkgDirs; + postponed.swap(pkgDirs); + for (const auto & pkgDir : pkgDirs) + addPkg(pkgDir, priorityCounter++); + } + + debug("created %d symlinks in user environment", state.symlinks); +} + +void builtinBuildenv(const BasicDerivation & drv) +{ + auto getAttr = [&](const string & name) { + auto i = drv.env.find(name); + if (i == drv.env.end()) throw Error("attribute '%s' missing", name); + return i->second; + }; + + Path out = getAttr("out"); + createDirs(out); + + /* Convert the stuff we get from the environment back into a + * coherent data type. */ + Packages pkgs; + auto derivations = tokenizeString(getAttr("derivations")); + while (!derivations.empty()) { + /* !!! We're trusting the caller to structure derivations env var correctly */ + auto active = derivations.front(); derivations.pop_front(); + auto priority = stoi(derivations.front()); derivations.pop_front(); + auto outputs = stoi(derivations.front()); derivations.pop_front(); + for (auto n = 0; n < outputs; n++) { + auto path = derivations.front(); derivations.pop_front(); + pkgs.emplace_back(path, active != "false", priority); + } + } + + buildProfile(out, std::move(pkgs)); + + createSymlink(getAttr("manifest"), out + "/manifest.nix"); +} + +} diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh new file mode 100644 index 00000000000..0a37459b0f0 --- /dev/null +++ b/src/libstore/builtins/buildenv.hh @@ -0,0 +1,21 @@ +#pragma once + +#include "derivations.hh" +#include "store-api.hh" + +namespace nix { + +struct Package { + Path path; + bool active; + int priority; + Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} +}; + +typedef std::vector Packages; + +void buildProfile(const Path & out, Packages && pkgs); + +void builtinBuildenv(const BasicDerivation & drv); + +} diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc new file mode 100644 index 00000000000..2048f8f876b --- /dev/null +++ b/src/libstore/builtins/fetchurl.cc @@ -0,0 +1,78 @@ +#include "builtins.hh" +#include "filetransfer.hh" +#include "store-api.hh" +#include "archive.hh" +#include "compression.hh" + +namespace nix { + +void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) +{ + /* Make the host's netrc data available. Too bad curl requires + this to be stored in a file. It would be nice if we could just + pass a pointer to the data. */ + if (netrcData != "") { + settings.netrcFile = "netrc"; + writeFile(settings.netrcFile, netrcData, 0600); + } + + auto getAttr = [&](const string & name) { + auto i = drv.env.find(name); + if (i == drv.env.end()) throw Error("attribute '%s' missing", name); + return i->second; + }; + + Path storePath = getAttr("out"); + auto mainUrl = getAttr("url"); + bool unpack = get(drv.env, "unpack").value_or("") == "1"; + + /* Note: have to use a fresh fileTransfer here because we're in + a forked process. */ + auto fileTransfer = makeFileTransfer(); + + auto fetch = [&](const std::string & url) { + + auto source = sinkToSource([&](Sink & sink) { + + /* No need to do TLS verification, because we check the hash of + the result anyway. */ + FileTransferRequest request(url); + request.verifyTLS = false; + request.decompress = false; + + auto decompressor = makeDecompressionSink( + unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink); + fileTransfer->download(std::move(request), *decompressor); + decompressor->finish(); + }); + + if (unpack) + restorePath(storePath, *source); + else + writeFile(storePath, *source); + + auto executable = drv.env.find("executable"); + if (executable != drv.env.end() && executable->second == "1") { + if (chmod(storePath.c_str(), 0755) == -1) + throw SysError("making '%1%' executable", storePath); + } + }; + + /* Try the hashed mirrors first. */ + if (getAttr("outputHashMode") == "flat") + for (auto hashedMirror : settings.hashedMirrors.get()) + try { + if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; + auto ht = parseHashType(getAttr("outputHashAlgo")); + auto h = Hash(getAttr("outputHash"), ht); + fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); + return; + } catch (Error & e) { + debug(e.what()); + } + + /* Otherwise try the specified URL. */ + fetch(mainUrl); +} + +} diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc new file mode 100644 index 00000000000..d18e3ddafbc --- /dev/null +++ b/src/libstore/builtins/unpack-channel.cc @@ -0,0 +1,29 @@ +#include "builtins.hh" +#include "tarfile.hh" + +namespace nix { + +void builtinUnpackChannel(const BasicDerivation & drv) +{ + auto getAttr = [&](const string & name) { + auto i = drv.env.find(name); + if (i == drv.env.end()) throw Error("attribute '%s' missing", name); + return i->second; + }; + + Path out = getAttr("out"); + auto channelName = getAttr("channelName"); + auto src = getAttr("src"); + + createDirs(out); + + unpackTarfile(src, out); + + auto entries = readDirectory(out); + if (entries.size() != 1) + throw Error("channel tarball '%s' contains more than one file", src); + if (rename((out + "/" + entries[0].name).c_str(), (out + "/" + channelName).c_str()) == -1) + throw SysError("renaming channel directory"); +} + +} diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 747483afb30..9ec8abd228e 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -105,12 +105,12 @@ PublicKeys getDefaultPublicKeys() // FIXME: filter duplicates - for (auto s : settings.get("binary-cache-public-keys", Strings())) { + for (auto s : settings.trustedPublicKeys.get()) { PublicKey key(s); publicKeys.emplace(key.name, key); } - for (auto secretKeyFile : settings.get("secret-key-files", Strings())) { + for (auto secretKeyFile : settings.secretKeyFiles.get()) { try { SecretKey secretKey(readFile(secretKeyFile)); publicKeys.emplace(secretKey.name, secretKey.toPublicKey()); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc new file mode 100644 index 00000000000..6207225161b --- /dev/null +++ b/src/libstore/daemon.cc @@ -0,0 +1,865 @@ +#include "daemon.hh" +#include "monitor-fd.hh" +#include "worker-protocol.hh" +#include "store-api.hh" +#include "local-store.hh" +#include "finally.hh" +#include "affinity.hh" +#include "archive.hh" +#include "derivations.hh" +#include "args.hh" + +namespace nix::daemon { + +Sink & operator << (Sink & sink, const Logger::Fields & fields) +{ + sink << fields.size(); + for (auto & f : fields) { + sink << f.type; + if (f.type == Logger::Field::tInt) + sink << f.i; + else if (f.type == Logger::Field::tString) + sink << f.s; + else abort(); + } + return sink; +} + +/* Logger that forwards log messages to the client, *if* we're in a + state where the protocol allows it (i.e., when canSendStderr is + true). */ +struct TunnelLogger : public Logger +{ + FdSink & to; + + struct State + { + bool canSendStderr = false; + std::vector pendingMsgs; + }; + + Sync state_; + + unsigned int clientVersion; + + TunnelLogger(FdSink & to, unsigned int clientVersion) + : to(to), clientVersion(clientVersion) { } + + void enqueueMsg(const std::string & s) + { + auto state(state_.lock()); + + if (state->canSendStderr) { + assert(state->pendingMsgs.empty()); + try { + to(s); + to.flush(); + } catch (...) { + /* Write failed; that means that the other side is + gone. */ + state->canSendStderr = false; + throw; + } + } else + state->pendingMsgs.push_back(s); + } + + void log(Verbosity lvl, const FormatOrString & fs) override + { + if (lvl > verbosity) return; + + StringSink buf; + buf << STDERR_NEXT << (fs.s + "\n"); + enqueueMsg(*buf.s); + } + + void logEI(const ErrorInfo & ei) override + { + if (ei.level > verbosity) return; + + std::stringstream oss; + oss << ei; + + StringSink buf; + buf << STDERR_NEXT << oss.str() << "\n"; // (fs.s + "\n"); + enqueueMsg(*buf.s); + } + + /* startWork() means that we're starting an operation for which we + want to send out stderr to the client. */ + void startWork() + { + auto state(state_.lock()); + state->canSendStderr = true; + + for (auto & msg : state->pendingMsgs) + to(msg); + + state->pendingMsgs.clear(); + + to.flush(); + } + + /* stopWork() means that we're done; stop sending stderr to the + client. */ + void stopWork(bool success = true, const string & msg = "", unsigned int status = 0) + { + auto state(state_.lock()); + + state->canSendStderr = false; + + if (success) + to << STDERR_LAST; + else { + to << STDERR_ERROR << msg; + if (status != 0) to << status; + } + } + + void startActivity(ActivityId act, Verbosity lvl, ActivityType type, + const std::string & s, const Fields & fields, ActivityId parent) override + { + if (GET_PROTOCOL_MINOR(clientVersion) < 20) { + if (!s.empty()) + log(lvl, s + "..."); + return; + } + + StringSink buf; + buf << STDERR_START_ACTIVITY << act << lvl << type << s << fields << parent; + enqueueMsg(*buf.s); + } + + void stopActivity(ActivityId act) override + { + if (GET_PROTOCOL_MINOR(clientVersion) < 20) return; + StringSink buf; + buf << STDERR_STOP_ACTIVITY << act; + enqueueMsg(*buf.s); + } + + void result(ActivityId act, ResultType type, const Fields & fields) override + { + if (GET_PROTOCOL_MINOR(clientVersion) < 20) return; + StringSink buf; + buf << STDERR_RESULT << act << type << fields; + enqueueMsg(*buf.s); + } +}; + +struct TunnelSink : Sink +{ + Sink & to; + TunnelSink(Sink & to) : to(to) { } + virtual void operator () (const unsigned char * data, size_t len) + { + to << STDERR_WRITE; + writeString(data, len, to); + } +}; + +struct TunnelSource : BufferedSource +{ + Source & from; + BufferedSink & to; + TunnelSource(Source & from, BufferedSink & to) : from(from), to(to) { } + size_t readUnbuffered(unsigned char * data, size_t len) override + { + to << STDERR_READ << len; + to.flush(); + size_t n = readString(data, len, from); + if (n == 0) throw EndOfFile("unexpected end-of-file"); + return n; + } +}; + +/* If the NAR archive contains a single file at top-level, then save + the contents of the file to `s'. Otherwise barf. */ +struct RetrieveRegularNARSink : ParseSink +{ + bool regular; + string s; + + RetrieveRegularNARSink() : regular(true) { } + + void createDirectory(const Path & path) + { + regular = false; + } + + void receiveContents(unsigned char * data, unsigned int len) + { + s.append((const char *) data, len); + } + + void createSymlink(const Path & path, const string & target) + { + regular = false; + } +}; + +struct ClientSettings +{ + bool keepFailed; + bool keepGoing; + bool tryFallback; + Verbosity verbosity; + unsigned int maxBuildJobs; + time_t maxSilentTime; + bool verboseBuild; + unsigned int buildCores; + bool useSubstitutes; + StringMap overrides; + + void apply(TrustedFlag trusted) + { + settings.keepFailed = keepFailed; + settings.keepGoing = keepGoing; + settings.tryFallback = tryFallback; + nix::verbosity = verbosity; + settings.maxBuildJobs.assign(maxBuildJobs); + settings.maxSilentTime = maxSilentTime; + settings.verboseBuild = verboseBuild; + settings.buildCores = buildCores; + settings.useSubstitutes = useSubstitutes; + + for (auto & i : overrides) { + auto & name(i.first); + auto & value(i.second); + + auto setSubstituters = [&](Setting & res) { + if (name != res.name && res.aliases.count(name) == 0) + return false; + StringSet trusted = settings.trustedSubstituters; + for (auto & s : settings.substituters.get()) + trusted.insert(s); + Strings subs; + auto ss = tokenizeString(value); + for (auto & s : ss) + if (trusted.count(s)) + subs.push_back(s); + else + warn("ignoring untrusted substituter '%s'", s); + res = subs; + return true; + }; + + try { + if (name == "ssh-auth-sock") // obsolete + ; + else if (trusted + || name == settings.buildTimeout.name + || name == "connect-timeout" + || (name == "builders" && value == "")) + settings.set(name, value); + else if (setSubstituters(settings.substituters)) + ; + else if (setSubstituters(settings.extraSubstituters)) + ; + else + warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); + } catch (UsageError & e) { + warn(e.what()); + } + } + } +}; + +static void performOp(TunnelLogger * logger, ref store, + TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion, + Source & from, BufferedSink & to, unsigned int op) +{ + switch (op) { + + case wopIsValidPath: { + auto path = store->parseStorePath(readString(from)); + logger->startWork(); + bool result = store->isValidPath(path); + logger->stopWork(); + to << result; + break; + } + + case wopQueryValidPaths: { + auto paths = readStorePaths(*store, from); + logger->startWork(); + auto res = store->queryValidPaths(paths); + logger->stopWork(); + writeStorePaths(*store, to, res); + break; + } + + case wopHasSubstitutes: { + auto path = store->parseStorePath(readString(from)); + logger->startWork(); + StorePathSet paths; // FIXME + paths.insert(path.clone()); + auto res = store->querySubstitutablePaths(paths); + logger->stopWork(); + to << (res.count(path) != 0); + break; + } + + case wopQuerySubstitutablePaths: { + auto paths = readStorePaths(*store, from); + logger->startWork(); + auto res = store->querySubstitutablePaths(paths); + logger->stopWork(); + writeStorePaths(*store, to, res); + break; + } + + case wopQueryPathHash: { + auto path = store->parseStorePath(readString(from)); + logger->startWork(); + auto hash = store->queryPathInfo(path)->narHash; + logger->stopWork(); + to << hash.to_string(Base16, false); + break; + } + + case wopQueryReferences: + case wopQueryReferrers: + case wopQueryValidDerivers: + case wopQueryDerivationOutputs: { + auto path = store->parseStorePath(readString(from)); + logger->startWork(); + StorePathSet paths; + if (op == wopQueryReferences) + for (auto & i : store->queryPathInfo(path)->references) + paths.insert(i.clone()); + else if (op == wopQueryReferrers) + store->queryReferrers(path, paths); + else if (op == wopQueryValidDerivers) + paths = store->queryValidDerivers(path); + else paths = store->queryDerivationOutputs(path); + logger->stopWork(); + writeStorePaths(*store, to, paths); + break; + } + + case wopQueryDerivationOutputNames: { + auto path = store->parseStorePath(readString(from)); + logger->startWork(); + auto names = store->readDerivation(path).outputNames(); + logger->stopWork(); + to << names; + break; + } + + case wopQueryDeriver: { + auto path = store->parseStorePath(readString(from)); + logger->startWork(); + auto info = store->queryPathInfo(path); + logger->stopWork(); + to << (info->deriver ? store->printStorePath(*info->deriver) : ""); + break; + } + + case wopQueryPathFromHashPart: { + auto hashPart = readString(from); + logger->startWork(); + auto path = store->queryPathFromHashPart(hashPart); + logger->stopWork(); + to << (path ? store->printStorePath(*path) : ""); + break; + } + + case wopAddToStore: { + std::string s, baseName; + FileIngestionMethod method; + { + bool fixed; uint8_t recursive; + from >> baseName >> fixed /* obsolete */ >> recursive >> s; + if (recursive > (uint8_t) FileIngestionMethod::Recursive) + throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); + method = FileIngestionMethod { recursive }; + /* Compatibility hack. */ + if (!fixed) { + s = "sha256"; + method = FileIngestionMethod::Recursive; + } + } + HashType hashAlgo = parseHashType(s); + + TeeSource savedNAR(from); + RetrieveRegularNARSink savedRegular; + + if (method == FileIngestionMethod::Recursive) { + /* Get the entire NAR dump from the client and save it to + a string so that we can pass it to + addToStoreFromDump(). */ + ParseSink sink; /* null sink; just parse the NAR */ + parseDump(sink, savedNAR); + } else + parseDump(savedRegular, from); + + logger->startWork(); + if (!savedRegular.regular) throw Error("regular file expected"); + + auto path = store->addToStoreFromDump( + method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s, + baseName, + method, + hashAlgo); + logger->stopWork(); + + to << store->printStorePath(path); + break; + } + + case wopAddTextToStore: { + string suffix = readString(from); + string s = readString(from); + auto refs = readStorePaths(*store, from); + logger->startWork(); + auto path = store->addTextToStore(suffix, s, refs, NoRepair); + logger->stopWork(); + to << store->printStorePath(path); + break; + } + + case wopExportPath: { + auto path = store->parseStorePath(readString(from)); + readInt(from); // obsolete + logger->startWork(); + TunnelSink sink(to); + store->exportPath(path, sink); + logger->stopWork(); + to << 1; + break; + } + + case wopImportPaths: { + logger->startWork(); + TunnelSource source(from, to); + auto paths = store->importPaths(source, nullptr, + trusted ? NoCheckSigs : CheckSigs); + logger->stopWork(); + Strings paths2; + for (auto & i : paths) paths2.push_back(store->printStorePath(i)); + to << paths2; + break; + } + + case wopBuildPaths: { + std::vector drvs; + for (auto & s : readStrings(from)) + drvs.push_back(store->parsePathWithOutputs(s)); + BuildMode mode = bmNormal; + if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { + mode = (BuildMode) readInt(from); + + /* Repairing is not atomic, so disallowed for "untrusted" + clients. */ + if (mode == bmRepair && !trusted) + throw Error("repairing is not allowed because you are not in 'trusted-users'"); + } + logger->startWork(); + store->buildPaths(drvs, mode); + logger->stopWork(); + to << 1; + break; + } + + case wopBuildDerivation: { + auto drvPath = store->parseStorePath(readString(from)); + BasicDerivation drv; + readDerivation(from, *store, drv); + BuildMode buildMode = (BuildMode) readInt(from); + logger->startWork(); + if (!trusted) + throw Error("you are not privileged to build derivations"); + auto res = store->buildDerivation(drvPath, drv, buildMode); + logger->stopWork(); + to << res.status << res.errorMsg; + break; + } + + case wopEnsurePath: { + auto path = store->parseStorePath(readString(from)); + logger->startWork(); + store->ensurePath(path); + logger->stopWork(); + to << 1; + break; + } + + case wopAddTempRoot: { + auto path = store->parseStorePath(readString(from)); + logger->startWork(); + store->addTempRoot(path); + logger->stopWork(); + to << 1; + break; + } + + case wopAddIndirectRoot: { + Path path = absPath(readString(from)); + logger->startWork(); + store->addIndirectRoot(path); + logger->stopWork(); + to << 1; + break; + } + + case wopSyncWithGC: { + logger->startWork(); + store->syncWithGC(); + logger->stopWork(); + to << 1; + break; + } + + case wopFindRoots: { + logger->startWork(); + Roots roots = store->findRoots(!trusted); + logger->stopWork(); + + size_t size = 0; + for (auto & i : roots) + size += i.second.size(); + + to << size; + + for (auto & [target, links] : roots) + for (auto & link : links) + to << link << store->printStorePath(target); + + break; + } + + case wopCollectGarbage: { + GCOptions options; + options.action = (GCOptions::GCAction) readInt(from); + options.pathsToDelete = readStorePaths(*store, from); + from >> options.ignoreLiveness >> options.maxFreed; + // obsolete fields + readInt(from); + readInt(from); + readInt(from); + + GCResults results; + + logger->startWork(); + if (options.ignoreLiveness) + throw Error("you are not allowed to ignore liveness"); + store->collectGarbage(options, results); + logger->stopWork(); + + to << results.paths << results.bytesFreed << 0 /* obsolete */; + + break; + } + + case wopSetOptions: { + + ClientSettings clientSettings; + + clientSettings.keepFailed = readInt(from); + clientSettings.keepGoing = readInt(from); + clientSettings.tryFallback = readInt(from); + clientSettings.verbosity = (Verbosity) readInt(from); + clientSettings.maxBuildJobs = readInt(from); + clientSettings.maxSilentTime = readInt(from); + readInt(from); // obsolete useBuildHook + clientSettings.verboseBuild = lvlError == (Verbosity) readInt(from); + readInt(from); // obsolete logType + readInt(from); // obsolete printBuildTrace + clientSettings.buildCores = readInt(from); + clientSettings.useSubstitutes = readInt(from); + + if (GET_PROTOCOL_MINOR(clientVersion) >= 12) { + unsigned int n = readInt(from); + for (unsigned int i = 0; i < n; i++) { + string name = readString(from); + string value = readString(from); + clientSettings.overrides.emplace(name, value); + } + } + + logger->startWork(); + + // FIXME: use some setting in recursive mode. Will need to use + // non-global variables. + if (!recursive) + clientSettings.apply(trusted); + + logger->stopWork(); + break; + } + + case wopQuerySubstitutablePathInfo: { + auto path = store->parseStorePath(readString(from)); + logger->startWork(); + SubstitutablePathInfos infos; + StorePathSet paths; + paths.insert(path.clone()); // FIXME + store->querySubstitutablePathInfos(paths, infos); + logger->stopWork(); + auto i = infos.find(path); + if (i == infos.end()) + to << 0; + else { + to << 1 + << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); + writeStorePaths(*store, to, i->second.references); + to << i->second.downloadSize + << i->second.narSize; + } + break; + } + + case wopQuerySubstitutablePathInfos: { + auto paths = readStorePaths(*store, from); + logger->startWork(); + SubstitutablePathInfos infos; + store->querySubstitutablePathInfos(paths, infos); + logger->stopWork(); + to << infos.size(); + for (auto & i : infos) { + to << store->printStorePath(i.first) + << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); + writeStorePaths(*store, to, i.second.references); + to << i.second.downloadSize << i.second.narSize; + } + break; + } + + case wopQueryAllValidPaths: { + logger->startWork(); + auto paths = store->queryAllValidPaths(); + logger->stopWork(); + writeStorePaths(*store, to, paths); + break; + } + + case wopQueryPathInfo: { + auto path = store->parseStorePath(readString(from)); + std::shared_ptr info; + logger->startWork(); + try { + info = store->queryPathInfo(path); + } catch (InvalidPath &) { + if (GET_PROTOCOL_MINOR(clientVersion) < 17) throw; + } + logger->stopWork(); + if (info) { + if (GET_PROTOCOL_MINOR(clientVersion) >= 17) + to << 1; + to << (info->deriver ? store->printStorePath(*info->deriver) : "") + << info->narHash.to_string(Base16, false); + writeStorePaths(*store, to, info->references); + to << info->registrationTime << info->narSize; + if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { + to << info->ultimate + << info->sigs + << info->ca; + } + } else { + assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); + to << 0; + } + break; + } + + case wopOptimiseStore: + logger->startWork(); + store->optimiseStore(); + logger->stopWork(); + to << 1; + break; + + case wopVerifyStore: { + bool checkContents, repair; + from >> checkContents >> repair; + logger->startWork(); + if (repair && !trusted) + throw Error("you are not privileged to repair paths"); + bool errors = store->verifyStore(checkContents, (RepairFlag) repair); + logger->stopWork(); + to << errors; + break; + } + + case wopAddSignatures: { + auto path = store->parseStorePath(readString(from)); + StringSet sigs = readStrings(from); + logger->startWork(); + if (!trusted) + throw Error("you are not privileged to add signatures"); + store->addSignatures(path, sigs); + logger->stopWork(); + to << 1; + break; + } + + case wopNarFromPath: { + auto path = store->parseStorePath(readString(from)); + logger->startWork(); + logger->stopWork(); + dumpPath(store->printStorePath(path), to); + break; + } + + case wopAddToStoreNar: { + bool repair, dontCheckSigs; + ValidPathInfo info(store->parseStorePath(readString(from))); + auto deriver = readString(from); + if (deriver != "") + info.deriver = store->parseStorePath(deriver); + info.narHash = Hash(readString(from), htSHA256); + info.references = readStorePaths(*store, from); + from >> info.registrationTime >> info.narSize >> info.ultimate; + info.sigs = readStrings(from); + from >> info.ca >> repair >> dontCheckSigs; + if (!trusted && dontCheckSigs) + dontCheckSigs = false; + if (!trusted) + info.ultimate = false; + + std::string saved; + std::unique_ptr source; + if (GET_PROTOCOL_MINOR(clientVersion) >= 21) + source = std::make_unique(from, to); + else { + TeeSink tee(from); + parseDump(tee, tee.source); + saved = std::move(*tee.source.data); + source = std::make_unique(saved); + } + + logger->startWork(); + + // FIXME: race if addToStore doesn't read source? + store->addToStore(info, *source, (RepairFlag) repair, + dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr); + + logger->stopWork(); + break; + } + + case wopQueryMissing: { + std::vector targets; + for (auto & s : readStrings(from)) + targets.push_back(store->parsePathWithOutputs(s)); + logger->startWork(); + StorePathSet willBuild, willSubstitute, unknown; + unsigned long long downloadSize, narSize; + store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); + logger->stopWork(); + writeStorePaths(*store, to, willBuild); + writeStorePaths(*store, to, willSubstitute); + writeStorePaths(*store, to, unknown); + to << downloadSize << narSize; + break; + } + + default: + throw Error("invalid operation %1%", op); + } +} + +void processConnection( + ref store, + FdSource & from, + FdSink & to, + TrustedFlag trusted, + RecursiveFlag recursive, + const std::string & userName, + uid_t userId) +{ + auto monitor = !recursive ? std::make_unique(from.fd) : nullptr; + + /* Exchange the greeting. */ + unsigned int magic = readInt(from); + if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); + to << WORKER_MAGIC_2 << PROTOCOL_VERSION; + to.flush(); + unsigned int clientVersion = readInt(from); + + if (clientVersion < 0x10a) + throw Error("the Nix client version is too old"); + + auto tunnelLogger = new TunnelLogger(to, clientVersion); + auto prevLogger = nix::logger; + // FIXME + if (!recursive) + logger = tunnelLogger; + + unsigned int opCount = 0; + + Finally finally([&]() { + _isInterrupted = false; + prevLogger->log(lvlDebug, fmt("%d operations", opCount)); + }); + + if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) { + auto affinity = readInt(from); + setAffinityTo(affinity); + } + + readInt(from); // obsolete reserveSpace + + /* Send startup error messages to the client. */ + tunnelLogger->startWork(); + + try { + + /* If we can't accept clientVersion, then throw an error + *here* (not above). */ + +#if 0 + /* Prevent users from doing something very dangerous. */ + if (geteuid() == 0 && + querySetting("build-users-group", "") == "") + throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); +#endif + + store->createUser(userName, userId); + + tunnelLogger->stopWork(); + to.flush(); + + /* Process client requests. */ + while (true) { + WorkerOp op; + try { + op = (WorkerOp) readInt(from); + } catch (Interrupted & e) { + break; + } catch (EndOfFile & e) { + break; + } + + opCount++; + + try { + performOp(tunnelLogger, store, trusted, recursive, clientVersion, from, to, op); + } catch (Error & e) { + /* If we're not in a state where we can send replies, then + something went wrong processing the input of the + client. This can happen especially if I/O errors occur + during addTextToStore() / importPath(). If that + happens, just send the error message and exit. */ + bool errorAllowed = tunnelLogger->state_.lock()->canSendStderr; + tunnelLogger->stopWork(false, e.msg(), e.status); + if (!errorAllowed) throw; + } catch (std::bad_alloc & e) { + tunnelLogger->stopWork(false, "Nix daemon out of memory", 1); + throw; + } + + to.flush(); + + assert(!tunnelLogger->state_.lock()->canSendStderr); + }; + + } catch (std::exception & e) { + tunnelLogger->stopWork(false, e.what(), 1); + to.flush(); + return; + } +} + +} diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh new file mode 100644 index 00000000000..26693201325 --- /dev/null +++ b/src/libstore/daemon.hh @@ -0,0 +1,18 @@ +#include "serialise.hh" +#include "store-api.hh" + +namespace nix::daemon { + +enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; +enum RecursiveFlag : bool { NotRecursive = false, Recursive = true }; + +void processConnection( + ref store, + FdSource & from, + FdSink & to, + TrustedFlag trusted, + RecursiveFlag recursive, + const std::string & userName, + uid_t userId); + +} diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 79526c594f7..915e02eed51 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -4,47 +4,57 @@ #include "util.hh" #include "worker-protocol.hh" #include "fs-accessor.hh" - +#include "istringstream_nocopy.hh" namespace nix { -void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const +void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const { - recursive = false; + recursive = FileIngestionMethod::Flat; string algo = hashAlgo; if (string(algo, 0, 2) == "r:") { - recursive = true; + recursive = FileIngestionMethod::Recursive; algo = string(algo, 2); } HashType hashType = parseHashType(algo); if (hashType == htUnknown) - throw Error(format("unknown hash algorithm ‘%1%’") % algo); + throw Error("unknown hash algorithm '%s'", algo); - hash = parseHash(hashType, this->hash); + hash = Hash(this->hash, hashType); } -Path BasicDerivation::findOutput(const string & id) const +BasicDerivation::BasicDerivation(const BasicDerivation & other) + : platform(other.platform) + , builder(other.builder) + , args(other.args) + , env(other.env) { - auto i = outputs.find(id); - if (i == outputs.end()) - throw Error(format("derivation has no output ‘%1%’") % id); - return i->second.path; + for (auto & i : other.outputs) + outputs.insert_or_assign(i.first, + DerivationOutput(i.second.path.clone(), std::string(i.second.hashAlgo), std::string(i.second.hash))); + for (auto & i : other.inputSrcs) + inputSrcs.insert(i.clone()); } -bool BasicDerivation::willBuildLocally() const +Derivation::Derivation(const Derivation & other) + : BasicDerivation(other) { - return get(env, "preferLocalBuild") == "1" && canBuildLocally(); + for (auto & i : other.inputDrvs) + inputDrvs.insert_or_assign(i.first.clone(), i.second); } -bool BasicDerivation::substitutesAllowed() const +const StorePath & BasicDerivation::findOutput(const string & id) const { - return get(env, "allowSubstitutes", "1") == "1"; + auto i = outputs.find(id); + if (i == outputs.end()) + throw Error("derivation has no output '%s'", id); + return i->second.path; } @@ -54,34 +64,17 @@ bool BasicDerivation::isBuiltin() const } -bool BasicDerivation::canBuildLocally() const -{ - return platform == settings.thisSystem - || isBuiltin() -#if __linux__ - || (platform == "i686-linux" && settings.thisSystem == "x86_64-linux") - || (platform == "armv6l-linux" && settings.thisSystem == "armv7l-linux") - || (platform == "armv5tel-linux" && (settings.thisSystem == "armv7l-linux" || settings.thisSystem == "armv6l-linux")) -#elif __FreeBSD__ - || (platform == "i686-linux" && settings.thisSystem == "x86_64-freebsd") - || (platform == "i686-linux" && settings.thisSystem == "i686-freebsd") -#endif - ; -} - - -Path writeDerivation(ref store, - const Derivation & drv, const string & name, bool repair) +StorePath writeDerivation(ref store, + const Derivation & drv, std::string_view name, RepairFlag repair) { - PathSet references; - references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); + auto references = cloneStorePathSet(drv.inputSrcs); for (auto & i : drv.inputDrvs) - references.insert(i.first); + references.insert(i.first.clone()); /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be held during a garbage collection). */ - string suffix = name + drvExtension; - string contents = drv.unparse(); + auto suffix = std::string(name) + drvExtension; + auto contents = drv.unparse(*store, false); return settings.readOnlyMode ? store->computeStorePathForText(suffix, contents, references) : store->addTextToStore(suffix, contents, references, repair); @@ -94,7 +87,7 @@ static void expect(std::istream & str, const string & s) char s2[s.size()]; str.read(s2, s.size()); if (string(s2, s.size()) != s) - throw FormatError(format("expected string ‘%1%’") % s); + throw FormatError("expected string '%1%'", s); } @@ -121,7 +114,7 @@ static Path parsePath(std::istream & str) { string s = parseString(str); if (s.size() == 0 || s[0] != '/') - throw FormatError(format("bad path ‘%1%’ in derivation") % s); + throw FormatError("bad path '%1%' in derivation", s); return s; } @@ -149,21 +142,20 @@ static StringSet parseStrings(std::istream & str, bool arePaths) } -static Derivation parseDerivation(const string & s) +static Derivation parseDerivation(const Store & store, const string & s) { Derivation drv; - std::istringstream str(s); + istringstream_nocopy str(s); expect(str, "Derive(["); /* Parse the list of outputs. */ while (!endOfList(str)) { - DerivationOutput out; - expect(str, "("); string id = parseString(str); - expect(str, ","); out.path = parsePath(str); - expect(str, ","); out.hashAlgo = parseString(str); - expect(str, ","); out.hash = parseString(str); + expect(str, "("); std::string id = parseString(str); + expect(str, ","); auto path = store.parseStorePath(parsePath(str)); + expect(str, ","); auto hashAlgo = parseString(str); + expect(str, ","); auto hash = parseString(str); expect(str, ")"); - drv.outputs[id] = out; + drv.outputs.emplace(id, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash))); } /* Parse the list of input derivations. */ @@ -172,11 +164,11 @@ static Derivation parseDerivation(const string & s) expect(str, "("); Path drvPath = parsePath(str); expect(str, ",["); - drv.inputDrvs[drvPath] = parseStrings(str, false); + drv.inputDrvs.insert_or_assign(store.parseStorePath(drvPath), parseStrings(str, false)); expect(str, ")"); } - expect(str, ",["); drv.inputSrcs = parseStrings(str, true); + expect(str, ",["); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true)); expect(str, ","); drv.platform = parseString(str); expect(str, ","); drv.builder = parseString(str); @@ -199,38 +191,54 @@ static Derivation parseDerivation(const string & s) } -Derivation readDerivation(const Path & drvPath) +Derivation readDerivation(const Store & store, const Path & drvPath) { try { - return parseDerivation(readFile(drvPath)); + return parseDerivation(store, readFile(drvPath)); } catch (FormatError & e) { - throw Error(format("error parsing derivation ‘%1%’: %2%") % drvPath % e.msg()); + throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg()); } } -Derivation Store::derivationFromPath(const Path & drvPath) +Derivation Store::derivationFromPath(const StorePath & drvPath) { - assertStorePath(drvPath); ensurePath(drvPath); + return readDerivation(drvPath); +} + + +Derivation Store::readDerivation(const StorePath & drvPath) +{ auto accessor = getFSAccessor(); try { - return parseDerivation(accessor->readFile(drvPath)); + return parseDerivation(*this, accessor->readFile(printStorePath(drvPath))); } catch (FormatError & e) { - throw Error(format("error parsing derivation ‘%1%’: %2%") % drvPath % e.msg()); + throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg()); } } -static void printString(string & res, const string & s) +static void printString(string & res, std::string_view s) +{ + char buf[s.size() * 2 + 2]; + char * p = buf; + *p++ = '"'; + for (auto c : s) + if (c == '\"' || c == '\\') { *p++ = '\\'; *p++ = c; } + else if (c == '\n') { *p++ = '\\'; *p++ = 'n'; } + else if (c == '\r') { *p++ = '\\'; *p++ = 'r'; } + else if (c == '\t') { *p++ = '\\'; *p++ = 't'; } + else *p++ = c; + *p++ = '"'; + res.append(buf, p - buf); +} + + +static void printUnquotedString(string & res, std::string_view s) { res += '"'; - for (const char * i = s.c_str(); *i; i++) - if (*i == '\"' || *i == '\\') { res += "\\"; res += *i; } - else if (*i == '\n') res += "\\n"; - else if (*i == '\r') res += "\\r"; - else if (*i == '\t') res += "\\t"; - else res += *i; + res.append(s); res += '"'; } @@ -248,7 +256,21 @@ static void printStrings(string & res, ForwardIterator i, ForwardIterator j) } -string Derivation::unparse() const +template +static void printUnquotedStrings(string & res, ForwardIterator i, ForwardIterator j) +{ + res += '['; + bool first = true; + for ( ; i != j; ++i) { + if (first) first = false; else res += ','; + printUnquotedString(res, *i); + } + res += ']'; +} + + +string Derivation::unparse(const Store & store, bool maskOutputs, + std::map * actualInputs) const { string s; s.reserve(65536); @@ -257,26 +279,36 @@ string Derivation::unparse() const bool first = true; for (auto & i : outputs) { if (first) first = false; else s += ','; - s += '('; printString(s, i.first); - s += ','; printString(s, i.second.path); - s += ','; printString(s, i.second.hashAlgo); - s += ','; printString(s, i.second.hash); + s += '('; printUnquotedString(s, i.first); + s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path)); + s += ','; printUnquotedString(s, i.second.hashAlgo); + s += ','; printUnquotedString(s, i.second.hash); s += ')'; } s += "],["; first = true; - for (auto & i : inputDrvs) { - if (first) first = false; else s += ','; - s += '('; printString(s, i.first); - s += ','; printStrings(s, i.second.begin(), i.second.end()); - s += ')'; + if (actualInputs) { + for (auto & i : *actualInputs) { + if (first) first = false; else s += ','; + s += '('; printUnquotedString(s, i.first); + s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end()); + s += ')'; + } + } else { + for (auto & i : inputDrvs) { + if (first) first = false; else s += ','; + s += '('; printUnquotedString(s, store.printStorePath(i.first)); + s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end()); + s += ')'; + } } s += "],"; - printStrings(s, inputSrcs.begin(), inputSrcs.end()); + auto paths = store.printStorePathSet(inputSrcs); // FIXME: slow + printUnquotedStrings(s, paths.begin(), paths.end()); - s += ','; printString(s, platform); + s += ','; printUnquotedString(s, platform); s += ','; printString(s, builder); s += ','; printStrings(s, args.begin(), args.end()); @@ -285,7 +317,7 @@ string Derivation::unparse() const for (auto & i : env) { if (first) first = false; else s += ','; s += '('; printString(s, i.first); - s += ','; printString(s, i.second); + s += ','; printString(s, maskOutputs && outputs.count(i.first) ? "" : i.second); s += ')'; } @@ -295,6 +327,7 @@ string Derivation::unparse() const } +// FIXME: remove bool isDerivation(const string & fileName) { return hasSuffix(fileName, drvExtension); @@ -332,7 +365,7 @@ DrvHashes drvHashes; paths have been replaced by the result of a recursive call to this function, and that for fixed-output derivations we return a hash of its output path. */ -Hash hashDerivationModulo(Store & store, Derivation drv) +Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) { /* Return a fixed hash for fixed-output derivations. */ if (drv.isFixedOutput()) { @@ -340,42 +373,31 @@ Hash hashDerivationModulo(Store & store, Derivation drv) return hashString(htSHA256, "fixed:out:" + i->second.hashAlgo + ":" + i->second.hash + ":" - + i->second.path); + + store.printStorePath(i->second.path)); } /* For other derivations, replace the inputs paths with recursive calls to this function.*/ - DerivationInputs inputs2; + std::map inputs2; for (auto & i : drv.inputDrvs) { - Hash h = drvHashes[i.first]; - if (!h) { + auto h = drvHashes.find(i.first); + if (h == drvHashes.end()) { assert(store.isValidPath(i.first)); - Derivation drv2 = readDerivation(i.first); - h = hashDerivationModulo(store, drv2); - drvHashes[i.first] = h; + h = drvHashes.insert_or_assign(i.first.clone(), hashDerivationModulo(store, + store.readDerivation(i.first), false)).first; } - inputs2[printHash(h)] = i.second; + inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second); } - drv.inputDrvs = inputs2; - - return hashString(htSHA256, drv.unparse()); -} - -DrvPathWithOutputs parseDrvPathWithOutputs(const string & s) -{ - size_t n = s.find("!"); - return n == s.npos - ? DrvPathWithOutputs(s, std::set()) - : DrvPathWithOutputs(string(s, 0, n), tokenizeString >(string(s, n + 1), ",")); + return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); } -Path makeDrvPathWithOutputs(const Path & drvPath, const std::set & outputs) +std::string StorePathWithOutputs::to_string(const Store & store) const { return outputs.empty() - ? drvPath - : drvPath + "!" + concatStringsSep(",", outputs); + ? store.printStorePath(path) + : store.printStorePath(path) + "!" + concatStringsSep(",", outputs); } @@ -385,33 +407,42 @@ bool wantOutput(const string & output, const std::set & wanted) } -PathSet BasicDerivation::outputPaths() const +StorePathSet BasicDerivation::outputPaths() const { - PathSet paths; + StorePathSet paths; for (auto & i : outputs) - paths.insert(i.second.path); + paths.insert(i.second.path.clone()); return paths; } -Source & readDerivation(Source & in, Store & store, BasicDerivation & drv) +StringSet BasicDerivation::outputNames() const +{ + StringSet names; + for (auto & i : outputs) + names.insert(i.first); + return names; +} + + +Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) { drv.outputs.clear(); - auto nr = readInt(in); - for (unsigned int n = 0; n < nr; n++) { + auto nr = readNum(in); + for (size_t n = 0; n < nr; n++) { auto name = readString(in); - DerivationOutput o; - in >> o.path >> o.hashAlgo >> o.hash; - store.assertStorePath(o.path); - drv.outputs[name] = o; + auto path = store.parseStorePath(readString(in)); + auto hashAlgo = readString(in); + auto hash = readString(in); + drv.outputs.emplace(name, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash))); } - drv.inputSrcs = readStorePaths(store, in); + drv.inputSrcs = readStorePaths(store, in); in >> drv.platform >> drv.builder; drv.args = readStrings(in); - nr = readInt(in); - for (unsigned int n = 0; n < nr; n++) { + nr = readNum(in); + for (size_t n = 0; n < nr; n++) { auto key = readString(in); auto value = readString(in); drv.env[key] = value; @@ -421,23 +452,23 @@ Source & readDerivation(Source & in, Store & store, BasicDerivation & drv) } -Sink & operator << (Sink & out, const BasicDerivation & drv) +void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv) { out << drv.outputs.size(); for (auto & i : drv.outputs) - out << i.first << i.second.path << i.second.hashAlgo << i.second.hash; - out << drv.inputSrcs << drv.platform << drv.builder << drv.args; + out << i.first << store.printStorePath(i.second.path) << i.second.hashAlgo << i.second.hash; + writeStorePaths(store, out, drv.inputSrcs); + out << drv.platform << drv.builder << drv.args; out << drv.env.size(); for (auto & i : drv.env) out << i.first << i.second; - return out; } std::string hashPlaceholder(const std::string & outputName) { // FIXME: memoize? - return "/" + printHash32(hashString(htSHA256, "nix-output:" + outputName)); + return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false); } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 9717a81e469..88aed66bf29 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -2,6 +2,7 @@ #include "types.hh" #include "hash.hh" +#include "store-api.hh" #include @@ -9,66 +10,56 @@ namespace nix { -/* Extension of derivations in the Nix store. */ -const string drvExtension = ".drv"; - - /* Abstract syntax of derivations. */ struct DerivationOutput { - Path path; - string hashAlgo; /* hash used for expected hash computation */ - string hash; /* expected hash, may be null */ - DerivationOutput() - { - } - DerivationOutput(Path path, string hashAlgo, string hash) - { - this->path = path; - this->hashAlgo = hashAlgo; - this->hash = hash; - } - void parseHashInfo(bool & recursive, Hash & hash) const; + StorePath path; + std::string hashAlgo; /* hash used for expected hash computation */ + std::string hash; /* expected hash, may be null */ + DerivationOutput(StorePath && path, std::string && hashAlgo, std::string && hash) + : path(std::move(path)) + , hashAlgo(std::move(hashAlgo)) + , hash(std::move(hash)) + { } + void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const; }; typedef std::map DerivationOutputs; /* For inputs that are sub-derivations, we specify exactly which output IDs we are interested in. */ -typedef std::map DerivationInputs; +typedef std::map DerivationInputs; typedef std::map StringPairs; struct BasicDerivation { DerivationOutputs outputs; /* keyed on symbolic IDs */ - PathSet inputSrcs; /* inputs that are sources */ + StorePathSet inputSrcs; /* inputs that are sources */ string platform; Path builder; Strings args; StringPairs env; + BasicDerivation() { } + explicit BasicDerivation(const BasicDerivation & other); virtual ~BasicDerivation() { }; /* Return the path corresponding to the output identifier `id' in the given derivation. */ - Path findOutput(const string & id) const; - - bool willBuildLocally() const; - - bool substitutesAllowed() const; + const StorePath & findOutput(const std::string & id) const; bool isBuiltin() const; - bool canBuildLocally() const; - /* Return true iff this is a fixed-output derivation. */ bool isFixedOutput() const; /* Return the output paths of a derivation. */ - PathSet outputPaths() const; + StorePathSet outputPaths() const; + /* Return the output names of a derivation. */ + StringSet outputNames() const; }; struct Derivation : BasicDerivation @@ -76,7 +67,12 @@ struct Derivation : BasicDerivation DerivationInputs inputDrvs; /* inputs that are sub-derivations */ /* Print a derivation. */ - std::string unparse() const; + std::string unparse(const Store & store, bool maskOutputs, + std::map * actualInputs = nullptr) const; + + Derivation() { } + Derivation(Derivation && other) = default; + explicit Derivation(const Derivation & other); }; @@ -84,38 +80,29 @@ class Store; /* Write a derivation to the Nix store, and return its path. */ -Path writeDerivation(ref store, - const Derivation & drv, const string & name, bool repair = false); +StorePath writeDerivation(ref store, + const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair); /* Read a derivation from a file. */ -Derivation readDerivation(const Path & drvPath); +Derivation readDerivation(const Store & store, const Path & drvPath); -/* Check whether a file name ends with the extension for - derivations. */ +// FIXME: remove bool isDerivation(const string & fileName); -Hash hashDerivationModulo(Store & store, Derivation drv); +Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); /* Memoisation of hashDerivationModulo(). */ -typedef std::map DrvHashes; +typedef std::map DrvHashes; extern DrvHashes drvHashes; // FIXME: global, not thread-safe -/* Split a string specifying a derivation and a set of outputs - (/nix/store/hash-foo!out1,out2,...) into the derivation path and - the outputs. */ -typedef std::pair > DrvPathWithOutputs; -DrvPathWithOutputs parseDrvPathWithOutputs(const string & s); - -Path makeDrvPathWithOutputs(const Path & drvPath, const std::set & outputs); - bool wantOutput(const string & output, const std::set & wanted); struct Source; struct Sink; -Source & readDerivation(Source & in, Store & store, BasicDerivation & drv); -Sink & operator << (Sink & out, const BasicDerivation & drv); +Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv); +void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv); std::string hashPlaceholder(const std::string & outputName); diff --git a/src/libstore/download.cc b/src/libstore/download.cc deleted file mode 100644 index 63442f3029d..00000000000 --- a/src/libstore/download.cc +++ /dev/null @@ -1,657 +0,0 @@ -#include "download.hh" -#include "util.hh" -#include "globals.hh" -#include "hash.hh" -#include "store-api.hh" -#include "archive.hh" - -#include -#include - -#include - -#include -#include -#include -#include -#include - -namespace nix { - -MakeError(URLEncodeError, Error); - -double getTime() -{ - struct timeval tv; - gettimeofday(&tv, 0); - return tv.tv_sec + (tv.tv_usec / 1000000.0); -} - -std::string resolveUri(const std::string & uri) -{ - if (uri.compare(0, 8, "channel:") == 0) - return "https://nixos.org/channels/" + std::string(uri, 8) + "/nixexprs.tar.xz"; - else - return uri; -} - -struct CurlDownloader : public Downloader -{ - CURLM * curlm = 0; - - std::random_device rd; - std::mt19937 mt19937; - - bool enableHttp2; - - struct DownloadItem : public std::enable_shared_from_this - { - CurlDownloader & downloader; - DownloadRequest request; - DownloadResult result; - bool done = false; // whether either the success or failure function has been called - std::function success; - std::function failure; - CURL * req = 0; - bool active = false; // whether the handle has been added to the multi object - std::string status; - - bool showProgress = false; - double prevProgressTime{0}, startTime{0}; - unsigned int moveBack{1}; - - unsigned int attempt = 0; - - /* Don't start this download until the specified time point - has been reached. */ - std::chrono::steady_clock::time_point embargo; - - struct curl_slist * requestHeaders = 0; - - DownloadItem(CurlDownloader & downloader, const DownloadRequest & request) - : downloader(downloader), request(request) - { - showProgress = - request.showProgress == DownloadRequest::yes || - (request.showProgress == DownloadRequest::automatic && isatty(STDERR_FILENO)); - - if (!request.expectedETag.empty()) - requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); - } - - ~DownloadItem() - { - if (req) { - if (active) - curl_multi_remove_handle(downloader.curlm, req); - curl_easy_cleanup(req); - } - if (requestHeaders) curl_slist_free_all(requestHeaders); - try { - if (!done) - fail(DownloadError(Interrupted, format("download of ‘%s’ was interrupted") % request.uri)); - } catch (...) { - ignoreException(); - } - } - - template - void fail(const T & e) - { - assert(!done); - done = true; - callFailure(failure, std::make_exception_ptr(e)); - } - - size_t writeCallback(void * contents, size_t size, size_t nmemb) - { - size_t realSize = size * nmemb; - result.data->append((char *) contents, realSize); - return realSize; - } - - static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) - { - return ((DownloadItem *) userp)->writeCallback(contents, size, nmemb); - } - - size_t headerCallback(void * contents, size_t size, size_t nmemb) - { - size_t realSize = size * nmemb; - std::string line((char *) contents, realSize); - printMsg(lvlVomit, format("got header for ‘%s’: %s") % request.uri % trim(line)); - if (line.compare(0, 5, "HTTP/") == 0) { // new response starts - result.etag = ""; - auto ss = tokenizeString>(line, " "); - status = ss.size() >= 2 ? ss[1] : ""; - result.data = std::make_shared(); - } else { - auto i = line.find(':'); - if (i != string::npos) { - string name = toLower(trim(string(line, 0, i))); - if (name == "etag") { - result.etag = trim(string(line, i + 1)); - /* Hack to work around a GitHub bug: it sends - ETags, but ignores If-None-Match. So if we get - the expected ETag on a 200 response, then shut - down the connection because we already have the - data. */ - if (result.etag == request.expectedETag && status == "200") { - debug(format("shutting down on 200 HTTP response with expected ETag")); - return 0; - } - } - } - } - return realSize; - } - - static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) - { - return ((DownloadItem *) userp)->headerCallback(contents, size, nmemb); - } - - int progressCallback(double dltotal, double dlnow) - { - if (showProgress) { - double now = getTime(); - if (prevProgressTime <= now - 1) { - string s = (format(" [%1$.0f/%2$.0f KiB, %3$.1f KiB/s]") - % (dlnow / 1024.0) - % (dltotal / 1024.0) - % (now == startTime ? 0 : dlnow / 1024.0 / (now - startTime))).str(); - std::cerr << "\e[" << moveBack << "D" << s; - moveBack = s.size(); - std::cerr.flush(); - prevProgressTime = now; - } - } - return _isInterrupted; - } - - static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow) - { - return ((DownloadItem *) userp)->progressCallback(dltotal, dlnow); - } - - static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr) - { - if (type == CURLINFO_TEXT) - vomit("curl: %s", chomp(std::string(data, size))); - return 0; - } - - void init() - { - // FIXME: handle parallel downloads. - if (showProgress) { - std::cerr << (format("downloading ‘%1%’... ") % request.uri); - std::cerr.flush(); - startTime = getTime(); - } - - if (!req) req = curl_easy_init(); - - curl_easy_reset(req); - - if (verbosity >= lvlVomit) { - curl_easy_setopt(req, CURLOPT_VERBOSE, 1); - curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, DownloadItem::debugCallback); - } - - curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str()); - curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); - curl_easy_setopt(req, CURLOPT_USERAGENT, ("Nix/" + nixVersion).c_str()); - #if LIBCURL_VERSION_NUM >= 0x072b00 - curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); - #endif - #if LIBCURL_VERSION_NUM >= 0x072f00 - if (downloader.enableHttp2) - curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); - #endif - curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper); - curl_easy_setopt(req, CURLOPT_WRITEDATA, this); - curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, DownloadItem::headerCallbackWrapper); - curl_easy_setopt(req, CURLOPT_HEADERDATA, this); - - curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper); - curl_easy_setopt(req, CURLOPT_PROGRESSDATA, this); - curl_easy_setopt(req, CURLOPT_NOPROGRESS, 0); - - curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders); - - if (request.head) - curl_easy_setopt(req, CURLOPT_NOBODY, 1); - - if (request.verifyTLS) - curl_easy_setopt(req, CURLOPT_CAINFO, - getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt")).c_str()); - else { - curl_easy_setopt(req, CURLOPT_SSL_VERIFYPEER, 0); - curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); - } - - result.data = std::make_shared(); - } - - void finish(CURLcode code) - { - if (showProgress) - //std::cerr << "\e[" << moveBack << "D\e[K\n"; - std::cerr << "\n"; - - long httpStatus = 0; - curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); - - char * effectiveUrlCStr; - curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUrlCStr); - if (effectiveUrlCStr) - result.effectiveUrl = effectiveUrlCStr; - - debug(format("finished download of ‘%s’; curl status = %d, HTTP status = %d, body = %d bytes") - % request.uri % code % httpStatus % (result.data ? result.data->size() : 0)); - - if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) { - code = CURLE_OK; - httpStatus = 304; - } - - if (code == CURLE_OK && - (httpStatus == 200 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) - { - result.cached = httpStatus == 304; - done = true; - callSuccess(success, failure, const_cast(result)); - } else { - Error err = - (httpStatus == 404 || code == CURLE_FILE_COULDNT_READ_FILE) ? NotFound : - httpStatus == 403 ? Forbidden : - (httpStatus == 408 || httpStatus == 500 || httpStatus == 503 - || httpStatus == 504 || httpStatus == 522 || httpStatus == 524 - || code == CURLE_COULDNT_RESOLVE_HOST) ? Transient : - Misc; - - attempt++; - - auto exc = - code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted - ? DownloadError(Interrupted, format("download of ‘%s’ was interrupted") % request.uri) - : httpStatus != 0 - ? DownloadError(err, format("unable to download ‘%s’: HTTP error %d") % request.uri % httpStatus) - : DownloadError(err, format("unable to download ‘%s’: %s (%d)") % request.uri % curl_easy_strerror(code) % code); - - /* If this is a transient error, then maybe retry the - download after a while. */ - if (err == Transient && attempt < request.tries) { - int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); - printError(format("warning: %s; retrying in %d ms") % exc.what() % ms); - embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); - downloader.enqueueItem(shared_from_this()); - } - else - fail(exc); - } - } - }; - - struct State - { - struct EmbargoComparator { - bool operator() (const std::shared_ptr & i1, const std::shared_ptr & i2) { - return i1->embargo > i2->embargo; - } - }; - bool quit = false; - std::priority_queue, std::vector>, EmbargoComparator> incoming; - }; - - Sync state_; - - /* We can't use a std::condition_variable to wake up the curl - thread, because it only monitors file descriptors. So use a - pipe instead. */ - Pipe wakeupPipe; - - std::thread workerThread; - - CurlDownloader() - : mt19937(rd()) - { - static std::once_flag globalInit; - std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL); - - curlm = curl_multi_init(); - - #if LIBCURL_VERSION_NUM >= 0x072b00 // correct? - curl_multi_setopt(curlm, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX); - #endif - curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, - settings.get("binary-caches-parallel-connections", 25)); - - enableHttp2 = settings.get("enable-http2", true); - - wakeupPipe.create(); - fcntl(wakeupPipe.readSide.get(), F_SETFL, O_NONBLOCK); - - workerThread = std::thread([&]() { workerThreadEntry(); }); - } - - ~CurlDownloader() - { - stopWorkerThread(); - - workerThread.join(); - - if (curlm) curl_multi_cleanup(curlm); - } - - void stopWorkerThread() - { - /* Signal the worker thread to exit. */ - { - auto state(state_.lock()); - state->quit = true; - } - writeFull(wakeupPipe.writeSide.get(), " ", false); - } - - void workerThreadMain() - { - /* Cause this thread to be notified on SIGINT. */ - auto callback = createInterruptCallback([&]() { - stopWorkerThread(); - }); - - std::map> items; - - bool quit = false; - - std::chrono::steady_clock::time_point nextWakeup; - - while (!quit) { - checkInterrupt(); - - /* Let curl do its thing. */ - int running; - CURLMcode mc = curl_multi_perform(curlm, &running); - if (mc != CURLM_OK) - throw nix::Error(format("unexpected error from curl_multi_perform(): %s") % curl_multi_strerror(mc)); - - /* Set the promises of any finished requests. */ - CURLMsg * msg; - int left; - while ((msg = curl_multi_info_read(curlm, &left))) { - if (msg->msg == CURLMSG_DONE) { - auto i = items.find(msg->easy_handle); - assert(i != items.end()); - i->second->finish(msg->data.result); - curl_multi_remove_handle(curlm, i->second->req); - i->second->active = false; - items.erase(i); - } - } - - /* Wait for activity, including wakeup events. */ - int numfds = 0; - struct curl_waitfd extraFDs[1]; - extraFDs[0].fd = wakeupPipe.readSide.get(); - extraFDs[0].events = CURL_WAIT_POLLIN; - extraFDs[0].revents = 0; - auto sleepTimeMs = - nextWakeup != std::chrono::steady_clock::time_point() - ? std::max(0, (int) std::chrono::duration_cast(nextWakeup - std::chrono::steady_clock::now()).count()) - : 1000000000; - vomit("download thread waiting for %d ms", sleepTimeMs); - mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds); - if (mc != CURLM_OK) - throw nix::Error(format("unexpected error from curl_multi_wait(): %s") % curl_multi_strerror(mc)); - - nextWakeup = std::chrono::steady_clock::time_point(); - - /* Add new curl requests from the incoming requests queue, - except for requests that are embargoed (waiting for a - retry timeout to expire). */ - if (extraFDs[0].revents & CURL_WAIT_POLLIN) { - char buf[1024]; - auto res = read(extraFDs[0].fd, buf, sizeof(buf)); - if (res == -1 && errno != EINTR) - throw SysError("reading curl wakeup socket"); - } - - std::vector> incoming; - auto now = std::chrono::steady_clock::now(); - - { - auto state(state_.lock()); - while (!state->incoming.empty()) { - auto item = state->incoming.top(); - if (item->embargo <= now) { - incoming.push_back(item); - state->incoming.pop(); - } else { - if (nextWakeup == std::chrono::steady_clock::time_point() - || item->embargo < nextWakeup) - nextWakeup = item->embargo; - break; - } - } - quit = state->quit; - } - - for (auto & item : incoming) { - debug(format("starting download of %s") % item->request.uri); - item->init(); - curl_multi_add_handle(curlm, item->req); - item->active = true; - items[item->req] = item; - } - } - - debug("download thread shutting down"); - } - - void workerThreadEntry() - { - try { - workerThreadMain(); - } catch (nix::Interrupted & e) { - } catch (std::exception & e) { - printError(format("unexpected error in download thread: %s") % e.what()); - } - - { - auto state(state_.lock()); - while (!state->incoming.empty()) state->incoming.pop(); - state->quit = true; - } - } - - void enqueueItem(std::shared_ptr item) - { - { - auto state(state_.lock()); - if (state->quit) - throw nix::Error("cannot enqueue download request because the download thread is shutting down"); - state->incoming.push(item); - } - writeFull(wakeupPipe.writeSide.get(), " "); - } - - void enqueueDownload(const DownloadRequest & request, - std::function success, - std::function failure) override - { - auto item = std::make_shared(*this, request); - item->success = success; - item->failure = failure; - enqueueItem(item); - } - - std::string urlEncode(const std::string & param) override { - //TODO reuse curl handle or move function to another class/file - CURL *curl = curl_easy_init(); - char *encoded = NULL; - if (curl) { - encoded = curl_easy_escape(curl, param.c_str(), 0); - } - if ((curl == NULL) || (encoded == NULL)) { - throw URLEncodeError("Could not encode param"); - } - std::string ret(encoded); - curl_free(encoded); - curl_easy_cleanup(curl); - return ret; - } -}; - -ref getDownloader() -{ - static std::shared_ptr downloader; - static std::once_flag downloaderCreated; - std::call_once(downloaderCreated, [&]() { downloader = makeDownloader(); }); - return ref(downloader); -} - -ref makeDownloader() -{ - return make_ref(); -} - -std::future Downloader::enqueueDownload(const DownloadRequest & request) -{ - auto promise = std::make_shared>(); - enqueueDownload(request, - [promise](const DownloadResult & result) { promise->set_value(result); }, - [promise](std::exception_ptr exc) { promise->set_exception(exc); }); - return promise->get_future(); -} - -DownloadResult Downloader::download(const DownloadRequest & request) -{ - return enqueueDownload(request).get(); -} - -Path Downloader::downloadCached(ref store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl) -{ - auto url = resolveUri(url_); - - if (name == "") { - auto p = url.rfind('/'); - if (p != string::npos) name = string(url, p + 1); - } - - Path expectedStorePath; - if (expectedHash) { - expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name); - if (store->isValidPath(expectedStorePath)) - return expectedStorePath; - } - - Path cacheDir = getCacheDir() + "/nix/tarballs"; - createDirs(cacheDir); - - string urlHash = printHash32(hashString(htSHA256, url)); - - Path dataFile = cacheDir + "/" + urlHash + ".info"; - Path fileLink = cacheDir + "/" + urlHash + "-file"; - - Path storePath; - - string expectedETag; - - int ttl = settings.get("tarball-ttl", 60 * 60); - bool skip = false; - - if (pathExists(fileLink) && pathExists(dataFile)) { - storePath = readLink(fileLink); - store->addTempRoot(storePath); - if (store->isValidPath(storePath)) { - auto ss = tokenizeString>(readFile(dataFile), "\n"); - if (ss.size() >= 3 && ss[0] == url) { - time_t lastChecked; - if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0)) { - skip = true; - if (effectiveUrl) - *effectiveUrl = url_; - } else if (!ss[1].empty()) { - debug(format("verifying previous ETag ‘%1%’") % ss[1]); - expectedETag = ss[1]; - } - } - } else - storePath = ""; - } - - if (!skip) { - - try { - DownloadRequest request(url); - request.expectedETag = expectedETag; - auto res = download(request); - if (effectiveUrl) - *effectiveUrl = res.effectiveUrl; - - if (!res.cached) { - ValidPathInfo info; - StringSink sink; - dumpString(*res.data, sink); - Hash hash = hashString(expectedHash ? expectedHash.type : htSHA256, *res.data); - info.path = store->makeFixedOutputPath(false, hash, name); - info.narHash = hashString(htSHA256, *sink.s); - store->addToStore(info, sink.s, false, true); - storePath = info.path; - } - - assert(!storePath.empty()); - replaceSymlink(storePath, fileLink); - - writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); - } catch (DownloadError & e) { - if (storePath.empty()) throw; - printError(format("warning: %1%; using cached result") % e.msg()); - } - } - - if (unpack) { - Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked"; - Path unpackedStorePath; - if (pathExists(unpackedLink)) { - unpackedStorePath = readLink(unpackedLink); - store->addTempRoot(unpackedStorePath); - if (!store->isValidPath(unpackedStorePath)) - unpackedStorePath = ""; - } - if (unpackedStorePath.empty()) { - printInfo(format("unpacking ‘%1%’...") % url); - Path tmpDir = createTempDir(); - AutoDelete autoDelete(tmpDir, true); - // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", storePath, "-C", tmpDir, "--strip-components", "1"}, ""); - unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, false); - } - replaceSymlink(unpackedStorePath, unpackedLink); - storePath = unpackedStorePath; - } - - if (expectedStorePath != "" && storePath != expectedStorePath) - throw nix::Error(format("hash mismatch in file downloaded from ‘%s’") % url); - - return storePath; -} - -std::string Downloader::urlEncode(const std::string & param) { - throw URLEncodeError("not implemented"); -} - -bool isUri(const string & s) -{ - if (s.compare(0, 8, "channel:") == 0) return true; - size_t pos = s.find("://"); - if (pos == string::npos) return false; - string scheme(s, 0, pos); - return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git"; -} - - -} diff --git a/src/libstore/download.hh b/src/libstore/download.hh deleted file mode 100644 index 77419973d6f..00000000000 --- a/src/libstore/download.hh +++ /dev/null @@ -1,78 +0,0 @@ -#pragma once - -#include "types.hh" -#include "hash.hh" - -#include -#include - -namespace nix { - -struct DownloadRequest -{ - std::string uri; - std::string expectedETag; - bool verifyTLS = true; - enum { yes, no, automatic } showProgress = yes; - bool head = false; - size_t tries = 1; - unsigned int baseRetryTimeMs = 250; - - DownloadRequest(const std::string & uri) : uri(uri) { } -}; - -struct DownloadResult -{ - bool cached; - std::string etag; - std::string effectiveUrl; - std::shared_ptr data; -}; - -class Store; - -struct Downloader -{ - /* Enqueue a download request, returning a future to the result of - the download. The future may throw a DownloadError - exception. */ - virtual void enqueueDownload(const DownloadRequest & request, - std::function success, - std::function failure) = 0; - - std::future enqueueDownload(const DownloadRequest & request); - - /* Synchronously download a file. */ - DownloadResult download(const DownloadRequest & request); - - /* Check if the specified file is already in ~/.cache/nix/tarballs - and is more recent than ‘tarball-ttl’ seconds. Otherwise, - use the recorded ETag to verify if the server has a more - recent version, and if so, download it to the Nix store. */ - Path downloadCached(ref store, const string & uri, bool unpack, string name = "", - const Hash & expectedHash = Hash(), string * effectiveUri = nullptr); - - virtual std::string urlEncode(const std::string & param); - - enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; -}; - -/* Return a shared Downloader object. Using this object is preferred - because it enables connection reuse and HTTP/2 multiplexing. */ -ref getDownloader(); - -/* Return a new Downloader object. */ -ref makeDownloader(); - -class DownloadError : public Error -{ -public: - Downloader::Error error; - DownloadError(Downloader::Error error, const FormatOrString & fs) - : Error(fs), error(error) - { } -}; - -bool isUri(const string & s); - -} diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index c5618c826c5..54471d4a3b1 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -1,3 +1,4 @@ +#include "serialise.hh" #include "store-api.hh" #include "archive.hh" #include "worker-protocol.hh" @@ -24,25 +25,25 @@ struct HashAndWriteSink : Sink } }; -void Store::exportPaths(const Paths & paths, Sink & sink) +void Store::exportPaths(const StorePathSet & paths, Sink & sink) { - Paths sorted = topoSortPaths(PathSet(paths.begin(), paths.end())); + auto sorted = topoSortPaths(paths); std::reverse(sorted.begin(), sorted.end()); std::string doneLabel("paths exported"); - logger->incExpected(doneLabel, sorted.size()); + //logger->incExpected(doneLabel, sorted.size()); for (auto & path : sorted) { - Activity act(*logger, lvlInfo, format("exporting path ‘%s’") % path); + //Activity act(*logger, lvlInfo, format("exporting path '%s'") % path); sink << 1; exportPath(path, sink); - logger->incProgress(doneLabel); + //logger->incProgress(doneLabel); } sink << 0; } -void Store::exportPath(const Path & path, Sink & sink) +void Store::exportPath(const StorePath & path, Sink & sink) { auto info = queryPathInfo(path); @@ -55,71 +56,56 @@ void Store::exportPath(const Path & path, Sink & sink) Don't complain if the stored hash is zero (unknown). */ Hash hash = hashAndWriteSink.currentHash(); if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) - throw Error(format("hash of path ‘%1%’ has changed from ‘%2%’ to ‘%3%’!") % path - % printHash(info->narHash) % printHash(hash)); - - hashAndWriteSink << exportMagic << path << info->references << info->deriver << 0; + throw Error("hash of path '%s' has changed from '%s' to '%s'!", + printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); + + hashAndWriteSink + << exportMagic + << printStorePath(path); + writeStorePaths(*this, hashAndWriteSink, info->references); + hashAndWriteSink + << (info->deriver ? printStorePath(*info->deriver) : "") + << 0; } -struct TeeSource : Source -{ - Source & readSource; - ref data; - TeeSource(Source & readSource) - : readSource(readSource) - , data(make_ref()) - { - } - size_t read(unsigned char * data, size_t len) - { - size_t n = readSource.read(data, len); - this->data->append((char *) data, n); - return n; - } -}; - -struct NopSink : ParseSink -{ -}; - -Paths Store::importPaths(Source & source, std::shared_ptr accessor, bool dontCheckSigs) +StorePaths Store::importPaths(Source & source, std::shared_ptr accessor, CheckSigsFlag checkSigs) { - Paths res; + StorePaths res; while (true) { - unsigned long long n = readLongLong(source); + auto n = readNum(source); if (n == 0) break; - if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’"); + if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'"); /* Extract the NAR from the source. */ - TeeSource tee(source); - NopSink sink; - parseDump(sink, tee); + TeeSink tee(source); + parseDump(tee, tee.source); uint32_t magic = readInt(source); if (magic != exportMagic) throw Error("Nix archive cannot be imported; wrong format"); - ValidPathInfo info; - - info.path = readStorePath(*this, source); + ValidPathInfo info(parseStorePath(readString(source))); - Activity act(*logger, lvlInfo, format("importing path ‘%s’") % info.path); + //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path); - info.references = readStorePaths(*this, source); + info.references = readStorePaths(*this, source); - info.deriver = readString(source); - if (info.deriver != "") assertStorePath(info.deriver); + auto deriver = readString(source); + if (deriver != "") + info.deriver = parseStorePath(deriver); - info.narHash = hashString(htSHA256, *tee.data); - info.narSize = tee.data->size(); + info.narHash = hashString(htSHA256, *tee.source.data); + info.narSize = tee.source.data->size(); // Ignore optional legacy signature. if (readInt(source) == 1) readString(source); - addToStore(info, tee.data, false, dontCheckSigs, accessor); + // Can't use underlying source, which would have been exhausted + auto source = StringSource { *tee.source.data }; + addToStore(info, source, NoRepair, checkSigs, accessor); - res.push_back(info.path); + res.push_back(info.path.clone()); } return res; diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc new file mode 100644 index 00000000000..7fc098e73ad --- /dev/null +++ b/src/libstore/filetransfer.cc @@ -0,0 +1,846 @@ +#include "filetransfer.hh" +#include "util.hh" +#include "globals.hh" +#include "store-api.hh" +#include "s3.hh" +#include "compression.hh" +#include "finally.hh" + +#ifdef ENABLE_S3 +#include +#endif + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace std::string_literals; + +namespace nix { + +FileTransferSettings fileTransferSettings; + +static GlobalConfig::Register r1(&fileTransferSettings); + +MakeError(URLEncodeError, Error); + +std::string resolveUri(const std::string & uri) +{ + if (uri.compare(0, 8, "channel:") == 0) + return "https://nixos.org/channels/" + std::string(uri, 8) + "/nixexprs.tar.xz"; + else + return uri; +} + +struct curlFileTransfer : public FileTransfer +{ + CURLM * curlm = 0; + + std::random_device rd; + std::mt19937 mt19937; + + struct TransferItem : public std::enable_shared_from_this + { + curlFileTransfer & fileTransfer; + FileTransferRequest request; + FileTransferResult result; + Activity act; + bool done = false; // whether either the success or failure function has been called + Callback callback; + CURL * req = 0; + bool active = false; // whether the handle has been added to the multi object + std::string status; + + unsigned int attempt = 0; + + /* Don't start this download until the specified time point + has been reached. */ + std::chrono::steady_clock::time_point embargo; + + struct curl_slist * requestHeaders = 0; + + std::string encoding; + + bool acceptRanges = false; + + curl_off_t writtenToSink = 0; + + TransferItem(curlFileTransfer & fileTransfer, + const FileTransferRequest & request, + Callback && callback) + : fileTransfer(fileTransfer) + , request(request) + , act(*logger, lvlTalkative, actFileTransfer, + fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), + {request.uri}, request.parentAct) + , callback(std::move(callback)) + , finalSink([this](const unsigned char * data, size_t len) { + if (this->request.dataCallback) { + long httpStatus = 0; + curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + + /* Only write data to the sink if this is a + successful response. */ + if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) { + writtenToSink += len; + this->request.dataCallback((char *) data, len); + } + } else + this->result.data->append((char *) data, len); + }) + { + if (!request.expectedETag.empty()) + requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); + if (!request.mimeType.empty()) + requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str()); + } + + ~TransferItem() + { + if (req) { + if (active) + curl_multi_remove_handle(fileTransfer.curlm, req); + curl_easy_cleanup(req); + } + if (requestHeaders) curl_slist_free_all(requestHeaders); + try { + if (!done) + fail(FileTransferError(Interrupted, "download of '%s' was interrupted", request.uri)); + } catch (...) { + ignoreException(); + } + } + + void failEx(std::exception_ptr ex) + { + assert(!done); + done = true; + callback.rethrow(ex); + } + + template + void fail(const T & e) + { + failEx(std::make_exception_ptr(e)); + } + + LambdaSink finalSink; + std::shared_ptr decompressionSink; + + std::exception_ptr writeException; + + size_t writeCallback(void * contents, size_t size, size_t nmemb) + { + try { + size_t realSize = size * nmemb; + result.bodySize += realSize; + + if (!decompressionSink) + decompressionSink = makeDecompressionSink(encoding, finalSink); + + (*decompressionSink)((unsigned char *) contents, realSize); + + return realSize; + } catch (...) { + writeException = std::current_exception(); + return 0; + } + } + + static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) + { + return ((TransferItem *) userp)->writeCallback(contents, size, nmemb); + } + + size_t headerCallback(void * contents, size_t size, size_t nmemb) + { + size_t realSize = size * nmemb; + std::string line((char *) contents, realSize); + printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line)); + if (line.compare(0, 5, "HTTP/") == 0) { // new response starts + result.etag = ""; + auto ss = tokenizeString>(line, " "); + status = ss.size() >= 2 ? ss[1] : ""; + result.data = std::make_shared(); + result.bodySize = 0; + acceptRanges = false; + encoding = ""; + } else { + auto i = line.find(':'); + if (i != string::npos) { + string name = toLower(trim(string(line, 0, i))); + if (name == "etag") { + result.etag = trim(string(line, i + 1)); + /* Hack to work around a GitHub bug: it sends + ETags, but ignores If-None-Match. So if we get + the expected ETag on a 200 response, then shut + down the connection because we already have the + data. */ + if (result.etag == request.expectedETag && status == "200") { + debug(format("shutting down on 200 HTTP response with expected ETag")); + return 0; + } + } else if (name == "content-encoding") + encoding = trim(string(line, i + 1)); + else if (name == "accept-ranges" && toLower(trim(std::string(line, i + 1))) == "bytes") + acceptRanges = true; + } + } + return realSize; + } + + static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) + { + return ((TransferItem *) userp)->headerCallback(contents, size, nmemb); + } + + int progressCallback(double dltotal, double dlnow) + { + try { + act.progress(dlnow, dltotal); + } catch (nix::Interrupted &) { + assert(_isInterrupted); + } + return _isInterrupted; + } + + static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow) + { + return ((TransferItem *) userp)->progressCallback(dltotal, dlnow); + } + + static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr) + { + if (type == CURLINFO_TEXT) + vomit("curl: %s", chomp(std::string(data, size))); + return 0; + } + + size_t readOffset = 0; + size_t readCallback(char *buffer, size_t size, size_t nitems) + { + if (readOffset == request.data->length()) + return 0; + auto count = std::min(size * nitems, request.data->length() - readOffset); + assert(count); + memcpy(buffer, request.data->data() + readOffset, count); + readOffset += count; + return count; + } + + static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp) + { + return ((TransferItem *) userp)->readCallback(buffer, size, nitems); + } + + void init() + { + if (!req) req = curl_easy_init(); + + curl_easy_reset(req); + + if (verbosity >= lvlVomit) { + curl_easy_setopt(req, CURLOPT_VERBOSE, 1); + curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, TransferItem::debugCallback); + } + + curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str()); + curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(req, CURLOPT_MAXREDIRS, 10); + curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); + curl_easy_setopt(req, CURLOPT_USERAGENT, + ("curl/" LIBCURL_VERSION " Nix/" + nixVersion + + (fileTransferSettings.userAgentSuffix != "" ? " " + fileTransferSettings.userAgentSuffix.get() : "")).c_str()); + #if LIBCURL_VERSION_NUM >= 0x072b00 + curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); + #endif + #if LIBCURL_VERSION_NUM >= 0x072f00 + if (fileTransferSettings.enableHttp2) + curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); + else + curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); + #endif + curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, TransferItem::writeCallbackWrapper); + curl_easy_setopt(req, CURLOPT_WRITEDATA, this); + curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper); + curl_easy_setopt(req, CURLOPT_HEADERDATA, this); + + curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper); + curl_easy_setopt(req, CURLOPT_PROGRESSDATA, this); + curl_easy_setopt(req, CURLOPT_NOPROGRESS, 0); + + curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders); + + if (request.head) + curl_easy_setopt(req, CURLOPT_NOBODY, 1); + + if (request.data) { + curl_easy_setopt(req, CURLOPT_UPLOAD, 1L); + curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper); + curl_easy_setopt(req, CURLOPT_READDATA, this); + curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length()); + } + + if (request.verifyTLS) { + debug("verify TLS: Nix CA file = '%s'", settings.caFile); + if (settings.caFile != "") + curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str()); + } else { + curl_easy_setopt(req, CURLOPT_SSL_VERIFYPEER, 0); + curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); + } + + curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get()); + + curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); + curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get()); + + /* If no file exist in the specified path, curl continues to work + anyway as if netrc support was disabled. */ + curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str()); + curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); + + if (writtenToSink) + curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink); + + result.data = std::make_shared(); + result.bodySize = 0; + } + + void finish(CURLcode code) + { + long httpStatus = 0; + curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + + char * effectiveUriCStr; + curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); + if (effectiveUriCStr) + result.effectiveUri = effectiveUriCStr; + + debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", + request.verb(), request.uri, code, httpStatus, result.bodySize); + + if (decompressionSink) { + try { + decompressionSink->finish(); + } catch (...) { + writeException = std::current_exception(); + } + } + + if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) { + code = CURLE_OK; + httpStatus = 304; + } + + if (writeException) + failEx(writeException); + + else if (code == CURLE_OK && + (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) + { + result.cached = httpStatus == 304; + act.progress(result.bodySize, result.bodySize); + done = true; + callback(std::move(result)); + } + + else { + // We treat most errors as transient, but won't retry when hopeless + Error err = Transient; + + if (httpStatus == 404 || httpStatus == 410 || code == CURLE_FILE_COULDNT_READ_FILE) { + // The file is definitely not there + err = NotFound; + } else if (httpStatus == 401 || httpStatus == 403 || httpStatus == 407) { + // Don't retry on authentication/authorization failures + err = Forbidden; + } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408 && httpStatus != 429) { + // Most 4xx errors are client errors and are probably not worth retrying: + // * 408 means the server timed out waiting for us, so we try again + // * 429 means too many requests, so we retry (with a delay) + err = Misc; + } else if (httpStatus == 501 || httpStatus == 505 || httpStatus == 511) { + // Let's treat most 5xx (server) errors as transient, except for a handful: + // * 501 not implemented + // * 505 http version not supported + // * 511 we're behind a captive portal + err = Misc; + } else { + // Don't bother retrying on certain cURL errors either + switch (code) { + case CURLE_FAILED_INIT: + case CURLE_URL_MALFORMAT: + case CURLE_NOT_BUILT_IN: + case CURLE_REMOTE_ACCESS_DENIED: + case CURLE_FILE_COULDNT_READ_FILE: + case CURLE_FUNCTION_NOT_FOUND: + case CURLE_ABORTED_BY_CALLBACK: + case CURLE_BAD_FUNCTION_ARGUMENT: + case CURLE_INTERFACE_FAILED: + case CURLE_UNKNOWN_OPTION: + case CURLE_SSL_CACERT_BADFILE: + case CURLE_TOO_MANY_REDIRECTS: + case CURLE_WRITE_ERROR: + case CURLE_UNSUPPORTED_PROTOCOL: + err = Misc; + break; + default: // Shut up warnings + break; + } + } + + attempt++; + + auto exc = + code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted + ? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) + : httpStatus != 0 + ? FileTransferError(err, + fmt("unable to %s '%s': HTTP error %d", + request.verb(), request.uri, httpStatus) + + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) + ) + : FileTransferError(err, + fmt("unable to %s '%s': %s (%d)", + request.verb(), request.uri, curl_easy_strerror(code), code)); + + /* If this is a transient error, then maybe retry the + download after a while. If we're writing to a + sink, we can only retry if the server supports + ranged requests. */ + if (err == Transient + && attempt < request.tries + && (!this->request.dataCallback + || writtenToSink == 0 + || (acceptRanges && encoding.empty()))) + { + int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937)); + if (writtenToSink) + warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms); + else + warn("%s; retrying in %d ms", exc.what(), ms); + embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); + fileTransfer.enqueueItem(shared_from_this()); + } + else + fail(exc); + } + } + }; + + struct State + { + struct EmbargoComparator { + bool operator() (const std::shared_ptr & i1, const std::shared_ptr & i2) { + return i1->embargo > i2->embargo; + } + }; + bool quit = false; + std::priority_queue, std::vector>, EmbargoComparator> incoming; + }; + + Sync state_; + + /* We can't use a std::condition_variable to wake up the curl + thread, because it only monitors file descriptors. So use a + pipe instead. */ + Pipe wakeupPipe; + + std::thread workerThread; + + curlFileTransfer() + : mt19937(rd()) + { + static std::once_flag globalInit; + std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL); + + curlm = curl_multi_init(); + + #if LIBCURL_VERSION_NUM >= 0x072b00 // Multiplex requires >= 7.43.0 + curl_multi_setopt(curlm, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX); + #endif + #if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0 + curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, + fileTransferSettings.httpConnections.get()); + #endif + + wakeupPipe.create(); + fcntl(wakeupPipe.readSide.get(), F_SETFL, O_NONBLOCK); + + workerThread = std::thread([&]() { workerThreadEntry(); }); + } + + ~curlFileTransfer() + { + stopWorkerThread(); + + workerThread.join(); + + if (curlm) curl_multi_cleanup(curlm); + } + + void stopWorkerThread() + { + /* Signal the worker thread to exit. */ + { + auto state(state_.lock()); + state->quit = true; + } + writeFull(wakeupPipe.writeSide.get(), " ", false); + } + + void workerThreadMain() + { + /* Cause this thread to be notified on SIGINT. */ + auto callback = createInterruptCallback([&]() { + stopWorkerThread(); + }); + + std::map> items; + + bool quit = false; + + std::chrono::steady_clock::time_point nextWakeup; + + while (!quit) { + checkInterrupt(); + + /* Let curl do its thing. */ + int running; + CURLMcode mc = curl_multi_perform(curlm, &running); + if (mc != CURLM_OK) + throw nix::Error("unexpected error from curl_multi_perform(): %s", curl_multi_strerror(mc)); + + /* Set the promises of any finished requests. */ + CURLMsg * msg; + int left; + while ((msg = curl_multi_info_read(curlm, &left))) { + if (msg->msg == CURLMSG_DONE) { + auto i = items.find(msg->easy_handle); + assert(i != items.end()); + i->second->finish(msg->data.result); + curl_multi_remove_handle(curlm, i->second->req); + i->second->active = false; + items.erase(i); + } + } + + /* Wait for activity, including wakeup events. */ + int numfds = 0; + struct curl_waitfd extraFDs[1]; + extraFDs[0].fd = wakeupPipe.readSide.get(); + extraFDs[0].events = CURL_WAIT_POLLIN; + extraFDs[0].revents = 0; + long maxSleepTimeMs = items.empty() ? 10000 : 100; + auto sleepTimeMs = + nextWakeup != std::chrono::steady_clock::time_point() + ? std::max(0, (int) std::chrono::duration_cast(nextWakeup - std::chrono::steady_clock::now()).count()) + : maxSleepTimeMs; + vomit("download thread waiting for %d ms", sleepTimeMs); + mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds); + if (mc != CURLM_OK) + throw nix::Error("unexpected error from curl_multi_wait(): %s", curl_multi_strerror(mc)); + + nextWakeup = std::chrono::steady_clock::time_point(); + + /* Add new curl requests from the incoming requests queue, + except for requests that are embargoed (waiting for a + retry timeout to expire). */ + if (extraFDs[0].revents & CURL_WAIT_POLLIN) { + char buf[1024]; + auto res = read(extraFDs[0].fd, buf, sizeof(buf)); + if (res == -1 && errno != EINTR) + throw SysError("reading curl wakeup socket"); + } + + std::vector> incoming; + auto now = std::chrono::steady_clock::now(); + + { + auto state(state_.lock()); + while (!state->incoming.empty()) { + auto item = state->incoming.top(); + if (item->embargo <= now) { + incoming.push_back(item); + state->incoming.pop(); + } else { + if (nextWakeup == std::chrono::steady_clock::time_point() + || item->embargo < nextWakeup) + nextWakeup = item->embargo; + break; + } + } + quit = state->quit; + } + + for (auto & item : incoming) { + debug("starting %s of %s", item->request.verb(), item->request.uri); + item->init(); + curl_multi_add_handle(curlm, item->req); + item->active = true; + items[item->req] = item; + } + } + + debug("download thread shutting down"); + } + + void workerThreadEntry() + { + try { + workerThreadMain(); + } catch (nix::Interrupted & e) { + } catch (std::exception & e) { + logError({ + .name = "File transfer", + .hint = hintfmt("unexpected error in download thread: %s", + e.what()) + }); + } + + { + auto state(state_.lock()); + while (!state->incoming.empty()) state->incoming.pop(); + state->quit = true; + } + } + + void enqueueItem(std::shared_ptr item) + { + if (item->request.data + && !hasPrefix(item->request.uri, "http://") + && !hasPrefix(item->request.uri, "https://")) + throw nix::Error("uploading to '%s' is not supported", item->request.uri); + + { + auto state(state_.lock()); + if (state->quit) + throw nix::Error("cannot enqueue download request because the download thread is shutting down"); + state->incoming.push(item); + } + writeFull(wakeupPipe.writeSide.get(), " "); + } + +#ifdef ENABLE_S3 + std::tuple parseS3Uri(std::string uri) + { + auto [path, params] = splitUriAndParams(uri); + + auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix + if (slash == std::string::npos) + throw nix::Error("bad S3 URI '%s'", path); + + std::string bucketName(path, 5, slash - 5); + std::string key(path, slash + 1); + + return {bucketName, key, params}; + } +#endif + + void enqueueFileTransfer(const FileTransferRequest & request, + Callback callback) override + { + /* Ugly hack to support s3:// URIs. */ + if (hasPrefix(request.uri, "s3://")) { + // FIXME: do this on a worker thread + try { +#ifdef ENABLE_S3 + auto [bucketName, key, params] = parseS3Uri(request.uri); + + std::string profile = get(params, "profile").value_or(""); + std::string region = get(params, "region").value_or(Aws::Region::US_EAST_1); + std::string scheme = get(params, "scheme").value_or(""); + std::string endpoint = get(params, "endpoint").value_or(""); + + S3Helper s3Helper(profile, region, scheme, endpoint); + + // FIXME: implement ETag + auto s3Res = s3Helper.getObject(bucketName, key); + FileTransferResult res; + if (!s3Res.data) + throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); + res.data = s3Res.data; + callback(std::move(res)); +#else + throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri); +#endif + } catch (...) { callback.rethrow(); } + return; + } + + enqueueItem(std::make_shared(*this, request, std::move(callback))); + } + + std::string urlEncode(const std::string & param) override { + //TODO reuse curl handle or move function to another class/file + CURL *curl = curl_easy_init(); + char *encoded = NULL; + if (curl) { + encoded = curl_easy_escape(curl, param.c_str(), 0); + } + if ((curl == NULL) || (encoded == NULL)) { + throw URLEncodeError("Could not encode param"); + } + std::string ret(encoded); + curl_free(encoded); + curl_easy_cleanup(curl); + return ret; + } +}; + +ref getFileTransfer() +{ + static ref fileTransfer = makeFileTransfer(); + return fileTransfer; +} + +ref makeFileTransfer() +{ + return make_ref(); +} + +std::future FileTransfer::enqueueFileTransfer(const FileTransferRequest & request) +{ + auto promise = std::make_shared>(); + enqueueFileTransfer(request, + {[promise](std::future fut) { + try { + promise->set_value(fut.get()); + } catch (...) { + promise->set_exception(std::current_exception()); + } + }}); + return promise->get_future(); +} + +FileTransferResult FileTransfer::download(const FileTransferRequest & request) +{ + return enqueueFileTransfer(request).get(); +} + +FileTransferResult FileTransfer::upload(const FileTransferRequest & request) +{ + /* Note: this method is the same as download, but helps in readability */ + return enqueueFileTransfer(request).get(); +} + +void FileTransfer::download(FileTransferRequest && request, Sink & sink) +{ + /* Note: we can't call 'sink' via request.dataCallback, because + that would cause the sink to execute on the fileTransfer + thread. If 'sink' is a coroutine, this will fail. Also, if the + sink is expensive (e.g. one that does decompression and writing + to the Nix store), it would stall the download thread too much. + Therefore we use a buffer to communicate data between the + download thread and the calling thread. */ + + struct State { + bool quit = false; + std::exception_ptr exc; + std::string data; + std::condition_variable avail, request; + }; + + auto _state = std::make_shared>(); + + /* In case of an exception, wake up the download thread. FIXME: + abort the download request. */ + Finally finally([&]() { + auto state(_state->lock()); + state->quit = true; + state->request.notify_one(); + }); + + request.dataCallback = [_state](char * buf, size_t len) { + + auto state(_state->lock()); + + if (state->quit) return; + + /* If the buffer is full, then go to sleep until the calling + thread wakes us up (i.e. when it has removed data from the + buffer). We don't wait forever to prevent stalling the + download thread. (Hopefully sleeping will throttle the + sender.) */ + if (state->data.size() > 1024 * 1024) { + debug("download buffer is full; going to sleep"); + state.wait_for(state->request, std::chrono::seconds(10)); + } + + /* Append data to the buffer and wake up the calling + thread. */ + state->data.append(buf, len); + state->avail.notify_one(); + }; + + enqueueFileTransfer(request, + {[_state](std::future fut) { + auto state(_state->lock()); + state->quit = true; + try { + fut.get(); + } catch (...) { + state->exc = std::current_exception(); + } + state->avail.notify_one(); + state->request.notify_one(); + }}); + + while (true) { + checkInterrupt(); + + std::string chunk; + + /* Grab data if available, otherwise wait for the download + thread to wake us up. */ + { + auto state(_state->lock()); + + while (state->data.empty()) { + + if (state->quit) { + if (state->exc) std::rethrow_exception(state->exc); + return; + } + + state.wait(state->avail); + } + + chunk = std::move(state->data); + + state->request.notify_one(); + } + + /* Flush the data to the sink and wake up the download thread + if it's blocked on a full buffer. We don't hold the state + lock while doing this to prevent blocking the download + thread if sink() takes a long time. */ + sink((unsigned char *) chunk.data(), chunk.size()); + } +} + +std::string FileTransfer::urlEncode(const std::string & param) { + throw URLEncodeError("not implemented"); +} + +bool isUri(const string & s) +{ + if (s.compare(0, 8, "channel:") == 0) return true; + size_t pos = s.find("://"); + if (pos == string::npos) return false; + string scheme(s, 0, pos); + return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; +} + + +} diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh new file mode 100644 index 00000000000..090a8e21cec --- /dev/null +++ b/src/libstore/filetransfer.hh @@ -0,0 +1,119 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" +#include "config.hh" + +#include +#include + +namespace nix { + +struct FileTransferSettings : Config +{ + Setting enableHttp2{this, true, "http2", + "Whether to enable HTTP/2 support."}; + + Setting userAgentSuffix{this, "", "user-agent-suffix", + "String appended to the user agent in HTTP requests."}; + + Setting httpConnections{this, 25, "http-connections", + "Number of parallel HTTP connections.", + {"binary-caches-parallel-connections"}}; + + Setting connectTimeout{this, 0, "connect-timeout", + "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; + + Setting stalledDownloadTimeout{this, 300, "stalled-download-timeout", + "Timeout (in seconds) for receiving data from servers during download. Nix cancels idle downloads after this timeout's duration."}; + + Setting tries{this, 5, "download-attempts", + "How often Nix will attempt to download a file before giving up."}; +}; + +extern FileTransferSettings fileTransferSettings; + +struct FileTransferRequest +{ + std::string uri; + std::string expectedETag; + bool verifyTLS = true; + bool head = false; + size_t tries = fileTransferSettings.tries; + unsigned int baseRetryTimeMs = 250; + ActivityId parentAct; + bool decompress = true; + std::shared_ptr data; + std::string mimeType; + std::function dataCallback; + + FileTransferRequest(const std::string & uri) + : uri(uri), parentAct(getCurActivity()) { } + + std::string verb() + { + return data ? "upload" : "download"; + } +}; + +struct FileTransferResult +{ + bool cached = false; + std::string etag; + std::string effectiveUri; + std::shared_ptr data; + uint64_t bodySize = 0; +}; + +class Store; + +struct FileTransfer +{ + virtual ~FileTransfer() { } + + /* Enqueue a data transfer request, returning a future to the result of + the download. The future may throw a FileTransferError + exception. */ + virtual void enqueueFileTransfer(const FileTransferRequest & request, + Callback callback) = 0; + + std::future enqueueFileTransfer(const FileTransferRequest & request); + + /* Synchronously download a file. */ + FileTransferResult download(const FileTransferRequest & request); + + /* Synchronously upload a file. */ + FileTransferResult upload(const FileTransferRequest & request); + + /* Download a file, writing its data to a sink. The sink will be + invoked on the thread of the caller. */ + void download(FileTransferRequest && request, Sink & sink); + + virtual std::string urlEncode(const std::string & param); + + enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; +}; + +/* Return a shared FileTransfer object. Using this object is preferred + because it enables connection reuse and HTTP/2 multiplexing. */ +ref getFileTransfer(); + +/* Return a new FileTransfer object. */ +ref makeFileTransfer(); + +class FileTransferError : public Error +{ +public: + FileTransfer::Error error; + template + FileTransferError(FileTransfer::Error error, const Args & ... args) + : Error(args...), error(error) + { } +}; + +bool isUri(const string & s); + +/* Resolve deprecated 'channel:' URLs. */ +std::string resolveUri(const std::string & uri); + +} diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index a67e0775b97..64780a6daf4 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -13,11 +13,14 @@ public: struct Stat { - Type type; - uint64_t fileSize; // regular files only - bool isExecutable; // regular files only + Type type = tMissing; + uint64_t fileSize = 0; // regular files only + bool isExecutable = false; // regular files only + uint64_t narOffset = 0; // regular files only }; + virtual ~FSAccessor() { } + virtual Stat stat(const Path & path) = 0; virtual StringSet readDirectory(const Path & path) = 0; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 8e90913cc3f..04e3849f7f4 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,14 +1,17 @@ #include "derivations.hh" #include "globals.hh" #include "local-store.hh" +#include "finally.hh" #include #include #include #include +#include #include #include +#include #include #include #include @@ -18,7 +21,6 @@ namespace nix { static string gcLockName = "gc.lock"; -static string tempRootsDir = "temproots"; static string gcRootsDir = "gcroots"; @@ -27,19 +29,19 @@ static string gcRootsDir = "gcroots"; read. To be precise: when they try to create a new temporary root file, they will block until the garbage collector has finished / yielded the GC lock. */ -int LocalStore::openGCLock(LockType lockType) +AutoCloseFD LocalStore::openGCLock(LockType lockType) { Path fnGCLock = (format("%1%/%2%") % stateDir % gcLockName).str(); - debug(format("acquiring global GC lock ‘%1%’") % fnGCLock); + debug(format("acquiring global GC lock '%1%'") % fnGCLock); AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); if (!fdGCLock) - throw SysError(format("opening global GC lock ‘%1%’") % fnGCLock); + throw SysError("opening global GC lock '%1%'", fnGCLock); if (!lockFile(fdGCLock.get(), lockType, false)) { - printError(format("waiting for the big garbage collector lock...")); + printInfo("waiting for the big garbage collector lock..."); lockFile(fdGCLock.get(), lockType, true); } @@ -47,7 +49,7 @@ int LocalStore::openGCLock(LockType lockType) process that can open the file for reading can DoS the collector. */ - return fdGCLock.release(); + return fdGCLock; } @@ -58,13 +60,13 @@ static void makeSymlink(const Path & link, const Path & target) /* Create the new symlink. */ Path tempLink = (format("%1%.tmp-%2%-%3%") - % link % getpid() % rand()).str(); + % link % getpid() % random()).str(); createSymlink(target, tempLink); /* Atomically replace the old one. */ if (rename(tempLink.c_str(), link.c_str()) == -1) - throw SysError(format("cannot rename ‘%1%’ to ‘%2%’") - % tempLink % link); + throw SysError("cannot rename '%1%' to '%2%'", + tempLink , link); } @@ -76,31 +78,29 @@ void LocalStore::syncWithGC() void LocalStore::addIndirectRoot(const Path & path) { - string hash = printHash32(hashString(htSHA1, path)); + string hash = hashString(htSHA1, path).to_string(Base32, false); Path realRoot = canonPath((format("%1%/%2%/auto/%3%") % stateDir % gcRootsDir % hash).str()); makeSymlink(realRoot, path); } -Path LocalFSStore::addPermRoot(const Path & _storePath, +Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot, bool indirect, bool allowOutsideRootsDir) { - Path storePath(canonPath(_storePath)); Path gcRoot(canonPath(_gcRoot)); - assertStorePath(storePath); if (isInStore(gcRoot)) - throw Error(format( + throw Error( "creating a garbage collector root (%1%) in the Nix store is forbidden " - "(are you running nix-build inside the store?)") % gcRoot); + "(are you running nix-build inside the store?)", gcRoot); if (indirect) { /* Don't clobber the link if it already exists and doesn't point to the Nix store. */ if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) - throw Error(format("cannot create symlink ‘%1%’; already exists") % gcRoot); - makeSymlink(gcRoot, storePath); + throw Error("cannot create symlink '%1%'; already exists", gcRoot); + makeSymlink(gcRoot, printStorePath(storePath)); addIndirectRoot(gcRoot); } @@ -109,16 +109,16 @@ Path LocalFSStore::addPermRoot(const Path & _storePath, Path rootsDir = canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str()); if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/") - throw Error(format( - "path ‘%1%’ is not a valid garbage collector root; " - "it's not in the directory ‘%2%’") - % gcRoot % rootsDir); + throw Error( + "path '%1%' is not a valid garbage collector root; " + "it's not in the directory '%2%'", + gcRoot, rootsDir); } - if (baseNameOf(gcRoot) == baseNameOf(storePath)) + if (baseNameOf(gcRoot) == std::string(storePath.to_string())) writeFile(gcRoot, ""); else - makeSymlink(gcRoot, storePath); + makeSymlink(gcRoot, printStorePath(storePath)); } /* Check that the root can be found by the garbage collector. @@ -127,13 +127,14 @@ Path LocalFSStore::addPermRoot(const Path & _storePath, check if the root is in a directory in or linked from the gcroots directory. */ if (settings.checkRootReachability) { - Roots roots = findRoots(); - if (roots.find(gcRoot) == roots.end()) - printError( - format( - "warning: ‘%1%’ is not in a directory where the garbage collector looks for roots; " - "therefore, ‘%2%’ might be removed by the garbage collector") - % gcRoot % storePath); + auto roots = findRoots(false); + if (roots[storePath.clone()].count(gcRoot) == 0) + logWarning({ + .name = "GC root", + .hint = hintfmt("warning: '%1%' is not in a directory where the garbage collector looks for roots; " + "therefore, '%2%' might be removed by the garbage collector", + gcRoot, printStorePath(storePath)) + }); } /* Grab the global GC root, causing us to block while a GC is in @@ -145,7 +146,7 @@ Path LocalFSStore::addPermRoot(const Path & _storePath, } -void LocalStore::addTempRoot(const Path & path) +void LocalStore::addTempRoot(const StorePath & path) { auto state(_state.lock()); @@ -153,30 +154,25 @@ void LocalStore::addTempRoot(const Path & path) if (!state->fdTempRoots) { while (1) { - Path dir = (format("%1%/%2%") % stateDir % tempRootsDir).str(); - createDirs(dir); - - state->fnTempRoots = (format("%1%/%2%") % dir % getpid()).str(); - AutoCloseFD fdGCLock = openGCLock(ltRead); - if (pathExists(state->fnTempRoots)) + if (pathExists(fnTempRoots)) /* It *must* be stale, since there can be no two processes with the same pid. */ - unlink(state->fnTempRoots.c_str()); + unlink(fnTempRoots.c_str()); - state->fdTempRoots = openLockFile(state->fnTempRoots, true); + state->fdTempRoots = openLockFile(fnTempRoots, true); fdGCLock = -1; - debug(format("acquiring read lock on ‘%1%’") % state->fnTempRoots); + debug(format("acquiring read lock on '%1%'") % fnTempRoots); lockFile(state->fdTempRoots.get(), ltRead, true); /* Check whether the garbage collector didn't get in our way. */ struct stat st; if (fstat(state->fdTempRoots.get(), &st) == -1) - throw SysError(format("statting ‘%1%’") % state->fnTempRoots); + throw SysError("statting '%1%'", fnTempRoots); if (st.st_size == 0) break; /* The garbage collector deleted this file before we could @@ -188,34 +184,41 @@ void LocalStore::addTempRoot(const Path & path) /* Upgrade the lock to a write lock. This will cause us to block if the garbage collector is holding our lock. */ - debug(format("acquiring write lock on ‘%1%’") % state->fnTempRoots); + debug(format("acquiring write lock on '%1%'") % fnTempRoots); lockFile(state->fdTempRoots.get(), ltWrite, true); - string s = path + '\0'; + string s = printStorePath(path) + '\0'; writeFull(state->fdTempRoots.get(), s); /* Downgrade to a read lock. */ - debug(format("downgrading to read lock on ‘%1%’") % state->fnTempRoots); + debug(format("downgrading to read lock on '%1%'") % fnTempRoots); lockFile(state->fdTempRoots.get(), ltRead, true); } -void LocalStore::readTempRoots(PathSet & tempRoots, FDs & fds) +static std::string censored = "{censored}"; + + +void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) { /* Read the `temproots' directory for per-process temporary root files. */ - DirEntries tempRootFiles = readDirectory( - (format("%1%/%2%") % stateDir % tempRootsDir).str()); + for (auto & i : readDirectory(tempRootsDir)) { + if (i.name[0] == '.') { + // Ignore hidden files. Some package managers (notably portage) create + // those to keep the directory alive. + continue; + } + Path path = tempRootsDir + "/" + i.name; - for (auto & i : tempRootFiles) { - Path path = (format("%1%/%2%/%3%") % stateDir % tempRootsDir % i.name).str(); + pid_t pid = std::stoi(i.name); - debug(format("reading temporary root file ‘%1%’") % path); + debug(format("reading temporary root file '%1%'") % path); FDPtr fd(new AutoCloseFD(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666))); if (!*fd) { /* It's okay if the file has disappeared. */ if (errno == ENOENT) continue; - throw SysError(format("opening temporary roots file ‘%1%’") % path); + throw SysError("opening temporary roots file '%1%'", path); } /* This should work, but doesn't, for some reason. */ @@ -226,7 +229,7 @@ void LocalStore::readTempRoots(PathSet & tempRoots, FDs & fds) only succeed if the owning process has died. In that case we don't care about its temporary roots. */ if (lockFile(fd->get(), ltWrite, false)) { - printError(format("removing stale temporary roots file ‘%1%’") % path); + printInfo("removing stale temporary roots file '%1%'", path); unlink(path.c_str()); writeFull(fd->get(), "d"); continue; @@ -235,7 +238,7 @@ void LocalStore::readTempRoots(PathSet & tempRoots, FDs & fds) /* Acquire a read lock. This will prevent the owning process from upgrading to a write lock, therefore it will block in addTempRoot(). */ - debug(format("waiting for read lock on ‘%1%’") % path); + debug(format("waiting for read lock on '%1%'") % path); lockFile(fd->get(), ltRead, true); /* Read the entire file. */ @@ -246,9 +249,8 @@ void LocalStore::readTempRoots(PathSet & tempRoots, FDs & fds) while ((end = contents.find((char) 0, pos)) != string::npos) { Path root(contents, pos, end - pos); - debug(format("got temporary root ‘%1%’") % root); - assertStorePath(root); - tempRoots.insert(root); + debug("got temporary root '%s'", root); + tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{temp:%d}", pid)); pos = end + 1; } @@ -260,11 +262,11 @@ void LocalStore::readTempRoots(PathSet & tempRoots, FDs & fds) void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) { auto foundRoot = [&](const Path & path, const Path & target) { - Path storePath = toStorePath(target); - if (isStorePath(storePath) && isValidPath(storePath)) - roots[path] = storePath; + auto storePath = maybeParseStorePath(toStorePath(target)); + if (storePath && isValidPath(*storePath)) + roots[std::move(*storePath)].emplace(path); else - printInfo(format("skipping invalid root from ‘%1%’ to ‘%2%’") % path % storePath); + printInfo("skipping invalid root from '%1%' to '%2%'", path, target); }; try { @@ -287,7 +289,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) target = absPath(target, dirOf(path)); if (!pathExists(target)) { if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) { - printInfo(format("removing stale link from ‘%1%’ to ‘%2%’") % path % target); + printInfo(format("removing stale link from '%1%' to '%2%'") % path % target); unlink(path.c_str()); } } else { @@ -300,9 +302,9 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) } else if (type == DT_REG) { - Path storePath = storeDir + "/" + baseNameOf(path); - if (isStorePath(storePath) && isValidPath(storePath)) - roots[path] = storePath; + auto storePath = maybeParseStorePath(storeDir + "/" + std::string(baseNameOf(path))); + if (storePath && isValidPath(*storePath)) + roots[std::move(*storePath)].emplace(path); } } @@ -310,28 +312,40 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) catch (SysError & e) { /* We only ignore permanent failures. */ if (e.errNo == EACCES || e.errNo == ENOENT || e.errNo == ENOTDIR) - printInfo(format("cannot read potential root ‘%1%’") % path); + printInfo("cannot read potential root '%1%'", path); else throw; } } -Roots LocalStore::findRoots() +void LocalStore::findRootsNoTemp(Roots & roots, bool censor) { - Roots roots; - - /* Process direct roots in {gcroots,manifests,profiles}. */ + /* Process direct roots in {gcroots,profiles}. */ findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots); - if (pathExists(stateDir + "/manifests")) - findRoots(stateDir + "/manifests", DT_UNKNOWN, roots); findRoots(stateDir + "/profiles", DT_UNKNOWN, roots); + /* Add additional roots returned by different platforms-specific + heuristics. This is typically used to add running programs to + the set of roots (to prevent them from being garbage collected). */ + findRuntimeRoots(roots, censor); +} + + +Roots LocalStore::findRoots(bool censor) +{ + Roots roots; + findRootsNoTemp(roots, censor); + + FDs fds; + findTempRoots(fds, roots, censor); + return roots; } +typedef std::unordered_map> UncheckedRoots; -static void readProcLink(const string & file, StringSet & paths) +static void readProcLink(const string & file, UncheckedRoots & roots) { /* 64 is the starting buffer size gnu readlink uses... */ auto bufsiz = ssize_t{64}; @@ -339,7 +353,7 @@ static void readProcLink(const string & file, StringSet & paths) char buf[bufsiz]; auto res = readlink(file.c_str(), buf, bufsiz); if (res == -1) { - if (errno == ENOENT || errno == EACCES) + if (errno == ENOENT || errno == EACCES || errno == ESRCH) return; throw SysError("reading symlink"); } @@ -350,8 +364,8 @@ static void readProcLink(const string & file, StringSet & paths) goto try_again; } if (res > 0 && buf[0] == '/') - paths.emplace(static_cast(buf), res); - return; + roots[std::string(static_cast(buf), res)] + .emplace(file); } static string quoteRegexChars(const string & raw) @@ -360,19 +374,20 @@ static string quoteRegexChars(const string & raw) return std::regex_replace(raw, specialRegex, R"(\$&)"); } -static void readFileRoots(const char * path, StringSet & paths) +static void readFileRoots(const char * path, UncheckedRoots & roots) { try { - paths.emplace(readFile(path)); + roots[readFile(path)].emplace(path); } catch (SysError & e) { if (e.errNo != ENOENT && e.errNo != EACCES) throw; } } -void LocalStore::findRuntimeRoots(PathSet & roots) +void LocalStore::findRuntimeRoots(Roots & roots, bool censor) { - StringSet paths; + UncheckedRoots unchecked; + auto procDir = AutoCloseDir{opendir("/proc")}; if (procDir) { struct dirent * ent; @@ -382,41 +397,44 @@ void LocalStore::findRuntimeRoots(PathSet & roots) while (errno = 0, ent = readdir(procDir.get())) { checkInterrupt(); if (std::regex_match(ent->d_name, digitsRegex)) { - readProcLink((format("/proc/%1%/exe") % ent->d_name).str(), paths); - readProcLink((format("/proc/%1%/cwd") % ent->d_name).str(), paths); + readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked); + readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); - auto fdStr = (format("/proc/%1%/fd") % ent->d_name).str(); + auto fdStr = fmt("/proc/%s/fd", ent->d_name); auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); if (!fdDir) { if (errno == ENOENT || errno == EACCES) continue; - throw SysError(format("opening %1%") % fdStr); + throw SysError("opening %1%", fdStr); } struct dirent * fd_ent; while (errno = 0, fd_ent = readdir(fdDir.get())) { - if (fd_ent->d_name[0] != '.') { - readProcLink((format("%1%/%2%") % fdStr % fd_ent->d_name).str(), paths); - } + if (fd_ent->d_name[0] != '.') + readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked); } - if (errno) - throw SysError(format("iterating /proc/%1%/fd") % ent->d_name); - fdDir.reset(); - - auto mapLines = - tokenizeString>(readFile((format("/proc/%1%/maps") % ent->d_name).str(), true), "\n"); - for (const auto& line : mapLines) { - auto match = std::smatch{}; - if (std::regex_match(line, match, mapRegex)) - paths.emplace(match[1]); + if (errno) { + if (errno == ESRCH) + continue; + throw SysError("iterating /proc/%1%/fd", ent->d_name); } + fdDir.reset(); try { - auto envString = readFile((format("/proc/%1%/environ") % ent->d_name).str(), true); + auto mapFile = fmt("/proc/%s/maps", ent->d_name); + auto mapLines = tokenizeString>(readFile(mapFile), "\n"); + for (const auto & line : mapLines) { + auto match = std::smatch{}; + if (std::regex_match(line, match, mapRegex)) + unchecked[match[1]].emplace(mapFile); + } + + auto envFile = fmt("/proc/%s/environ", ent->d_name); + auto envString = readFile(envFile); auto env_end = std::sregex_iterator{}; for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i) - paths.emplace(i->str()); + unchecked[i->str()].emplace(envFile); } catch (SysError & e) { - if (errno == ENOENT || errno == EACCES) + if (errno == ENOENT || errno == EACCES || errno == ESRCH) continue; throw; } @@ -426,31 +444,44 @@ void LocalStore::findRuntimeRoots(PathSet & roots) throw SysError("iterating /proc"); } - try { - auto lsofRegex = std::regex(R"(^n(/.*)$)"); - auto lsofLines = - tokenizeString>(runProgram("lsof", true, { "-n", "-w", "-F", "n" }), "\n"); - for (const auto & line : lsofLines) { - auto match = std::smatch{}; - if (std::regex_match(line, match, lsofRegex)) - paths.emplace(match[1]); - } - } catch (ExecError & e) { - /* lsof not installed, lsof failed */ - } - - readFileRoots("/proc/sys/kernel/modprobe", paths); - readFileRoots("/proc/sys/kernel/fbsplash", paths); - readFileRoots("/proc/sys/kernel/poweroff_cmd", paths); - - for (auto & i : paths) - if (isInStore(i)) { - Path path = toStorePath(i); - if (roots.find(path) == roots.end() && isStorePath(path) && isValidPath(path)) { - debug(format("got additional root ‘%1%’") % path); - roots.insert(path); +#if !defined(__linux__) + // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail. + // See: https://github.com/NixOS/nix/issues/3011 + // Because of this we disable lsof when running the tests. + if (getEnv("_NIX_TEST_NO_LSOF") != "1") { + try { + std::regex lsofRegex(R"(^n(/.*)$)"); + auto lsofLines = + tokenizeString>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); + for (const auto & line : lsofLines) { + std::smatch match; + if (std::regex_match(line, match, lsofRegex)) + unchecked[match[1]].emplace("{lsof}"); } + } catch (ExecError & e) { + /* lsof not installed, lsof failed */ } + } +#endif + +#if defined(__linux__) + readFileRoots("/proc/sys/kernel/modprobe", unchecked); + readFileRoots("/proc/sys/kernel/fbsplash", unchecked); + readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); +#endif + + for (auto & [target, links] : unchecked) { + if (!isInStore(target)) continue; + Path pathS = toStorePath(target); + if (!isStorePath(pathS)) continue; + auto path = parseStorePath(pathS); + if (!isValidPath(path)) continue; + debug("got additional root '%1%'", pathS); + if (censor) + roots[path.clone()].insert(censored); + else + roots[path.clone()].insert(links.begin(), links.end()); + } } @@ -459,18 +490,19 @@ struct GCLimitReached { }; struct LocalStore::GCState { - GCOptions options; + const GCOptions & options; GCResults & results; - PathSet roots; - PathSet tempRoots; - PathSet dead; - PathSet alive; + StorePathSet roots; + StorePathSet tempRoots; + StorePathSet dead; + StorePathSet alive; bool gcKeepOutputs; bool gcKeepDerivations; unsigned long long bytesInvalidated; bool moveToTrash = true; bool shouldDelete; - GCState(GCResults & results_) : results(results_), bytesInvalidated(0) { } + GCState(const GCOptions & options, GCResults & results) + : options(options), results(results), bytesInvalidated(0) { } }; @@ -478,7 +510,7 @@ bool LocalStore::isActiveTempFile(const GCState & state, const Path & path, const string & suffix) { return hasSuffix(path, suffix) - && state.tempRoots.find(string(path, 0, path.size() - suffix.size())) != state.tempRoots.end(); + && state.tempRoots.count(parseStorePath(string(path, 0, path.size() - suffix.size()))); } @@ -496,24 +528,25 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) unsigned long long size = 0; - if (isStorePath(path) && isValidPath(path)) { - PathSet referrers; - queryReferrers(path, referrers); + auto storePath = maybeParseStorePath(path); + if (storePath && isValidPath(*storePath)) { + StorePathSet referrers; + queryReferrers(*storePath, referrers); for (auto & i : referrers) - if (i != path) deletePathRecursive(state, i); - size = queryPathInfo(path)->narSize; - invalidatePathChecked(path); + if (printStorePath(i) != path) deletePathRecursive(state, printStorePath(i)); + size = queryPathInfo(*storePath)->narSize; + invalidatePathChecked(*storePath); } - Path realPath = realStoreDir + "/" + baseNameOf(path); + Path realPath = realStoreDir + "/" + std::string(baseNameOf(path)); struct stat st; if (lstat(realPath.c_str(), &st)) { if (errno == ENOENT) return; - throw SysError(format("getting status of %1%") % realPath); + throw SysError("getting status of %1%", realPath); } - printInfo(format("deleting ‘%1%’") % path); + printInfo(format("deleting '%1%'") % path); state.results.paths.insert(path); @@ -528,14 +561,14 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) // size. try { if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1) - throw SysError(format("making ‘%1%’ writable") % realPath); - Path tmp = trashDir + "/" + baseNameOf(path); + throw SysError("making '%1%' writable", realPath); + Path tmp = trashDir + "/" + std::string(baseNameOf(path)); if (rename(realPath.c_str(), tmp.c_str())) - throw SysError(format("unable to rename ‘%1%’ to ‘%2%’") % realPath % tmp); + throw SysError("unable to rename '%1%' to '%2%'", realPath, tmp); state.bytesInvalidated += size; } catch (SysError & e) { if (e.errNo == ENOSPC) { - printInfo(format("note: can't create move ‘%1%’: %2%") % realPath % e.msg()); + printInfo(format("note: can't create move '%1%': %2%") % realPath % e.msg()); deleteGarbage(state, realPath); } } @@ -549,54 +582,49 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) } -bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & path) +bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path) { - if (visited.find(path) != visited.end()) return false; + if (visited.count(path)) return false; - if (state.alive.find(path) != state.alive.end()) { - return true; - } + if (state.alive.count(path)) return true; - if (state.dead.find(path) != state.dead.end()) { - return false; - } + if (state.dead.count(path)) return false; - if (state.roots.find(path) != state.roots.end()) { - debug(format("cannot delete ‘%1%’ because it's a root") % path); - state.alive.insert(path); + if (state.roots.count(path)) { + debug("cannot delete '%1%' because it's a root", printStorePath(path)); + state.alive.insert(path.clone()); return true; } - visited.insert(path); + visited.insert(path.clone()); - if (!isStorePath(path) || !isValidPath(path)) return false; + if (!isValidPath(path)) return false; - PathSet incoming; + StorePathSet incoming; /* Don't delete this path if any of its referrers are alive. */ queryReferrers(path, incoming); - /* If gc-keep-derivations is set and this is a derivation, then + /* If keep-derivations is set and this is a derivation, then don't delete the derivation if any of the outputs are alive. */ - if (state.gcKeepDerivations && isDerivation(path)) { - PathSet outputs = queryDerivationOutputs(path); - for (auto & i : outputs) + if (state.gcKeepDerivations && path.isDerivation()) { + for (auto & i : queryDerivationOutputs(path)) if (isValidPath(i) && queryPathInfo(i)->deriver == path) - incoming.insert(i); + incoming.insert(i.clone()); } - /* If gc-keep-outputs is set, then don't delete this path if there + /* If keep-outputs is set, then don't delete this path if there are derivers of this path that are not garbage. */ if (state.gcKeepOutputs) { - PathSet derivers = queryValidDerivers(path); + auto derivers = queryValidDerivers(path); for (auto & i : derivers) - incoming.insert(i); + incoming.insert(i.clone()); } for (auto & i : incoming) if (i != path) if (canReachRoot(state, visited, i)) { - state.alive.insert(path); + state.alive.insert(path.clone()); return true; } @@ -608,12 +636,14 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) { checkInterrupt(); - auto realPath = realStoreDir + "/" + baseNameOf(path); + auto realPath = realStoreDir + "/" + std::string(baseNameOf(path)); if (realPath == linksDir || realPath == trashDir) return; - Activity act(*logger, lvlDebug, format("considering whether to delete ‘%1%’") % path); + //Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path); + + auto storePath = maybeParseStorePath(path); - if (!isStorePath(path) || !isValidPath(path)) { + if (!storePath || !isValidPath(*storePath)) { /* A lock file belonging to a path that we're building right now isn't garbage. */ if (isActiveTempFile(state, path, ".lock")) return; @@ -628,16 +658,17 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) if (isActiveTempFile(state, path, ".check")) return; } - PathSet visited; + StorePathSet visited; - if (canReachRoot(state, visited, path)) { - debug(format("cannot delete ‘%1%’ because it's still reachable") % path); + if (storePath && canReachRoot(state, visited, *storePath)) { + debug("cannot delete '%s' because it's still reachable", path); } else { /* No path we visited was a root, so everything is garbage. But we only delete ‘path’ and its referrers here so that ‘nix-store --delete’ doesn't have the unexpected effect of recursing into derivations and outputs. */ - state.dead.insert(visited.begin(), visited.end()); + for (auto & i : visited) + state.dead.insert(i.clone()); if (state.shouldDelete) deletePathRecursive(state, path); } @@ -652,7 +683,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) void LocalStore::removeUnusedLinks(const GCState & state) { AutoCloseDir dir(opendir(linksDir.c_str())); - if (!dir) throw SysError(format("opening directory ‘%1%’") % linksDir); + if (!dir) throw SysError("opening directory '%1%'", linksDir); long long actualSize = 0, unsharedSize = 0; @@ -665,26 +696,25 @@ void LocalStore::removeUnusedLinks(const GCState & state) struct stat st; if (lstat(path.c_str(), &st) == -1) - throw SysError(format("statting ‘%1%’") % path); + throw SysError("statting '%1%'", path); if (st.st_nlink != 1) { - unsigned long long size = st.st_blocks * 512ULL; - actualSize += size; - unsharedSize += (st.st_nlink - 1) * size; + actualSize += st.st_size; + unsharedSize += (st.st_nlink - 1) * st.st_size; continue; } - printMsg(lvlTalkative, format("deleting unused link ‘%1%’") % path); + printMsg(lvlTalkative, format("deleting unused link '%1%'") % path); if (unlink(path.c_str()) == -1) - throw SysError(format("deleting ‘%1%’") % path); + throw SysError("deleting '%1%'", path); - state.results.bytesFreed += st.st_blocks * 512; + state.results.bytesFreed += st.st_size; } struct stat st; if (stat(linksDir.c_str(), &st) == -1) - throw SysError(format("statting ‘%1%’") % linksDir); + throw SysError("statting '%1%'", linksDir); long long overhead = st.st_blocks * 512ULL; printInfo(format("note: currently hard linking saves %.2f MiB") @@ -694,15 +724,14 @@ void LocalStore::removeUnusedLinks(const GCState & state) void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) { - GCState state(results); - state.options = options; + GCState state(options, results); state.gcKeepOutputs = settings.gcKeepOutputs; state.gcKeepDerivations = settings.gcKeepDerivations; /* Using `--ignore-liveness' with `--delete' can have unintended - consequences if `gc-keep-outputs' or `gc-keep-derivations' are - true (the garbage collector will recurse into deleting the - outputs or derivers, respectively). So disable them. */ + consequences if `keep-outputs' or `keep-derivations' are true + (the garbage collector will recurse into deleting the outputs + or derivers, respectively). So disable them. */ if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { state.gcKeepOutputs = false; state.gcKeepDerivations = false; @@ -720,24 +749,23 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Find the roots. Since we've grabbed the GC lock, the set of permanent roots cannot increase now. */ - printError(format("finding garbage collector roots...")); - Roots rootMap = options.ignoreLiveness ? Roots() : findRoots(); - - for (auto & i : rootMap) state.roots.insert(i.second); - - /* Add additional roots returned by the program specified by the - NIX_ROOT_FINDER environment variable. This is typically used - to add running programs to the set of roots (to prevent them - from being garbage collected). */ + printInfo("finding garbage collector roots..."); + Roots rootMap; if (!options.ignoreLiveness) - findRuntimeRoots(state.roots); + findRootsNoTemp(rootMap, true); + + for (auto & i : rootMap) state.roots.insert(i.first.clone()); /* Read the temporary roots. This acquires read locks on all per-process temporary root files. So after this point no paths can be added to the set of temporary roots. */ FDs fds; - readTempRoots(state.tempRoots, fds); - state.roots.insert(state.tempRoots.begin(), state.tempRoots.end()); + Roots tempRoots; + findTempRoots(fds, tempRoots, true); + for (auto & root : tempRoots) { + state.tempRoots.insert(root.first.clone()); + state.roots.insert(root.first.clone()); + } /* After this point the set of roots or temporary roots cannot increase, since we hold locks on everything. So everything @@ -749,7 +777,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) createDirs(trashDir); } catch (SysError & e) { if (e.errNo == ENOSPC) { - printInfo(format("note: can't create trash directory: %1%") % e.msg()); + printInfo("note: can't create trash directory: %s", e.msg()); state.moveToTrash = false; } } @@ -761,23 +789,26 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) if (options.action == GCOptions::gcDeleteSpecific) { for (auto & i : options.pathsToDelete) { - assertStorePath(i); - tryToDelete(state, i); + tryToDelete(state, printStorePath(i)); if (state.dead.find(i) == state.dead.end()) - throw Error(format("cannot delete path ‘%1%’ since it is still alive") % i); + throw Error( + "cannot delete path '%1%' since it is still alive. " + "To find out why use: " + "nix-store --query --roots", + printStorePath(i)); } } else if (options.maxFreed > 0) { if (state.shouldDelete) - printError(format("deleting garbage...")); + printInfo("deleting garbage..."); else - printError(format("determining live/dead paths...")); + printInfo("determining live/dead paths..."); try { AutoCloseDir dir(opendir(realStoreDir.c_str())); - if (!dir) throw SysError(format("opening directory ‘%1%’") % realStoreDir); + if (!dir) throw SysError("opening directory '%1%'", realStoreDir); /* Read the store and immediately delete all paths that aren't valid. When using --max-freed etc., deleting @@ -792,7 +823,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) string name = dirent->d_name; if (name == "." || name == "..") continue; Path path = storeDir + "/" + name; - if (isStorePath(path) && isValidPath(path)) + auto storePath = maybeParseStorePath(path); + if (storePath && isValidPath(*storePath)) entries.push_back(path); else tryToDelete(state, path); @@ -806,7 +838,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) alphabetically first (e.g. /nix/store/000...). This matters when using --max-freed etc. */ vector entries_(entries.begin(), entries.end()); - random_shuffle(entries_.begin(), entries_.end()); + std::mt19937 gen(1); + std::shuffle(entries_.begin(), entries_.end(), gen); for (auto & i : entries_) tryToDelete(state, i); @@ -816,12 +849,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } if (state.options.action == GCOptions::gcReturnLive) { - state.results.paths = state.alive; + for (auto & i : state.alive) + state.results.paths.insert(printStorePath(i)); return; } if (state.options.action == GCOptions::gcReturnDead) { - state.results.paths = state.dead; + for (auto & i : state.dead) + state.results.paths.insert(printStorePath(i)); return; } @@ -830,12 +865,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) fds.clear(); /* Delete the trash directory. */ - printInfo(format("deleting ‘%1%’") % trashDir); + printInfo(format("deleting '%1%'") % trashDir); deleteGarbage(state, trashDir); /* Clean up the links directory. */ if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) { - printError(format("deleting unused links...")); + printInfo("deleting unused links..."); removeUnusedLinks(state); } @@ -844,4 +879,85 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } +void LocalStore::autoGC(bool sync) +{ + static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE"); + + auto getAvail = [this]() -> uint64_t { + if (fakeFreeSpaceFile) + return std::stoll(readFile(*fakeFreeSpaceFile)); + + struct statvfs st; + if (statvfs(realStoreDir.c_str(), &st)) + throw SysError("getting filesystem info about '%s'", realStoreDir); + + return (uint64_t) st.f_bavail * st.f_frsize; + }; + + std::shared_future future; + + { + auto state(_state.lock()); + + if (state->gcRunning) { + future = state->gcFuture; + debug("waiting for auto-GC to finish"); + goto sync; + } + + auto now = std::chrono::steady_clock::now(); + + if (now < state->lastGCCheck + std::chrono::seconds(settings.minFreeCheckInterval)) return; + + auto avail = getAvail(); + + state->lastGCCheck = now; + + if (avail >= settings.minFree || avail >= settings.maxFree) return; + + if (avail > state->availAfterGC * 0.97) return; + + state->gcRunning = true; + + std::promise promise; + future = state->gcFuture = promise.get_future().share(); + + std::thread([promise{std::move(promise)}, this, avail, getAvail]() mutable { + + try { + + /* Wake up any threads waiting for the auto-GC to finish. */ + Finally wakeup([&]() { + auto state(_state.lock()); + state->gcRunning = false; + state->lastGCCheck = std::chrono::steady_clock::now(); + promise.set_value(); + }); + + GCOptions options; + options.maxFreed = settings.maxFree - avail; + + printInfo("running auto-GC to free %d bytes", options.maxFreed); + + GCResults results; + + collectGarbage(options, results); + + _state.lock()->availAfterGC = getAvail(); + + } catch (...) { + // FIXME: we could propagate the exception to the + // future, but we don't really care. + ignoreException(); + } + + }).detach(); + } + + sync: + // Wait for the future outside of the state lock. + if (sync) future.get(); +} + + } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 00b46889252..bee94cbd88c 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -1,12 +1,13 @@ -#include "config.h" - #include "globals.hh" #include "util.hh" #include "archive.hh" +#include "args.hh" -#include #include -#include +#include +#include +#include +#include namespace nix { @@ -19,252 +20,208 @@ namespace nix { must be deleted and recreated on startup.) */ #define DEFAULT_SOCKET_PATH "/daemon-socket/socket" - Settings settings; +static GlobalConfig::Register r1(&settings); Settings::Settings() + : nixPrefix(NIX_PREFIX) + , nixStore(canonPath(getEnv("NIX_STORE_DIR").value_or(getEnv("NIX_STORE").value_or(NIX_STORE_DIR)))) + , nixDataDir(canonPath(getEnv("NIX_DATA_DIR").value_or(NIX_DATA_DIR))) + , nixLogDir(canonPath(getEnv("NIX_LOG_DIR").value_or(NIX_LOG_DIR))) + , nixStateDir(canonPath(getEnv("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) + , nixConfDir(canonPath(getEnv("NIX_CONF_DIR").value_or(NIX_CONF_DIR))) + , nixUserConfFiles(getUserConfigFiles()) + , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR))) + , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR))) + , nixManDir(canonPath(NIX_MAN_DIR)) + , nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH)) { - keepFailed = false; - keepGoing = false; - tryFallback = false; - maxBuildJobs = 1; - buildCores = 1; -#ifdef _SC_NPROCESSORS_ONLN - long res = sysconf(_SC_NPROCESSORS_ONLN); - if (res > 0) buildCores = res; -#endif - readOnlyMode = false; - thisSystem = SYSTEM; - maxSilentTime = 0; - buildTimeout = 0; - useBuildHook = true; - reservedSize = 8 * 1024 * 1024; - fsyncMetadata = true; - useSQLiteWAL = true; - syncBeforeRegistering = false; - useSubstitutes = true; buildUsersGroup = getuid() == 0 ? "nixbld" : ""; - useSshSubstituter = true; - impersonateLinux26 = false; - keepLog = true; - compressLog = true; - maxLogSize = 0; - pollInterval = 5; - checkRootReachability = false; - gcKeepOutputs = false; - gcKeepDerivations = true; - autoOptimiseStore = false; - envKeepDerivations = false; - lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1"; - showTrace = false; - enableImportNative = false; -} + lockCPU = getEnv("NIX_AFFINITY_HACK") == "1"; + + caFile = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); + if (caFile == "") { + for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) + if (pathExists(fn)) { + caFile = fn; + break; + } + } + /* Backwards compatibility. */ + auto s = getEnv("NIX_REMOTE_SYSTEMS"); + if (s) { + Strings ss; + for (auto & p : tokenizeString(*s, ":")) + ss.push_back("@" + p); + builders = concatStringsSep(" ", ss); + } -void Settings::processEnvironment() -{ - nixPrefix = NIX_PREFIX; - nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR))); - nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR)); - nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR)); - nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR)); - nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR)); - nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR)); - nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR)); - nixDaemonSocketFile = canonPath(nixStateDir + DEFAULT_SOCKET_PATH); - - // should be set with the other config options, but depends on nixLibexecDir -#ifdef __APPLE__ - preBuildHook = nixLibexecDir + "/nix/resolve-system-dependencies"; +#if defined(__linux__) && defined(SANDBOX_SHELL) + sandboxPaths = tokenizeString("/bin/sh=" SANDBOX_SHELL); #endif -} - -void Settings::loadConfFile() -{ - Path settingsFile = (format("%1%/%2%") % nixConfDir % "nix.conf").str(); - if (!pathExists(settingsFile)) return; - string contents = readFile(settingsFile); - - unsigned int pos = 0; - - while (pos < contents.size()) { - string line; - while (pos < contents.size() && contents[pos] != '\n') - line += contents[pos++]; - pos++; - string::size_type hash = line.find('#'); - if (hash != string::npos) - line = string(line, 0, hash); - - vector tokens = tokenizeString >(line); - if (tokens.empty()) continue; +/* chroot-like behavior from Apple's sandbox */ +#if __APPLE__ + sandboxPaths = tokenizeString("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /bin/bash /private/tmp /private/var/tmp /usr/lib"); + allowedImpureHostPrefixes = tokenizeString("/System/Library /usr/lib /dev /bin/sh"); +#endif +} - if (tokens.size() < 2 || tokens[1] != "=") - throw Error(format("illegal configuration line ‘%1%’ in ‘%2%’") % line % settingsFile); +void loadConfFile() +{ + globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf"); - string name = tokens[0]; + /* We only want to send overrides to the daemon, i.e. stuff from + ~/.nix/nix.conf or the command line. */ + globalConfig.resetOverriden(); - vector::iterator i = tokens.begin(); - advance(i, 2); - settings[name] = concatStringsSep(" ", Strings(i, tokens.end())); // FIXME: slow - }; + auto files = settings.nixUserConfFiles; + for (auto file = files.rbegin(); file != files.rend(); file++) { + globalConfig.applyConfigFile(*file); + } } - -void Settings::set(const string & name, const string & value) +std::vector getUserConfigFiles() { - settings[name] = value; - overrides[name] = value; -} + // Use the paths specified in NIX_USER_CONF_FILES if it has been defined + auto nixConfFiles = getEnv("NIX_USER_CONF_FILES"); + if (nixConfFiles.has_value()) { + return tokenizeString>(nixConfFiles.value(), ":"); + } + // Use the paths specified by the XDG spec + std::vector files; + auto dirs = getConfigDirs(); + for (auto & dir : dirs) { + files.insert(files.end(), dir + "/nix/nix.conf"); + } + return files; +} -string Settings::get(const string & name, const string & def) +unsigned int Settings::getDefaultCores() { - auto i = settings.find(name); - if (i == settings.end()) return def; - return i->second; + return std::max(1U, std::thread::hardware_concurrency()); } - -Strings Settings::get(const string & name, const Strings & def) +StringSet Settings::getDefaultSystemFeatures() { - auto i = settings.find(name); - if (i == settings.end()) return def; - return tokenizeString(i->second); -} + /* For backwards compatibility, accept some "features" that are + used in Nixpkgs to route builds to certain machines but don't + actually require anything special on the machines. */ + StringSet features{"nixos-test", "benchmark", "big-parallel", "recursive-nix"}; + #if __linux__ + if (access("/dev/kvm", R_OK | W_OK) == 0) + features.insert("kvm"); + #endif -bool Settings::get(const string & name, bool def) -{ - bool res = def; - _get(res, name); - return res; + return features; } - -int Settings::get(const string & name, int def) +bool Settings::isExperimentalFeatureEnabled(const std::string & name) { - int res = def; - _get(res, name); - return res; + auto & f = experimentalFeatures.get(); + return std::find(f.begin(), f.end(), name) != f.end(); } - -void Settings::update() +void Settings::requireExperimentalFeature(const std::string & name) { - _get(tryFallback, "build-fallback"); - _get(maxBuildJobs, "build-max-jobs"); - _get(buildCores, "build-cores"); - _get(thisSystem, "system"); - _get(maxSilentTime, "build-max-silent-time"); - _get(buildTimeout, "build-timeout"); - _get(reservedSize, "gc-reserved-space"); - _get(fsyncMetadata, "fsync-metadata"); - _get(useSQLiteWAL, "use-sqlite-wal"); - _get(syncBeforeRegistering, "sync-before-registering"); - _get(useSubstitutes, "build-use-substitutes"); - _get(buildUsersGroup, "build-users-group"); - _get(impersonateLinux26, "build-impersonate-linux-26"); - _get(keepLog, "build-keep-log"); - _get(compressLog, "build-compress-log"); - _get(maxLogSize, "build-max-log-size"); - _get(pollInterval, "build-poll-interval"); - _get(checkRootReachability, "gc-check-reachability"); - _get(gcKeepOutputs, "gc-keep-outputs"); - _get(gcKeepDerivations, "gc-keep-derivations"); - _get(autoOptimiseStore, "auto-optimise-store"); - _get(envKeepDerivations, "env-keep-derivations"); - _get(sshSubstituterHosts, "ssh-substituter-hosts"); - _get(useSshSubstituter, "use-ssh-substituter"); - _get(logServers, "log-servers"); - _get(enableImportNative, "allow-unsafe-native-code-during-evaluation"); - _get(useCaseHack, "use-case-hack"); - _get(preBuildHook, "pre-build-hook"); - _get(keepGoing, "keep-going"); - _get(keepFailed, "keep-failed"); + if (!isExperimentalFeatureEnabled(name)) + throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name); } - -void Settings::_get(string & res, const string & name) +bool Settings::isWSL1() { - SettingsMap::iterator i = settings.find(name); - if (i == settings.end()) return; - res = i->second; + struct utsname utsbuf; + uname(&utsbuf); + // WSL1 uses -Microsoft suffix + // WSL2 uses -microsoft-standard suffix + return hasSuffix(utsbuf.release, "-Microsoft"); } +const string nixVersion = PACKAGE_VERSION; -void Settings::_get(bool & res, const string & name) +template<> void BaseSetting::set(const std::string & str) { - SettingsMap::iterator i = settings.find(name); - if (i == settings.end()) return; - if (i->second == "true") res = true; - else if (i->second == "false") res = false; - else throw Error(format("configuration option ‘%1%’ should be either ‘true’ or ‘false’, not ‘%2%’") - % name % i->second); + if (str == "true") value = smEnabled; + else if (str == "relaxed") value = smRelaxed; + else if (str == "false") value = smDisabled; + else throw UsageError("option '%s' has invalid value '%s'", name, str); } - -void Settings::_get(StringSet & res, const string & name) +template<> std::string BaseSetting::to_string() const { - SettingsMap::iterator i = settings.find(name); - if (i == settings.end()) return; - res.clear(); - Strings ss = tokenizeString(i->second); - res.insert(ss.begin(), ss.end()); + if (value == smEnabled) return "true"; + else if (value == smRelaxed) return "relaxed"; + else if (value == smDisabled) return "false"; + else abort(); } -void Settings::_get(Strings & res, const string & name) +template<> void BaseSetting::toJSON(JSONPlaceholder & out) { - SettingsMap::iterator i = settings.find(name); - if (i == settings.end()) return; - res = tokenizeString(i->second); + AbstractSetting::toJSON(out); } - -template void Settings::_get(N & res, const string & name) +template<> void BaseSetting::convertToArg(Args & args, const std::string & category) { - SettingsMap::iterator i = settings.find(name); - if (i == settings.end()) return; - if (!string2Int(i->second, res)) - throw Error(format("configuration setting ‘%1%’ should have an integer value") % name); + args.addFlag({ + .longName = name, + .description = "Enable sandboxing.", + .category = category, + .handler = {[=]() { override(smEnabled); }} + }); + args.addFlag({ + .longName = "no-" + name, + .description = "Disable sandboxing.", + .category = category, + .handler = {[=]() { override(smDisabled); }} + }); + args.addFlag({ + .longName = "relaxed-" + name, + .description = "Enable sandboxing, but allow builds to disable it.", + .category = category, + .handler = {[=]() { override(smRelaxed); }} + }); } - -string Settings::pack() +void MaxBuildJobsSetting::set(const std::string & str) { - string s; - for (auto & i : settings) { - if (i.first.find('\n') != string::npos || - i.first.find('=') != string::npos || - i.second.find('\n') != string::npos) - throw Error("illegal option name/value"); - s += i.first; s += '='; s += i.second; s += '\n'; - } - return s; + if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency()); + else if (!string2Int(str, value)) + throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); } -void Settings::unpack(const string & pack) { - Strings lines = tokenizeString(pack, "\n"); - for (auto & i : lines) { - string::size_type eq = i.find('='); - if (eq == string::npos) - throw Error("illegal option name/value"); - set(i.substr(0, eq), i.substr(eq + 1)); +void initPlugins() +{ + for (const auto & pluginFile : settings.pluginFiles.get()) { + Paths pluginFiles; + try { + auto ents = readDirectory(pluginFile); + for (const auto & ent : ents) + pluginFiles.emplace_back(pluginFile + "/" + ent.name); + } catch (SysError & e) { + if (e.errNo != ENOTDIR) + throw; + pluginFiles.emplace_back(pluginFile); + } + for (const auto & file : pluginFiles) { + /* handle is purposefully leaked as there may be state in the + DSO needed by the action of the plugin. */ + void *handle = + dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL); + if (!handle) + throw Error("could not dynamically open plugin file '%s': %s", file, dlerror()); + } } -} - -Settings::SettingsMap Settings::getOverrides() -{ - return overrides; + /* Since plugins can add settings, try to re-apply previously + unknown settings. */ + globalConfig.reapplyUnknownSettings(); + globalConfig.warnUnknownSettings(); } - -const string nixVersion = PACKAGE_VERSION; - - } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index a423b4e5c0f..2fbcafff87b 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1,214 +1,389 @@ #pragma once #include "types.hh" -#include "logging.hh" +#include "config.hh" +#include "util.hh" #include -#include +#include +#include namespace nix { +typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode; + +struct MaxBuildJobsSetting : public BaseSetting +{ + MaxBuildJobsSetting(Config * options, + unsigned int def, + const std::string & name, + const std::string & description, + const std::set & aliases = {}) + : BaseSetting(def, name, description, aliases) + { + options->addSetting(this); + } + + void set(const std::string & str) override; +}; -struct Settings { - - typedef std::map SettingsMap; - - Settings(); - - void processEnvironment(); - - void loadConfFile(); - - void set(const string & name, const string & value); - - string get(const string & name, const string & def); - - Strings get(const string & name, const Strings & def); +class Settings : public Config { - bool get(const string & name, bool def); + unsigned int getDefaultCores(); - int get(const string & name, int def); + StringSet getDefaultSystemFeatures(); - void update(); + bool isWSL1(); - string pack(); +public: - void unpack(const string & pack); + Settings(); - SettingsMap getOverrides(); + Path nixPrefix; /* The directory where we store sources and derived files. */ Path nixStore; Path nixDataDir; /* !!! fix */ - Path nixPrefix; - /* The directory where we log various operations. */ Path nixLogDir; /* The directory where state is stored. */ Path nixStateDir; - /* The directory where configuration files are stored. */ + /* The directory where system configuration files are stored. */ Path nixConfDir; + /* A list of user configuration files to load. */ + std::vector nixUserConfFiles; + /* The directory where internal helper programs are stored. */ Path nixLibexecDir; /* The directory where the main programs are stored. */ Path nixBinDir; + /* The directory where the man pages are stored. */ + Path nixManDir; + /* File name of the socket the daemon listens to. */ Path nixDaemonSocketFile; - /* Whether to keep temporary directories of failed builds. */ - bool keepFailed; + Setting storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store", + "The default Nix store to use."}; + + Setting keepFailed{this, false, "keep-failed", + "Whether to keep temporary directories of failed builds."}; - /* Whether to keep building subgoals when a sibling (another - subgoal of the same goal) fails. */ - bool keepGoing; + Setting keepGoing{this, false, "keep-going", + "Whether to keep building derivations when another build fails."}; - /* Whether, if we cannot realise the known closure corresponding - to a derivation, we should try to normalise the derivation - instead. */ - bool tryFallback; + Setting tryFallback{this, false, "fallback", + "Whether to fall back to building when substitution fails.", + {"build-fallback"}}; /* Whether to show build log output in real time. */ bool verboseBuild = true; - /* If verboseBuild is false, the number of lines of the tail of - the log to show if a build fails. */ - size_t logLines = 10; + Setting logLines{this, 10, "log-lines", + "If verbose-build is false, the number of lines of the tail of " + "the log to show if a build fails."}; - /* Maximum number of parallel build jobs. 0 means unlimited. */ - unsigned int maxBuildJobs; + MaxBuildJobsSetting maxBuildJobs{this, 1, "max-jobs", + "Maximum number of parallel build jobs. \"auto\" means use number of cores.", + {"build-max-jobs"}}; - /* Number of CPU cores to utilize in parallel within a build, - i.e. by passing this number to Make via '-j'. 0 means that the - number of actual CPU cores on the local host ought to be - auto-detected. */ - unsigned int buildCores; + Setting buildCores{this, getDefaultCores(), "cores", + "Number of CPU cores to utilize in parallel within a build, " + "i.e. by passing this number to Make via '-j'. 0 means that the " + "number of actual CPU cores on the local host ought to be " + "auto-detected.", {"build-cores"}}; /* Read-only mode. Don't copy stuff to the store, don't change the database. */ - bool readOnlyMode; + bool readOnlyMode = false; - /* The canonical system name, as returned by config.guess. */ - string thisSystem; + Setting thisSystem{this, SYSTEM, "system", + "The canonical Nix system name."}; - /* The maximum time in seconds that a builer can go without - producing any output on stdout/stderr before it is killed. 0 - means infinity. */ - time_t maxSilentTime; + Setting maxSilentTime{this, 0, "max-silent-time", + "The maximum time in seconds that a builer can go without " + "producing any output on stdout/stderr before it is killed. " + "0 means infinity.", + {"build-max-silent-time"}}; - /* The maximum duration in seconds that a builder can run. 0 - means infinity. */ - time_t buildTimeout; + Setting buildTimeout{this, 0, "timeout", + "The maximum duration in seconds that a builder can run. " + "0 means infinity.", {"build-timeout"}}; - /* Whether to use build hooks (for distributed builds). Sometimes - users want to disable this from the command-line. */ - bool useBuildHook; + PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook", + "The path of the helper program that executes builds to remote machines."}; - /* Amount of reserved space for the garbage collector - (/nix/var/nix/db/reserved). */ - off_t reservedSize; + Setting builders{this, "@" + nixConfDir + "/machines", "builders", + "A semicolon-separated list of build machines, in the format of nix.machines."}; - /* Whether SQLite should use fsync. */ - bool fsyncMetadata; + Setting buildersUseSubstitutes{this, false, "builders-use-substitutes", + "Whether build machines should use their own substitutes for obtaining " + "build dependencies if possible, rather than waiting for this host to " + "upload them."}; - /* Whether SQLite should use WAL mode. */ - bool useSQLiteWAL; + Setting reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space", + "Amount of reserved disk space for the garbage collector."}; - /* Whether to call sync() before registering a path as valid. */ - bool syncBeforeRegistering; + Setting fsyncMetadata{this, true, "fsync-metadata", + "Whether SQLite should use fsync()."}; - /* Whether to use substitutes. */ - bool useSubstitutes; + Setting useSQLiteWAL{this, !isWSL1(), "use-sqlite-wal", + "Whether SQLite should use WAL mode."}; - /* The Unix group that contains the build users. */ - string buildUsersGroup; + Setting syncBeforeRegistering{this, false, "sync-before-registering", + "Whether to call sync() before registering a path as valid."}; - /* Set of ssh connection strings for the ssh substituter */ - Strings sshSubstituterHosts; + Setting useSubstitutes{this, true, "substitute", + "Whether to use substitutes.", + {"build-use-substitutes"}}; - /* Whether to use the ssh substituter at all */ - bool useSshSubstituter; + Setting buildUsersGroup{this, "", "build-users-group", + "The Unix group that contains the build users."}; - /* Whether to impersonate a Linux 2.6 machine on newer kernels. */ - bool impersonateLinux26; + Setting impersonateLinux26{this, false, "impersonate-linux-26", + "Whether to impersonate a Linux 2.6 machine on newer kernels.", + {"build-impersonate-linux-26"}}; - /* Whether to store build logs. */ - bool keepLog; + Setting keepLog{this, true, "keep-build-log", + "Whether to store build logs.", + {"build-keep-log"}}; - /* Whether to compress logs. */ - bool compressLog; + Setting compressLog{this, true, "compress-build-log", + "Whether to compress logs.", + {"build-compress-log"}}; - /* Maximum number of bytes a builder can write to stdout/stderr - before being killed (0 means no limit). */ - unsigned long maxLogSize; + Setting maxLogSize{this, 0, "max-build-log-size", + "Maximum number of bytes a builder can write to stdout/stderr " + "before being killed (0 means no limit).", + {"build-max-log-size"}}; - /* When build-repeat > 0 and verboseBuild == true, whether to - print repeated builds (i.e. builds other than the first one) to + /* When buildRepeat > 0 and verboseBuild == true, whether to print + repeated builds (i.e. builds other than the first one) to stderr. Hack to prevent Hydra logs from being polluted. */ bool printRepeatedBuilds = true; - /* How often (in seconds) to poll for locks. */ - unsigned int pollInterval; + Setting pollInterval{this, 5, "build-poll-interval", + "How often (in seconds) to poll for locks."}; - /* Whether to check if new GC roots can in fact be found by the - garbage collector. */ - bool checkRootReachability; + Setting checkRootReachability{this, false, "gc-check-reachability", + "Whether to check if new GC roots can in fact be found by the " + "garbage collector."}; - /* Whether the garbage collector should keep outputs of live - derivations. */ - bool gcKeepOutputs; + Setting gcKeepOutputs{this, false, "keep-outputs", + "Whether the garbage collector should keep outputs of live derivations.", + {"gc-keep-outputs"}}; - /* Whether the garbage collector should keep derivers of live - paths. */ - bool gcKeepDerivations; + Setting gcKeepDerivations{this, true, "keep-derivations", + "Whether the garbage collector should keep derivers of live paths.", + {"gc-keep-derivations"}}; - /* Whether to automatically replace files with identical contents - with hard links. */ - bool autoOptimiseStore; + Setting autoOptimiseStore{this, false, "auto-optimise-store", + "Whether to automatically replace files with identical contents with hard links."}; - /* Whether to add derivations as a dependency of user environments - (to prevent them from being GCed). */ - bool envKeepDerivations; + Setting envKeepDerivations{this, false, "keep-env-derivations", + "Whether to add derivations as a dependency of user environments " + "(to prevent them from being GCed).", + {"env-keep-derivations"}}; /* Whether to lock the Nix client and worker to the same CPU. */ bool lockCPU; /* Whether to show a stack trace if Nix evaluation fails. */ - bool showTrace; + Setting showTrace{this, false, "show-trace", + "Whether to show a stack trace on evaluation errors."}; + + Setting sandboxMode{this, + #if __linux__ + smEnabled + #else + smDisabled + #endif + , "sandbox", + "Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".", + {"build-use-chroot", "build-use-sandbox"}}; + + Setting sandboxPaths{this, {}, "sandbox-paths", + "The paths to make available inside the build sandbox.", + {"build-chroot-dirs", "build-sandbox-paths"}}; + + Setting sandboxFallback{this, true, "sandbox-fallback", + "Whether to disable sandboxing when the kernel doesn't allow it."}; + + Setting extraSandboxPaths{this, {}, "extra-sandbox-paths", + "Additional paths to make available inside the build sandbox.", + {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}}; + + Setting buildRepeat{this, 0, "repeat", + "The number of times to repeat a build in order to verify determinism.", + {"build-repeat"}}; + +#if __linux__ + Setting sandboxShmSize{this, "50%", "sandbox-dev-shm-size", + "The size of /dev/shm in the build sandbox."}; + + Setting sandboxBuildDir{this, "/build", "sandbox-build-dir", + "The build directory inside the sandbox."}; +#endif + + Setting allowedImpureHostPrefixes{this, {}, "allowed-impure-host-deps", + "Which prefixes to allow derivations to ask for access to (primarily for Darwin)."}; + +#if __APPLE__ + Setting darwinLogSandboxViolations{this, false, "darwin-log-sandbox-violations", + "Whether to log Darwin sandbox access violations to the system log."}; +#endif + + Setting runDiffHook{this, false, "run-diff-hook", + "Whether to run the program specified by the diff-hook setting " + "repeated builds produce a different result. Typically used to " + "plug in diffoscope."}; + + PathSetting diffHook{this, true, "", "diff-hook", + "A program that prints out the differences between the two paths " + "specified on its command line."}; + + Setting enforceDeterminism{this, true, "enforce-determinism", + "Whether to fail if repeated builds produce different output."}; + + Setting trustedPublicKeys{this, + {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, + "trusted-public-keys", + "Trusted public keys for secure substitution.", + {"binary-cache-public-keys"}}; + + Setting secretKeyFiles{this, {}, "secret-key-files", + "Secret keys with which to sign local builds."}; + + Setting tarballTtl{this, 60 * 60, "tarball-ttl", + "How long downloaded files are considered up-to-date."}; + + Setting requireSigs{this, true, "require-sigs", + "Whether to check that any non-content-addressed path added to the " + "Nix store has a valid signature (that is, one signed using a key " + "listed in 'trusted-public-keys'."}; + + Setting extraPlatforms{this, + std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{}, + "extra-platforms", + "Additional platforms that can be built on the local system. " + "These may be supported natively (e.g. armv7 on some aarch64 CPUs " + "or using hacks like qemu-user."}; + + Setting systemFeatures{this, getDefaultSystemFeatures(), + "system-features", + "Optional features that this system implements (like \"kvm\")."}; + + Setting substituters{this, + nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(), + "substituters", + "The URIs of substituters (such as https://cache.nixos.org/).", + {"binary-caches"}}; - /* A list of URL prefixes that can return Nix build logs. */ - Strings logServers; + // FIXME: provide a way to add to option values. + Setting extraSubstituters{this, {}, "extra-substituters", + "Additional URIs of substituters.", + {"extra-binary-caches"}}; - /* Whether the importNative primop should be enabled */ - bool enableImportNative; + Setting trustedSubstituters{this, {}, "trusted-substituters", + "Disabled substituters that may be enabled via the substituters option by untrusted users.", + {"trusted-binary-caches"}}; - /* The hook to run just before a build to set derivation-specific - build settings */ - Path preBuildHook; + Setting trustedUsers{this, {"root"}, "trusted-users", + "Which users or groups are trusted to ask the daemon to do unsafe things."}; -private: - SettingsMap settings, overrides; + Setting ttlNegativeNarInfoCache{this, 3600, "narinfo-cache-negative-ttl", + "The TTL in seconds for negative lookups in the disk cache i.e binary cache lookups that " + "return an invalid path result"}; - void _get(string & res, const string & name); - void _get(bool & res, const string & name); - void _get(StringSet & res, const string & name); - void _get(Strings & res, const string & name); - template void _get(N & res, const string & name); + Setting ttlPositiveNarInfoCache{this, 30 * 24 * 3600, "narinfo-cache-positive-ttl", + "The TTL in seconds for positive lookups in the disk cache i.e binary cache lookups that " + "return a valid path result."}; + + /* ?Who we trust to use the daemon in safe ways */ + Setting allowedUsers{this, {"*"}, "allowed-users", + "Which users or groups are allowed to connect to the daemon."}; + + Setting printMissing{this, true, "print-missing", + "Whether to print what paths need to be built or downloaded."}; + + Setting preBuildHook{this, "", + "pre-build-hook", + "A program to run just before a build to set derivation-specific build settings."}; + + Setting postBuildHook{this, "", "post-build-hook", + "A program to run just after each successful build."}; + + Setting netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", + "Path to the netrc file used to obtain usernames/passwords for downloads."}; + + /* Path to the SSL CA file used */ + Path caFile; + +#if __linux__ + Setting filterSyscalls{this, true, "filter-syscalls", + "Whether to prevent certain dangerous system calls, such as " + "creation of setuid/setgid files or adding ACLs or extended " + "attributes. Only disable this if you're aware of the " + "security implications."}; + + Setting allowNewPrivileges{this, false, "allow-new-privileges", + "Whether builders can acquire new privileges by calling programs with " + "setuid/setgid bits or with file capabilities."}; +#endif + + Setting hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors", + "A list of servers used by builtins.fetchurl to fetch files by hash."}; + + Setting minFree{this, 0, "min-free", + "Automatically run the garbage collector when free disk space drops below the specified amount."}; + + Setting maxFree{this, std::numeric_limits::max(), "max-free", + "Stop deleting garbage when free disk space is above the specified amount."}; + + Setting minFreeCheckInterval{this, 5, "min-free-check-interval", + "Number of seconds between checking free disk space."}; + + Setting pluginFiles{this, {}, "plugin-files", + "Plugins to dynamically load at nix initialization time."}; + + Setting githubAccessToken{this, "", "github-access-token", + "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."}; + + Setting experimentalFeatures{this, {}, "experimental-features", + "Experimental Nix features to enable."}; + + bool isExperimentalFeatureEnabled(const std::string & name); + + void requireExperimentalFeature(const std::string & name); + + Setting allowDirty{this, true, "allow-dirty", + "Whether to allow dirty Git/Mercurial trees."}; + + Setting warnDirty{this, true, "warn-dirty", + "Whether to warn about dirty Git/Mercurial trees."}; }; // FIXME: don't use a global variable. extern Settings settings; +/* This should be called after settings are initialized, but before + anything else */ +void initPlugins(); -extern const string nixVersion; +void loadConfFile(); +// Used by the Settings constructor +std::vector getUserConfigFiles(); + +extern const string nixVersion; } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 9d31f77c921..451a647859c 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -1,5 +1,5 @@ #include "binary-cache-store.hh" -#include "download.hh" +#include "filetransfer.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" @@ -13,6 +13,14 @@ class HttpBinaryCacheStore : public BinaryCacheStore Path cacheUri; + struct State + { + bool enabled = true; + std::chrono::steady_clock::time_point disabledUntil; + }; + + Sync _state; + public: HttpBinaryCacheStore( @@ -34,64 +42,119 @@ class HttpBinaryCacheStore : public BinaryCacheStore void init() override { // FIXME: do this lazily? - if (!diskCache->cacheExists(cacheUri, wantMassQuery_, priority)) { + if (auto cacheInfo = diskCache->cacheExists(cacheUri)) { + wantMassQuery.setDefault(cacheInfo->wantMassQuery ? "true" : "false"); + priority.setDefault(fmt("%d", cacheInfo->priority)); + } else { try { BinaryCacheStore::init(); } catch (UploadToHTTP &) { - throw Error(format("‘%s’ does not appear to be a binary cache") % cacheUri); + throw Error("'%s' does not appear to be a binary cache", cacheUri); } - diskCache->createCache(cacheUri, storeDir, wantMassQuery_, priority); + diskCache->createCache(cacheUri, storeDir, wantMassQuery, priority); } } protected: + void maybeDisable() + { + auto state(_state.lock()); + if (state->enabled && settings.tryFallback) { + int t = 60; + printError("disabling binary cache '%s' for %s seconds", getUri(), t); + state->enabled = false; + state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t); + } + } + + void checkEnabled() + { + auto state(_state.lock()); + if (state->enabled) return; + if (std::chrono::steady_clock::now() > state->disabledUntil) { + state->enabled = true; + debug("re-enabling binary cache '%s'", getUri()); + return; + } + throw SubstituterDisabled("substituter '%s' is disabled", getUri()); + } + bool fileExists(const std::string & path) override { + checkEnabled(); + try { - DownloadRequest request(cacheUri + "/" + path); - request.showProgress = DownloadRequest::no; + FileTransferRequest request(cacheUri + "/" + path); request.head = true; - request.tries = 5; - getDownloader()->download(request); + getFileTransfer()->download(request); return true; - } catch (DownloadError & e) { + } catch (FileTransferError & e) { /* S3 buckets return 403 if a file doesn't exist and the bucket is unlistable, so treat 403 as 404. */ - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) return false; + maybeDisable(); throw; } } - void upsertFile(const std::string & path, const std::string & data) override + void upsertFile(const std::string & path, + const std::string & data, + const std::string & mimeType) override { - throw UploadToHTTP("uploading to an HTTP binary cache is not supported"); + auto req = FileTransferRequest(cacheUri + "/" + path); + req.data = std::make_shared(data); // FIXME: inefficient + req.mimeType = mimeType; + try { + getFileTransfer()->upload(req); + } catch (FileTransferError & e) { + throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg()); + } + } + + FileTransferRequest makeRequest(const std::string & path) + { + FileTransferRequest request(cacheUri + "/" + path); + return request; + } + + void getFile(const std::string & path, Sink & sink) override + { + checkEnabled(); + auto request(makeRequest(path)); + try { + getFileTransfer()->download(std::move(request), sink); + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) + throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); + maybeDisable(); + throw; + } } void getFile(const std::string & path, - std::function)> success, - std::function failure) override + Callback> callback) noexcept override { - DownloadRequest request(cacheUri + "/" + path); - request.showProgress = DownloadRequest::no; - request.tries = 8; - - getDownloader()->enqueueDownload(request, - [success](const DownloadResult & result) { - success(result.data); - }, - [success, failure](std::exception_ptr exc) { + checkEnabled(); + + auto request(makeRequest(path)); + + auto callbackPtr = std::make_shared(std::move(callback)); + + getFileTransfer()->enqueueFileTransfer(request, + {[callbackPtr, this](std::future result) { try { - std::rethrow_exception(exc); - } catch (DownloadError & e) { - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) - return success(0); - failure(exc); + (*callbackPtr)(result.get().data); + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) + return (*callbackPtr)(std::shared_ptr()); + maybeDisable(); + callbackPtr->rethrow(); } catch (...) { - failure(exc); + callbackPtr->rethrow(); } - }); + }}); } }; @@ -100,14 +163,14 @@ static RegisterStoreImplementation regStore([]( const std::string & uri, const Store::Params & params) -> std::shared_ptr { + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; if (std::string(uri, 0, 7) != "http://" && std::string(uri, 0, 8) != "https://" && - (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://") - ) return 0; + (!forceHttp || std::string(uri, 0, 7) != "file://")) + return 0; auto store = std::make_shared(params, uri); store->init(); return store; }); } - diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index 69d33ce3275..0464edd531c 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -1,7 +1,7 @@ #include #include "binary-cache-store.hh" -#include "download.hh" +#include "filetransfer.hh" #include "nar-info-disk-cache.hh" #include "ipfs.hh" @@ -44,10 +44,10 @@ class IPFSBinaryCacheStore : public BinaryCacheStore const Params & params, const Path & _cacheUri) : BinaryCacheStore(params) , cacheUri(_cacheUri) - , ipfsAPIHost(get(params, "host", "127.0.0.1")) - , ipfsAPIPort(std::stoi(get(params, "port", "5001"))) - , useIpfsGateway(get(params, "use_gateway", "0") == "1") - , ipfsGatewayURL(get(params, "gateway", "https://ipfs.io")) + , ipfsAPIHost(get(params, "host").value_or("127.0.0.1")) + , ipfsAPIPort(std::stoi(get(params, "port").value_or("5001"))) + , useIpfsGateway(get(params, "use_gateway").value_or("0") == "1") + , ipfsGatewayURL(get(params, "gateway").value_or("https://ipfs.io")) { if (cacheUri.back() == '/') cacheUri.pop_back(); @@ -66,13 +66,16 @@ class IPFSBinaryCacheStore : public BinaryCacheStore void init() override { - if (!diskCache->cacheExists(cacheUri, wantMassQuery_, priority)) { + if (auto cacheInfo = diskCache->cacheExists(getUri())) { + wantMassQuery.setDefault(cacheInfo->wantMassQuery ? "true" : "false"); + priority.setDefault(fmt("%d", cacheInfo->priority)); + } else { try { BinaryCacheStore::init(); } catch (UploadToIPFS &) { - throw Error(format("‘%s’ does not appear to be a binary cache") % cacheUri); + throw Error("‘%s’ does not appear to be a binary cache", cacheUri); } - diskCache->createCache(cacheUri, storeDir, wantMassQuery_, priority); + diskCache->createCache(cacheUri, storeDir, wantMassQuery, priority); } } @@ -88,53 +91,52 @@ class IPFSBinaryCacheStore : public BinaryCacheStore /* TODO: perform ipfs ls instead instead of trying to fetch it */ auto uri = constructIPFSRequest(path); try { - DownloadRequest request(uri); - request.showProgress = DownloadRequest::no; + FileTransferRequest request(uri); + //request.showProgress = FileTransferRequest::no; request.tries = 5; if (useIpfsGateway) request.head = true; - getDownloader()->download(request); + getFileTransfer()->download(request); return true; - } catch (DownloadError & e) { - if (e.error == Downloader::NotFound) + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound) return false; throw; } } - void upsertFile(const std::string & path, const std::string & data) override + void upsertFile(const std::string & path, const std::string & data, const std::string & mimeType) override { throw UploadToIPFS("uploading to an IPFS binary cache is not supported"); } void getFile(const std::string & path, - std::function)> success, - std::function failure) override + Callback> callback) noexcept override { /* * TODO: Try local mount first, best to share code with * LocalBinaryCacheStore */ auto uri = constructIPFSRequest(path); - DownloadRequest request(uri); - request.showProgress = DownloadRequest::no; + FileTransferRequest request(uri); + //request.showProgress = FileTransferRequest::no; request.tries = 8; - getDownloader()->enqueueDownload(request, - [success](const DownloadResult & result) { - success(result.data); - }, - [success, failure](std::exception_ptr exc) { + auto callbackPtr = std::make_shared(std::move(callback)); + + getFileTransfer()->enqueueFileTransfer(request, + {[callbackPtr](std::future result){ try { - std::rethrow_exception(exc); - } catch (DownloadError & e) { - if (e.error == Downloader::NotFound) - return success(0); - failure(exc); + (*callbackPtr)(result.get().data); + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) + return (*callbackPtr)(std::shared_ptr()); + callbackPtr->rethrow(); } catch (...) { - failure(exc); + callbackPtr->rethrow(); } - }); + }} + ); } }; diff --git a/src/libstore/ipfs.hh b/src/libstore/ipfs.hh index c5e843e19f8..00e99dbbf09 100644 --- a/src/libstore/ipfs.hh +++ b/src/libstore/ipfs.hh @@ -4,7 +4,7 @@ #include #include "types.hh" -#include "download.hh" +#include "filetransfer.hh" namespace nix { namespace ipfs { @@ -21,8 +21,8 @@ inline std::string buildAPIURL(const std::string & host, inline std::string buildQuery(const std::vector> & params = {}) { std::string query = "?stream-channels=true&json=true&encoding=json"; for (auto& param : params) { - std::string key = getDownloader()->urlEncode(param.first); - std::string value = getDownloader()->urlEncode(param.second); + std::string key = getFileTransfer()->urlEncode(param.first); + std::string value = getFileTransfer()->urlEncode(param.second); query += "&" + key + "=" + value; } return query; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc new file mode 100644 index 00000000000..af20d389b06 --- /dev/null +++ b/src/libstore/legacy-ssh-store.cc @@ -0,0 +1,297 @@ +#include "archive.hh" +#include "pool.hh" +#include "remote-store.hh" +#include "serve-protocol.hh" +#include "store-api.hh" +#include "worker-protocol.hh" +#include "ssh.hh" +#include "derivations.hh" + +namespace nix { + +static std::string uriScheme = "ssh://"; + +struct LegacySSHStore : public Store +{ + const Setting maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"}; + const Setting sshKey{this, "", "ssh-key", "path to an SSH private key"}; + const Setting compress{this, false, "compress", "whether to compress the connection"}; + const Setting remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; + const Setting remoteStore{this, "", "remote-store", "URI of the store on the remote system"}; + + // Hack for getting remote build log output. + const Setting logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; + + struct Connection + { + std::unique_ptr sshConn; + FdSink to; + FdSource from; + int remoteVersion; + bool good = true; + }; + + std::string host; + + ref> connections; + + SSHMaster master; + + LegacySSHStore(const string & host, const Params & params) + : Store(params) + , host(host) + , connections(make_ref>( + std::max(1, (int) maxConnections), + [this]() { return openConnection(); }, + [](const ref & r) { return r->good; } + )) + , master( + host, + sshKey, + // Use SSH master only if using more than 1 connection. + connections->capacity() > 1, + compress, + logFD) + { + } + + ref openConnection() + { + auto conn = make_ref(); + conn->sshConn = master.startCommand( + fmt("%s --serve --write", remoteProgram) + + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); + conn->to = FdSink(conn->sshConn->in.get()); + conn->from = FdSource(conn->sshConn->out.get()); + + try { + conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION; + conn->to.flush(); + + unsigned int magic = readInt(conn->from); + if (magic != SERVE_MAGIC_2) + throw Error("protocol mismatch with 'nix-store --serve' on '%s'", host); + conn->remoteVersion = readInt(conn->from); + if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200) + throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host); + + } catch (EndOfFile & e) { + throw Error("cannot connect to '%1%'", host); + } + + return conn; + }; + + string getUri() override + { + return uriScheme + host; + } + + void queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept override + { + try { + auto conn(connections->get()); + + debug("querying remote host '%s' for info on '%s'", host, printStorePath(path)); + + conn->to << cmdQueryPathInfos << PathSet{printStorePath(path)}; + conn->to.flush(); + + auto p = readString(conn->from); + if (p.empty()) return callback(nullptr); + auto info = std::make_shared(parseStorePath(p)); + assert(path == info->path); + + PathSet references; + auto deriver = readString(conn->from); + if (deriver != "") + info->deriver = parseStorePath(deriver); + info->references = readStorePaths(*this, conn->from); + readLongLong(conn->from); // download size + info->narSize = readLongLong(conn->from); + + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { + auto s = readString(conn->from); + info->narHash = s.empty() ? Hash() : Hash(s); + conn->from >> info->ca; + info->sigs = readStrings(conn->from); + } + + auto s = readString(conn->from); + assert(s == ""); + + callback(std::move(info)); + } catch (...) { callback.rethrow(); } + } + + void addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr accessor) override + { + debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host); + + auto conn(connections->get()); + + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) { + + conn->to + << cmdAddToStoreNar + << printStorePath(info.path) + << (info.deriver ? printStorePath(*info.deriver) : "") + << info.narHash.to_string(Base16, false); + writeStorePaths(*this, conn->to, info.references); + conn->to + << info.registrationTime + << info.narSize + << info.ultimate + << info.sigs + << info.ca; + try { + copyNAR(source, conn->to); + } catch (...) { + conn->good = false; + throw; + } + conn->to.flush(); + + } else { + + conn->to + << cmdImportPaths + << 1; + try { + copyNAR(source, conn->to); + } catch (...) { + conn->good = false; + throw; + } + conn->to + << exportMagic + << printStorePath(info.path); + writeStorePaths(*this, conn->to, info.references); + conn->to + << (info.deriver ? printStorePath(*info.deriver) : "") + << 0 + << 0; + conn->to.flush(); + + } + + if (readInt(conn->from) != 1) + throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); + } + + void narFromPath(const StorePath & path, Sink & sink) override + { + auto conn(connections->get()); + + conn->to << cmdDumpStorePath << printStorePath(path); + conn->to.flush(); + copyNAR(conn->from, sink); + } + + std::optional queryPathFromHashPart(const std::string & hashPart) override + { unsupported("queryPathFromHashPart"); } + + StorePath addToStore(const string & name, const Path & srcPath, + FileIngestionMethod method, HashType hashAlgo, + PathFilter & filter, RepairFlag repair) override + { unsupported("addToStore"); } + + StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) override + { unsupported("addTextToStore"); } + + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode) override + { + auto conn(connections->get()); + + conn->to + << cmdBuildDerivation + << printStorePath(drvPath); + writeDerivation(conn->to, *this, drv); + conn->to + << settings.maxSilentTime + << settings.buildTimeout; + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2) + conn->to + << settings.maxLogSize; + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) + conn->to + << settings.buildRepeat + << settings.enforceDeterminism; + + conn->to.flush(); + + BuildResult status; + status.status = (BuildResult::Status) readInt(conn->from); + conn->from >> status.errorMsg; + + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) + conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; + + return status; + } + + void ensurePath(const StorePath & path) override + { unsupported("ensurePath"); } + + void computeFSClosure(const StorePathSet & paths, + StorePathSet & out, bool flipDirection = false, + bool includeOutputs = false, bool includeDerivers = false) override + { + if (flipDirection || includeDerivers) { + Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers); + return; + } + + auto conn(connections->get()); + + conn->to + << cmdQueryClosure + << includeOutputs; + writeStorePaths(*this, conn->to, paths); + conn->to.flush(); + + for (auto & i : readStorePaths(*this, conn->from)) + out.insert(i.clone()); + } + + StorePathSet queryValidPaths(const StorePathSet & paths, + SubstituteFlag maybeSubstitute = NoSubstitute) override + { + auto conn(connections->get()); + + conn->to + << cmdQueryValidPaths + << false // lock + << maybeSubstitute; + writeStorePaths(*this, conn->to, paths); + conn->to.flush(); + + return readStorePaths(*this, conn->from); + } + + void connect() override + { + auto conn(connections->get()); + } + + unsigned int getProtocol() override + { + auto conn(connections->get()); + return conn->remoteVersion; + } +}; + +static RegisterStoreImplementation regStore([]( + const std::string & uri, const Store::Params & params) + -> std::shared_ptr +{ + if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; + return std::make_shared(std::string(uri, uriScheme.size()), params); +}); + +} diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 0f377989bd8..48aca478cc1 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -30,31 +30,29 @@ class LocalBinaryCacheStore : public BinaryCacheStore bool fileExists(const std::string & path) override; - void upsertFile(const std::string & path, const std::string & data) override; + void upsertFile(const std::string & path, + const std::string & data, + const std::string & mimeType) override; - void getFile(const std::string & path, - std::function)> success, - std::function failure) override + void getFile(const std::string & path, Sink & sink) override { - sync2async>(success, failure, [&]() { - try { - return std::make_shared(readFile(binaryCacheDir + "/" + path)); - } catch (SysError & e) { - if (e.errNo == ENOENT) return std::shared_ptr(); - throw; - } - }); + try { + readFile(binaryCacheDir + "/" + path, sink); + } catch (SysError & e) { + if (e.errNo == ENOENT) + throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path); + } } - PathSet queryAllValidPaths() override + StorePathSet queryAllValidPaths() override { - PathSet paths; + StorePathSet paths; for (auto & entry : readDirectory(binaryCacheDir)) { if (entry.name.size() != 40 || !hasSuffix(entry.name, ".narinfo")) continue; - paths.insert(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)); + paths.insert(parseStorePath(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8))); } return paths; @@ -65,6 +63,8 @@ class LocalBinaryCacheStore : public BinaryCacheStore void LocalBinaryCacheStore::init() { createDirs(binaryCacheDir + "/nar"); + if (writeDebugInfo) + createDirs(binaryCacheDir + "/debuginfo"); BinaryCacheStore::init(); } @@ -74,7 +74,7 @@ static void atomicWrite(const Path & path, const std::string & s) AutoDelete del(tmp, false); writeFile(tmp, s); if (rename(tmp.c_str(), path.c_str())) - throw SysError(format("renaming ‘%1%’ to ‘%2%’") % tmp % path); + throw SysError("renaming '%1%' to '%2%'", tmp, path); del.cancel(); } @@ -83,7 +83,9 @@ bool LocalBinaryCacheStore::fileExists(const std::string & path) return pathExists(binaryCacheDir + "/" + path); } -void LocalBinaryCacheStore::upsertFile(const std::string & path, const std::string & data) +void LocalBinaryCacheStore::upsertFile(const std::string & path, + const std::string & data, + const std::string & mimeType) { atomicWrite(binaryCacheDir + "/" + path, data); } diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 4571a2211cd..2d564a0d795 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -2,14 +2,13 @@ #include "fs-accessor.hh" #include "store-api.hh" #include "globals.hh" +#include "compression.hh" +#include "derivations.hh" namespace nix { LocalFSStore::LocalFSStore(const Params & params) : Store(params) - , rootDir(get(params, "root")) - , stateDir(canonPath(get(params, "state", rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir))) - , logDir(canonPath(get(params, "log", rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir))) { } @@ -22,8 +21,8 @@ struct LocalStoreAccessor : public FSAccessor Path toRealPath(const Path & path) { Path storePath = store->toStorePath(path); - if (!store->isValidPath(storePath)) - throw InvalidPath(format("path ‘%1%’ is not a valid store path") % storePath); + if (!store->isValidPath(store->parseStorePath(storePath))) + throw InvalidPath("path '%1%' is not a valid store path", storePath); return store->getRealStoreDir() + std::string(path, store->storeDir.size()); } @@ -32,13 +31,13 @@ struct LocalStoreAccessor : public FSAccessor auto realPath = toRealPath(path); struct stat st; - if (lstat(path.c_str(), &st)) { + if (lstat(realPath.c_str(), &st)) { if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; - throw SysError(format("getting status of ‘%1%’") % path); + throw SysError("getting status of '%1%'", path); } if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) - throw Error(format("file ‘%1%’ has unsupported type") % path); + throw Error("file '%1%' has unsupported type", path); return { S_ISREG(st.st_mode) ? Type::tRegular : @@ -52,7 +51,7 @@ struct LocalStoreAccessor : public FSAccessor { auto realPath = toRealPath(path); - auto entries = nix::readDirectory(path); + auto entries = nix::readDirectory(realPath); StringSet res; for (auto & entry : entries) @@ -74,14 +73,57 @@ struct LocalStoreAccessor : public FSAccessor ref LocalFSStore::getFSAccessor() { - return make_ref(ref(std::dynamic_pointer_cast(shared_from_this()))); + return make_ref(ref( + std::dynamic_pointer_cast(shared_from_this()))); } -void LocalFSStore::narFromPath(const Path & path, Sink & sink) +void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) { if (!isValidPath(path)) - throw Error(format("path ‘%s’ is not valid") % path); - dumpPath(getRealStoreDir() + std::string(path, storeDir.size()), sink); + throw Error("path '%s' is not valid", printStorePath(path)); + dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()), sink); +} + +const string LocalFSStore::drvsLogDir = "drvs"; + + + +std::shared_ptr LocalFSStore::getBuildLog(const StorePath & path_) +{ + auto path = path_.clone(); + + if (!path.isDerivation()) { + try { + auto info = queryPathInfo(path); + if (!info->deriver) return nullptr; + path = info->deriver->clone(); + } catch (InvalidPath &) { + return nullptr; + } + } + + auto baseName = std::string(baseNameOf(printStorePath(path))); + + for (int j = 0; j < 2; j++) { + + Path logPath = + j == 0 + ? fmt("%s/%s/%s/%s", logDir, drvsLogDir, string(baseName, 0, 2), string(baseName, 2)) + : fmt("%s/%s/%s", logDir, drvsLogDir, baseName); + Path logBz2Path = logPath + ".bz2"; + + if (pathExists(logPath)) + return std::make_shared(readFile(logPath)); + + else if (pathExists(logBz2Path)) { + try { + return decompress("bzip2", readFile(logBz2Path)); + } catch (Error &) { } + } + + } + + return nullptr; } } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 612efde7bb8..f5c5bd9b786 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1,4 +1,3 @@ -#include "config.h" #include "local-store.hh" #include "globals.hh" #include "archive.hh" @@ -6,6 +5,7 @@ #include "worker-protocol.hh" #include "derivations.hh" #include "nar-info.hh" +#include "references.hh" #include #include @@ -28,6 +28,11 @@ #include #include #include +#include +#endif + +#ifdef __CYGWIN__ +#include #endif #include @@ -39,14 +44,17 @@ namespace nix { LocalStore::LocalStore(const Params & params) : Store(params) , LocalFSStore(params) - , realStoreDir(get(params, "real", rootDir != "" ? rootDir + "/nix/store" : storeDir)) + , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", + "physical path to the Nix store"} + , realStoreDir(realStoreDir_) , dbDir(stateDir + "/db") , linksDir(realStoreDir + "/.links") , reservedPath(dbDir + "/reserved") , schemaPath(dbDir + "/schema") , trashDir(realStoreDir + "/trash") - , requireSigs(trim(settings.get("signed-binary-caches", std::string(""))) != "") // FIXME: rename option - , publicKeys(getDefaultPublicKeys()) + , tempRootsDir(stateDir + "/temproots") + , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid())) + , locksHeld(tokenizeString(getEnv("NIX_HELD_LOCKS").value_or(""))) { auto state(_state.lock()); @@ -56,7 +64,7 @@ LocalStore::LocalStore(const Params & params) createDirs(linksDir); Path profilesDir = stateDir + "/profiles"; createDirs(profilesDir); - createDirs(stateDir + "/temproots"); + createDirs(tempRootsDir); createDirs(dbDir); Path gcRootsDir = stateDir + "/gcroots"; if (!pathExists(gcRootsDir)) { @@ -64,31 +72,37 @@ LocalStore::LocalStore(const Params & params) createSymlink(profilesDir, gcRootsDir + "/profiles"); } + for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { + createDirs(perUserDir); + if (chmod(perUserDir.c_str(), 0755) == -1) + throw SysError("could not set permissions on '%s' to 755", perUserDir); + } + + createUser(getUserName(), getuid()); + /* Optionally, create directories and set permissions for a multi-user install. */ if (getuid() == 0 && settings.buildUsersGroup != "") { - - Path perUserDir = profilesDir + "/per-user"; - createDirs(perUserDir); - if (chmod(perUserDir.c_str(), 01777) == -1) - throw SysError(format("could not set permissions on ‘%1%’ to 1777") % perUserDir); - mode_t perm = 01775; - struct group * gr = getgrnam(settings.buildUsersGroup.c_str()); + struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) - printError(format("warning: the group ‘%1%’ specified in ‘build-users-group’ does not exist") - % settings.buildUsersGroup); + logError({ + .name = "'build-users-group' not found", + .hint = hintfmt( + "warning: the group '%1%' specified in 'build-users-group' does not exist", + settings.buildUsersGroup) + }); else { struct stat st; if (stat(realStoreDir.c_str(), &st)) - throw SysError(format("getting attributes of path ‘%1%’") % realStoreDir); + throw SysError("getting attributes of path '%1%'", realStoreDir); if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) { if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1) - throw SysError(format("changing ownership of path ‘%1%’") % realStoreDir); + throw SysError("changing ownership of path '%1%'", realStoreDir); if (chmod(realStoreDir.c_str(), perm) == -1) - throw SysError(format("changing permissions on path ‘%1%’") % realStoreDir); + throw SysError("changing permissions on path '%1%'", realStoreDir); } } } @@ -99,12 +113,12 @@ LocalStore::LocalStore(const Params & params) struct stat st; while (path != "/") { if (lstat(path.c_str(), &st)) - throw SysError(format("getting status of ‘%1%’") % path); + throw SysError("getting status of '%1%'", path); if (S_ISLNK(st.st_mode)) - throw Error(format( - "the path ‘%1%’ is a symlink; " - "this is not allowed for the Nix store and its parent directories") - % path); + throw Error( + "the path '%1%' is a symlink; " + "this is not allowed for the Nix store and its parent directories", + path); path = dirOf(path); } } @@ -137,7 +151,7 @@ LocalStore::LocalStore(const Params & params) globalLock = openLockFile(globalLockPath.c_str(), true); if (!lockFile(globalLock.get(), ltRead, false)) { - printError("waiting for the big Nix store lock..."); + printInfo("waiting for the big Nix store lock..."); lockFile(globalLock.get(), ltRead, true); } @@ -145,8 +159,8 @@ LocalStore::LocalStore(const Params & params) upgrade. */ int curSchema = getSchema(); if (curSchema > nixSchemaVersion) - throw Error(format("current Nix store schema is version %1%, but I only support %2%") - % curSchema % nixSchemaVersion); + throw Error("current Nix store schema is version %1%, but I only support %2%", + curSchema, nixSchemaVersion); else if (curSchema == 0) { /* new store */ curSchema = nixSchemaVersion; @@ -168,7 +182,7 @@ LocalStore::LocalStore(const Params & params) "please upgrade Nix to version 1.11 first."); if (!lockFile(globalLock.get(), ltWrite, false)) { - printError("waiting for exclusive access to the Nix store..."); + printInfo("waiting for exclusive access to the Nix store..."); lockFile(globalLock.get(), ltWrite, true); } @@ -237,12 +251,24 @@ LocalStore::LocalStore(const Params & params) LocalStore::~LocalStore() { - auto state(_state.lock()); + std::shared_future future; + + { + auto state(_state.lock()); + if (state->gcRunning) + future = state->gcFuture; + } + + if (future.valid()) { + printInfo("waiting for auto-GC to finish on exit..."); + future.get(); + } try { + auto state(_state.lock()); if (state->fdTempRoots) { state->fdTempRoots = -1; - unlink(state->fnTempRoots.c_str()); + unlink(fnTempRoots.c_str()); } } catch (...) { ignoreException(); @@ -262,7 +288,7 @@ int LocalStore::getSchema() if (pathExists(schemaPath)) { string s = readFile(schemaPath); if (!string2Int(s, curSchema)) - throw Error(format("‘%1%’ is corrupt") % schemaPath); + throw Error("'%1%' is corrupt", schemaPath); } return curSchema; } @@ -271,19 +297,22 @@ int LocalStore::getSchema() void LocalStore::openDB(State & state, bool create) { if (access(dbDir.c_str(), R_OK | W_OK)) - throw SysError(format("Nix database directory ‘%1%’ is not writable") % dbDir); + throw SysError("Nix database directory '%1%' is not writable", dbDir); /* Open the Nix database. */ string dbPath = dbDir + "/db.sqlite"; auto & db(state.db); - if (sqlite3_open_v2(dbPath.c_str(), &db.db, - SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) - throw Error(format("cannot open Nix database ‘%1%’") % dbPath); - - if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) - throwSQLiteError(db, "setting timeout"); - - db.exec("pragma foreign_keys = 1"); + state.db = SQLite(dbPath, create); + +#ifdef __CYGWIN__ + /* The cygwin version of sqlite3 has a patch which calls + SetDllDirectory("/usr/bin") on init. It was intended to fix extension + loading, which we don't use, and the effect of SetDllDirectory is + inherited by child processes, and causes libraries to be loaded from + /usr/bin instead of $PATH. This breaks quite a few things (e.g. + checkPhase on openssh), so we set it back to default behaviour. */ + SetDllDirectoryW(L""); +#endif /* !!! check whether sqlite has been built with foreign key support */ @@ -318,8 +347,8 @@ void LocalStore::openDB(State & state, bool create) /* Initialise the database schema, if necessary. */ if (create) { - const char * schema = -#include "schema.sql.hh" + static const char schema[] = +#include "schema.sql.gen.hh" ; db.exec(schema); } @@ -342,7 +371,7 @@ void LocalStore::makeStoreWritable() throw SysError("setting up a private mount namespace"); if (mount(0, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) - throw SysError(format("remounting %1% writable") % realStoreDir); + throw SysError("remounting %1% writable", realStoreDir); } #endif } @@ -363,7 +392,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct | 0444 | (st.st_mode & S_IXUSR ? 0111 : 0); if (chmod(path.c_str(), mode) == -1) - throw SysError(format("changing mode of ‘%1%’ to %2$o") % path % mode); + throw SysError("changing mode of '%1%' to %2$o", path, mode); } } @@ -381,7 +410,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct #else if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) #endif - throw SysError(format("changing modification time of ‘%1%’") % path); + throw SysError("changing modification time of '%1%'", path); } } @@ -390,7 +419,7 @@ void canonicaliseTimestampAndPermissions(const Path & path) { struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path ‘%1%’") % path); + throw SysError("getting attributes of path '%1%'", path); canonicaliseTimestampAndPermissions(path, st); } @@ -399,13 +428,46 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe { checkInterrupt(); +#if __APPLE__ + /* Remove flags, in particular UF_IMMUTABLE which would prevent + the file from being garbage-collected. FIXME: Use + setattrlist() to remove other attributes as well. */ + if (lchflags(path.c_str(), 0)) { + if (errno != ENOTSUP) + throw SysError("clearing flags of path '%1%'", path); + } +#endif + struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path ‘%1%’") % path); + throw SysError("getting attributes of path '%1%'", path); /* Really make sure that the path is of a supported type. */ if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) - throw Error(format("file ‘%1%’ has an unsupported type") % path); + throw Error("file '%1%' has an unsupported type", path); + +#if __linux__ + /* Remove extended attributes / ACLs. */ + ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); + + if (eaSize < 0) { + if (errno != ENOTSUP && errno != ENODATA) + throw SysError("querying extended attributes of '%s'", path); + } else if (eaSize > 0) { + std::vector eaBuf(eaSize); + + if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0) + throw SysError("querying extended attributes of '%s'", path); + + for (auto & eaName: tokenizeString(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { + /* Ignore SELinux security labels since these cannot be + removed even by root. */ + if (eaName == "security.selinux") continue; + if (lremovexattr(path.c_str(), eaName.c_str()) == -1) + throw SysError("removing extended attribute '%s' from '%s'", eaName, path); + } + } +#endif /* Fail if the file is not owned by the build user. This prevents us from messing up the ownership/permissions of files @@ -416,7 +478,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe if (fromUid != (uid_t) -1 && st.st_uid != fromUid) { assert(!S_ISDIR(st.st_mode)); if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end()) - throw BuildError(format("invalid ownership on file ‘%1%’") % path); + throw BuildError("invalid ownership on file '%1%'", path); mode_t mode = st.st_mode & ~S_IFMT; assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); return; @@ -440,8 +502,8 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe if (!S_ISLNK(st.st_mode) && chown(path.c_str(), geteuid(), getegid()) == -1) #endif - throw SysError(format("changing owner of ‘%1%’ to %2%") - % path % geteuid()); + throw SysError("changing owner of '%1%' to %2%", + path, geteuid()); } if (S_ISDIR(st.st_mode)) { @@ -460,11 +522,11 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & ino be a symlink, since we can't change its ownership. */ struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path ‘%1%’") % path); + throw SysError("getting attributes of path '%1%'", path); if (st.st_uid != geteuid()) { assert(S_ISLNK(st.st_mode)); - throw Error(format("wrong ownership of top-level store path ‘%1%’") % path); + throw Error("wrong ownership of top-level store path '%1%'", path); } } @@ -476,43 +538,39 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid) } -void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & drv) +void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv) { - string drvName = storePathToName(drvPath); - assert(isDerivation(drvName)); + assert(drvPath.isDerivation()); + std::string drvName(drvPath.name()); drvName = string(drvName, 0, drvName.size() - drvExtension.size()); + auto check = [&](const StorePath & expected, const StorePath & actual, const std::string & varName) + { + if (actual != expected) + throw Error("derivation '%s' has incorrect output '%s', should be '%s'", + printStorePath(drvPath), printStorePath(actual), printStorePath(expected)); + auto j = drv.env.find(varName); + if (j == drv.env.end() || parseStorePath(j->second) != actual) + throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", + printStorePath(drvPath), varName, printStorePath(actual)); + }; + + if (drv.isFixedOutput()) { DerivationOutputs::const_iterator out = drv.outputs.find("out"); if (out == drv.outputs.end()) - throw Error(format("derivation ‘%1%’ does not have an output named ‘out’") % drvPath); + throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); - bool recursive; Hash h; - out->second.parseHashInfo(recursive, h); - Path outPath = makeFixedOutputPath(recursive, h, drvName); + FileIngestionMethod method; Hash h; + out->second.parseHashInfo(method, h); - StringPairs::const_iterator j = drv.env.find("out"); - if (out->second.path != outPath || j == drv.env.end() || j->second != outPath) - throw Error(format("derivation ‘%1%’ has incorrect output ‘%2%’, should be ‘%3%’") - % drvPath % out->second.path % outPath); + check(makeFixedOutputPath(method, h, drvName), out->second.path, "out"); } else { - Derivation drvCopy(drv); - for (auto & i : drvCopy.outputs) { - i.second.path = ""; - drvCopy.env[i.first] = ""; - } - - Hash h = hashDerivationModulo(*this, drvCopy); - - for (auto & i : drv.outputs) { - Path outPath = makeOutputPath(i.first, h, drvName); - StringPairs::const_iterator j = drv.env.find(i.first); - if (i.second.path != outPath || j == drv.env.end() || j->second != outPath) - throw Error(format("derivation ‘%1%’ has incorrect output ‘%2%’, should be ‘%3%’") - % drvPath % i.second.path % outPath); - } + Hash h = hashDerivationModulo(*this, drv, true); + for (auto & i : drv.outputs) + check(makeOutputPath(i.first, h, drvName), i.second.path, i.first); } } @@ -520,11 +578,15 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs) { + if (info.ca != "" && !info.isContentAddressed(*this)) + throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't", + printStorePath(info.path)); + state.stmtRegisterValidPath.use() - (info.path) - ("sha256:" + printHash(info.narHash)) + (printStorePath(info.path)) + (info.narHash.to_string(Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) - (info.deriver, info.deriver != "") + (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) @@ -536,8 +598,8 @@ uint64_t LocalStore::addValidPath(State & state, the database. This is useful for the garbage collector: it can efficiently query whether a path is an output of some derivation. */ - if (isDerivation(info.path)) { - Derivation drv = readDerivation(realStoreDir + "/" + baseNameOf(info.path)); + if (info.path.isDerivation()) { + auto drv = readDerivation(info.path); /* Verify that the output paths in the derivation are correct (i.e., follow the scheme for computing output paths from @@ -550,62 +612,48 @@ uint64_t LocalStore::addValidPath(State & state, state.stmtAddDerivationOutput.use() (id) (i.first) - (i.second.path) + (printStorePath(i.second.path)) .exec(); } } { auto state_(Store::state.lock()); - state_->pathInfoCache.upsert(storePathToHash(info.path), std::make_shared(info)); + state_->pathInfoCache.upsert(storePathToHash(printStorePath(info.path)), + PathInfoCacheValue{ .value = std::make_shared(info) }); } return id; } -Hash parseHashField(const Path & path, const string & s) -{ - string::size_type colon = s.find(':'); - if (colon == string::npos) - throw Error(format("corrupt hash ‘%1%’ in valid-path entry for ‘%2%’") - % s % path); - HashType ht = parseHashType(string(s, 0, colon)); - if (ht == htUnknown) - throw Error(format("unknown hash type ‘%1%’ in valid-path entry for ‘%2%’") - % string(s, 0, colon) % path); - return parseHash(ht, string(s, colon + 1)); -} - - -void LocalStore::queryPathInfoUncached(const Path & path, - std::function)> success, - std::function failure) +void LocalStore::queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept { - sync2async>(success, failure, [&]() { - - auto info = std::make_shared(); - info->path = path; - - assertStorePath(path); + try { + auto info = std::make_shared(path.clone()); - return retrySQLite>([&]() { + callback(retrySQLite>([&]() { auto state(_state.lock()); /* Get the path info. */ - auto useQueryPathInfo(state->stmtQueryPathInfo.use()(path)); + auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(info->path))); if (!useQueryPathInfo.next()) return std::shared_ptr(); info->id = useQueryPathInfo.getInt(0); - info->narHash = parseHashField(path, useQueryPathInfo.getStr(1)); + try { + info->narHash = Hash(useQueryPathInfo.getStr(1)); + } catch (BadHash & e) { + throw Error("in valid-path entry for '%s': %s", printStorePath(path), e.what()); + } info->registrationTime = useQueryPathInfo.getInt(2); auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3); - if (s) info->deriver = s; + if (s) info->deriver = parseStorePath(s); /* Note that narSize = NULL yields 0. */ info->narSize = useQueryPathInfo.getInt(4); @@ -622,11 +670,12 @@ void LocalStore::queryPathInfoUncached(const Path & path, auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); while (useQueryReferences.next()) - info->references.insert(useQueryReferences.getStr(0)); + info->references.insert(parseStorePath(useQueryReferences.getStr(0))); return info; - }); - }); + })); + + } catch (...) { callback.rethrow(); } } @@ -635,31 +684,31 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmtUpdatePathInfo.use() (info.narSize, info.narSize != 0) - ("sha256:" + printHash(info.narHash)) + (info.narHash.to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (info.ca, !info.ca.empty()) - (info.path) + (printStorePath(info.path)) .exec(); } -uint64_t LocalStore::queryValidPathId(State & state, const Path & path) +uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path) { - auto use(state.stmtQueryPathInfo.use()(path)); + auto use(state.stmtQueryPathInfo.use()(printStorePath(path))); if (!use.next()) - throw Error(format("path ‘%1%’ is not valid") % path); + throw Error("path '%s' is not valid", printStorePath(path)); return use.getInt(0); } -bool LocalStore::isValidPath_(State & state, const Path & path) +bool LocalStore::isValidPath_(State & state, const StorePath & path) { - return state.stmtQueryPathInfo.use()(path).next(); + return state.stmtQueryPathInfo.use()(printStorePath(path)).next(); } -bool LocalStore::isValidPathUncached(const Path & path) +bool LocalStore::isValidPathUncached(const StorePath & path) { return retrySQLite([&]() { auto state(_state.lock()); @@ -668,39 +717,38 @@ bool LocalStore::isValidPathUncached(const Path & path) } -PathSet LocalStore::queryValidPaths(const PathSet & paths) +StorePathSet LocalStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) { - PathSet res; + StorePathSet res; for (auto & i : paths) - if (isValidPath(i)) res.insert(i); + if (isValidPath(i)) res.insert(i.clone()); return res; } -PathSet LocalStore::queryAllValidPaths() +StorePathSet LocalStore::queryAllValidPaths() { - return retrySQLite([&]() { + return retrySQLite([&]() { auto state(_state.lock()); auto use(state->stmtQueryValidPaths.use()); - PathSet res; - while (use.next()) res.insert(use.getStr(0)); + StorePathSet res; + while (use.next()) res.insert(parseStorePath(use.getStr(0))); return res; }); } -void LocalStore::queryReferrers(State & state, const Path & path, PathSet & referrers) +void LocalStore::queryReferrers(State & state, const StorePath & path, StorePathSet & referrers) { - auto useQueryReferrers(state.stmtQueryReferrers.use()(path)); + auto useQueryReferrers(state.stmtQueryReferrers.use()(printStorePath(path))); while (useQueryReferrers.next()) - referrers.insert(useQueryReferrers.getStr(0)); + referrers.insert(parseStorePath(useQueryReferrers.getStr(0))); } -void LocalStore::queryReferrers(const Path & path, PathSet & referrers) +void LocalStore::queryReferrers(const StorePath & path, StorePathSet & referrers) { - assertStorePath(path); return retrySQLite([&]() { auto state(_state.lock()); queryReferrers(*state, path, referrers); @@ -708,97 +756,83 @@ void LocalStore::queryReferrers(const Path & path, PathSet & referrers) } -PathSet LocalStore::queryValidDerivers(const Path & path) +StorePathSet LocalStore::queryValidDerivers(const StorePath & path) { - assertStorePath(path); - - return retrySQLite([&]() { + return retrySQLite([&]() { auto state(_state.lock()); - auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(path)); + auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(printStorePath(path))); - PathSet derivers; + StorePathSet derivers; while (useQueryValidDerivers.next()) - derivers.insert(useQueryValidDerivers.getStr(1)); + derivers.insert(parseStorePath(useQueryValidDerivers.getStr(1))); return derivers; }); } -PathSet LocalStore::queryDerivationOutputs(const Path & path) +StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path) { - return retrySQLite([&]() { + return retrySQLite([&]() { auto state(_state.lock()); auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() (queryValidPathId(*state, path))); - PathSet outputs; + StorePathSet outputs; while (useQueryDerivationOutputs.next()) - outputs.insert(useQueryDerivationOutputs.getStr(1)); + outputs.insert(parseStorePath(useQueryDerivationOutputs.getStr(1))); return outputs; }); } -StringSet LocalStore::queryDerivationOutputNames(const Path & path) -{ - return retrySQLite([&]() { - auto state(_state.lock()); - - auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() - (queryValidPathId(*state, path))); - - StringSet outputNames; - while (useQueryDerivationOutputs.next()) - outputNames.insert(useQueryDerivationOutputs.getStr(0)); - - return outputNames; - }); -} - - -Path LocalStore::queryPathFromHashPart(const string & hashPart) +std::optional LocalStore::queryPathFromHashPart(const std::string & hashPart) { if (hashPart.size() != storePathHashLen) throw Error("invalid hash part"); Path prefix = storeDir + "/" + hashPart; - return retrySQLite([&]() -> std::string { + return retrySQLite>([&]() -> std::optional { auto state(_state.lock()); auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix)); - if (!useQueryPathFromHashPart.next()) return ""; + if (!useQueryPathFromHashPart.next()) return {}; const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0); - return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : ""; + if (s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0) + return parseStorePath(s); + return {}; }); } -PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) +StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) { - if (!settings.useSubstitutes) return PathSet(); + if (!settings.useSubstitutes) return StorePathSet(); + + StorePathSet remaining; + for (auto & i : paths) + remaining.insert(i.clone()); - auto remaining = paths; - PathSet res; + StorePathSet res; for (auto & sub : getDefaultSubstituters()) { if (remaining.empty()) break; if (sub->storeDir != storeDir) continue; - if (!sub->wantMassQuery()) continue; + if (!sub->wantMassQuery) continue; auto valid = sub->queryValidPaths(remaining); - PathSet remaining2; + StorePathSet remaining2; for (auto & path : remaining) if (valid.count(path)) - res.insert(path); + res.insert(path.clone()); else - remaining2.insert(path); + remaining2.insert(path.clone()); std::swap(remaining, remaining2); } @@ -807,7 +841,7 @@ PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) } -void LocalStore::querySubstitutablePathInfos(const PathSet & paths, +void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, SubstitutablePathInfos & infos) { if (!settings.useSubstitutes) return; @@ -815,18 +849,23 @@ void LocalStore::querySubstitutablePathInfos(const PathSet & paths, if (sub->storeDir != storeDir) continue; for (auto & path : paths) { if (infos.count(path)) continue; - debug(format("checking substituter ‘%s’ for path ‘%s’") - % sub->getUri() % path); + debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path)); try { auto info = sub->queryPathInfo(path); auto narInfo = std::dynamic_pointer_cast( std::shared_ptr(info)); - infos[path] = SubstitutablePathInfo{ - info->deriver, - info->references, + infos.insert_or_assign(path.clone(), SubstitutablePathInfo{ + info->deriver ? info->deriver->clone() : std::optional(), + cloneStorePathSet(info->references), narInfo ? narInfo->fileSize : 0, - info->narSize}; - } catch (InvalidPath) { + info->narSize}); + } catch (InvalidPath &) { + } catch (SubstituterDisabled &) { + } catch (Error & e) { + if (settings.tryFallback) + logError(e.info()); + else + throw; } } } @@ -853,7 +892,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) auto state(_state.lock()); SQLiteTxn txn(state->db); - PathSet paths; + StorePathSet paths; for (auto & i : infos) { assert(i.narHash.type == htSHA256); @@ -861,7 +900,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) updatePathInfo(*state, i); else addValidPath(*state, i, false); - paths.insert(i.path); + paths.insert(i.path.clone()); } for (auto & i : infos) { @@ -874,11 +913,9 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) this in addValidPath() above, because the references might not be valid yet. */ for (auto & i : infos) - if (isDerivation(i.path)) { - // FIXME: inefficient; we already loaded the - // derivation in addValidPath(). - Derivation drv = readDerivation(realStoreDir + "/" + baseNameOf(i.path)); - checkDerivationOutputs(i.path, drv); + if (i.path.isDerivation()) { + // FIXME: inefficient; we already loaded the derivation in addValidPath(). + checkDerivationOutputs(i.path, readDerivation(i.path)); } /* Do a topological sort of the paths. This will throw an @@ -894,32 +931,39 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) /* Invalidate a path. The caller is responsible for checking that there are no referrers. */ -void LocalStore::invalidatePath(State & state, const Path & path) +void LocalStore::invalidatePath(State & state, const StorePath & path) { - debug(format("invalidating path ‘%1%’") % path); + debug("invalidating path '%s'", printStorePath(path)); - state.stmtInvalidatePath.use()(path).exec(); + state.stmtInvalidatePath.use()(printStorePath(path)).exec(); /* Note that the foreign key constraints on the Refs table take care of deleting the references entries for `path'. */ { auto state_(Store::state.lock()); - state_->pathInfoCache.erase(storePathToHash(path)); + state_->pathInfoCache.erase(storePathToHash(printStorePath(path))); } } -void LocalStore::addToStore(const ValidPathInfo & info, const ref & nar, - bool repair, bool dontCheckSigs, std::shared_ptr accessor) +const PublicKeys & LocalStore::getPublicKeys() +{ + auto state(_state.lock()); + if (!state->publicKeys) + state->publicKeys = std::make_unique(getDefaultPublicKeys()); + return *state->publicKeys; +} + + +void LocalStore::addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr accessor) { - Hash h = hashString(htSHA256, *nar); - if (h != info.narHash) - throw Error(format("hash mismatch importing path ‘%s’; expected hash ‘%s’, got ‘%s’") % - info.path % info.narHash.to_string() % h.to_string()); + if (!info.narHash) + throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path)); - if (requireSigs && !dontCheckSigs && !info.checkSignatures(*this, publicKeys)) - throw Error(format("cannot import path ‘%s’ because it lacks a valid signature") % info.path); + if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys())) + throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path)); addTempRoot(info.path); @@ -927,21 +971,50 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref & PathLocks outputLock; - Path realPath = realStoreDir + "/" + baseNameOf(info.path); + Path realPath = realStoreDir + "/" + std::string(info.path.to_string()); /* Lock the output path. But don't lock if we're being called from a build hook (whose parent process already acquired a lock on this path). */ - Strings locksHeld = tokenizeString(getEnv("NIX_HELD_LOCKS")); - if (find(locksHeld.begin(), locksHeld.end(), info.path) == locksHeld.end()) + if (!locksHeld.count(printStorePath(info.path))) outputLock.lockPaths({realPath}); if (repair || !isValidPath(info.path)) { deletePath(realPath); - StringSource source(*nar); - restorePath(realPath, source); + if (info.ca != "" && + !((hasPrefix(info.ca, "text:") && !info.references.count(info.path)) + || info.references.empty())) + settings.requireExperimentalFeature("ca-references"); + + /* While restoring the path from the NAR, compute the hash + of the NAR. */ + std::unique_ptr hashSink; + if (info.ca == "" || !info.references.count(info.path)) + hashSink = std::make_unique(htSHA256); + else + hashSink = std::make_unique(htSHA256, storePathToHash(printStorePath(info.path))); + + LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t { + size_t n = source.read(data, len); + (*hashSink)(data, n); + return n; + }); + + restorePath(realPath, wrapperSource); + + auto hashResult = hashSink->finish(); + + if (hashResult.first != info.narHash) + throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", + printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); + + if (hashResult.second != info.narSize) + throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", + printStorePath(info.path), info.narSize, hashResult.second); + + autoGC(); canonicalisePathMetaData(realPath, -1); @@ -955,12 +1028,12 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref & } -Path LocalStore::addToStoreFromDump(const string & dump, const string & name, - bool recursive, HashType hashAlgo, bool repair) +StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) { Hash h = hashString(hashAlgo, dump); - Path dstPath = makeFixedOutputPath(recursive, h, name); + auto dstPath = makeFixedOutputPath(method, h, name); addTempRoot(dstPath); @@ -969,7 +1042,8 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, /* The first check above is an optimisation to prevent unnecessary lock acquisition. */ - Path realPath = realStoreDir + "/" + baseNameOf(dstPath); + Path realPath = realStoreDir + "/"; + realPath += dstPath.to_string(); PathLocks outputLock({realPath}); @@ -977,7 +1051,9 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, deletePath(realPath); - if (recursive) { + autoGC(); + + if (method == FileIngestionMethod::Recursive) { StringSource source(dump); restorePath(realPath, source); } else @@ -990,7 +1066,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, above (if called with recursive == true and hashAlgo == sha256); otherwise, compute it here. */ HashResult hash; - if (recursive) { + if (method == FileIngestionMethod::Recursive) { hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); hash.second = dump.size(); } else @@ -998,12 +1074,10 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, optimisePath(realPath); // FIXME: combine with hashPath() - ValidPathInfo info; - info.path = dstPath; + ValidPathInfo info(dstPath.clone()); info.narHash = hash.first; info.narSize = hash.second; - info.ultimate = true; - info.ca = "fixed:" + (recursive ? (std::string) "r:" : "") + h.to_string(); + info.ca = makeFixedOutputCA(method, h); registerValidPath(info); } @@ -1014,8 +1088,8 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, } -Path LocalStore::addToStore(const string & name, const Path & _srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) +StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { Path srcPath(absPath(_srcPath)); @@ -1023,17 +1097,17 @@ Path LocalStore::addToStore(const string & name, const Path & _srcPath, method for very large paths, but `copyPath' is mainly used for small files. */ StringSink sink; - if (recursive) + if (method == FileIngestionMethod::Recursive) dumpPath(srcPath, sink, filter); else sink.s = make_ref(readFile(srcPath)); - return addToStoreFromDump(*sink.s, name, recursive, hashAlgo, repair); + return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair); } -Path LocalStore::addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair) +StorePath LocalStore::addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) { auto hash = hashString(htSHA256, s); auto dstPath = makeTextPath(name, hash, references); @@ -1042,7 +1116,8 @@ Path LocalStore::addTextToStore(const string & name, const string & s, if (repair || !isValidPath(dstPath)) { - Path realPath = realStoreDir + "/" + baseNameOf(dstPath); + Path realPath = realStoreDir + "/"; + realPath += dstPath.to_string(); PathLocks outputLock({realPath}); @@ -1050,6 +1125,8 @@ Path LocalStore::addTextToStore(const string & name, const string & s, deletePath(realPath); + autoGC(); + writeFile(realPath, s); canonicalisePathMetaData(realPath, -1); @@ -1060,13 +1137,11 @@ Path LocalStore::addTextToStore(const string & name, const string & s, optimisePath(realPath); - ValidPathInfo info; - info.path = dstPath; + ValidPathInfo info(dstPath.clone()); info.narHash = narHash; info.narSize = sink.s->size(); - info.references = references; - info.ultimate = true; - info.ca = "text:" + hash.to_string(); + info.references = cloneStorePathSet(references); + info.ca = "text:" + hash.to_string(Base32, true); registerValidPath(info); } @@ -1087,27 +1162,25 @@ Path LocalStore::createTempDirInStore() the GC between createTempDir() and addTempRoot(), so repeat until `tmpDir' exists. */ tmpDir = createTempDir(realStoreDir); - addTempRoot(tmpDir); + addTempRoot(parseStorePath(tmpDir)); } while (!pathExists(tmpDir)); return tmpDir; } -void LocalStore::invalidatePathChecked(const Path & path) +void LocalStore::invalidatePathChecked(const StorePath & path) { - assertStorePath(path); - retrySQLite([&]() { auto state(_state.lock()); SQLiteTxn txn(state->db); if (isValidPath_(*state, path)) { - PathSet referrers; queryReferrers(*state, path, referrers); + StorePathSet referrers; queryReferrers(*state, path, referrers); referrers.erase(path); /* ignore self-references */ if (!referrers.empty()) - throw PathInUse(format("cannot delete path ‘%1%’ because it is in use by %2%") - % path % showPaths(referrers)); + throw PathInUse("cannot delete path '%s' because it is in use by %s", + printStorePath(path), showPaths(referrers)); invalidatePath(*state, path); } @@ -1116,33 +1189,58 @@ void LocalStore::invalidatePathChecked(const Path & path) } -bool LocalStore::verifyStore(bool checkContents, bool repair) +bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) { - printError(format("reading the Nix store...")); + printInfo(format("reading the Nix store...")); bool errors = false; - /* Acquire the global GC lock to prevent a garbage collection. */ + /* Acquire the global GC lock to get a consistent snapshot of + existing and valid paths. */ AutoCloseFD fdGCLock = openGCLock(ltWrite); - PathSet store; + StringSet store; for (auto & i : readDirectory(realStoreDir)) store.insert(i.name); /* Check whether all valid paths actually exist. */ printInfo("checking path existence..."); - PathSet validPaths2 = queryAllValidPaths(), validPaths, done; + StorePathSet validPaths; + PathSet done; - for (auto & i : validPaths2) - verifyPath(i, store, done, validPaths, repair, errors); - - /* Release the GC lock so that checking content hashes (which can - take ages) doesn't block the GC or builds. */ fdGCLock = -1; + for (auto & i : queryAllValidPaths()) + verifyPath(printStorePath(i), store, done, validPaths, repair, errors); + /* Optionally, check the content hashes (slow). */ if (checkContents) { - printInfo("checking hashes..."); + + printInfo("checking link hashes..."); + + for (auto & link : readDirectory(linksDir)) { + printMsg(lvlTalkative, "checking contents of '%s'", link.name); + Path linkPath = linksDir + "/" + link.name; + string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); + if (hash != link.name) { + logError({ + .name = "Invalid hash", + .hint = hintfmt( + "link '%s' was modified! expected hash '%s', got '%s'", + linkPath, link.name, hash) + }); + if (repair) { + if (unlink(linkPath.c_str()) == 0) + printInfo("removed link '%s'", linkPath); + else + throw SysError("removing corrupt link '%s'", linkPath); + } else { + errors = true; + } + } + } + + printInfo("checking store hashes..."); Hash nullHash(htSHA256); @@ -1151,13 +1249,23 @@ bool LocalStore::verifyStore(bool checkContents, bool repair) auto info = std::const_pointer_cast(std::shared_ptr(queryPathInfo(i))); /* Check the content hash (optionally - slow). */ - printMsg(lvlTalkative, format("checking contents of ‘%1%’") % i); - HashResult current = hashPath(info->narHash.type, i); + printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i)); + + std::unique_ptr hashSink; + if (info->ca == "" || !info->references.count(info->path)) + hashSink = std::make_unique(info->narHash.type); + else + hashSink = std::make_unique(info->narHash.type, storePathToHash(printStorePath(info->path))); + + dumpPath(Store::toRealPath(i), *hashSink); + auto current = hashSink->finish(); if (info->narHash != nullHash && info->narHash != current.first) { - printError(format("path ‘%1%’ was modified! " - "expected hash ‘%2%’, got ‘%3%’") - % i % printHash(info->narHash) % printHash(current.first)); + logError({ + .name = "Invalid hash - path modified", + .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'", + printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)) + }); if (repair) repairPath(i); else errors = true; } else { @@ -1165,14 +1273,14 @@ bool LocalStore::verifyStore(bool checkContents, bool repair) /* Fill in missing hashes. */ if (info->narHash == nullHash) { - printError(format("fixing missing hash on ‘%1%’") % i); + printInfo("fixing missing hash on '%s'", printStorePath(i)); info->narHash = current.first; update = true; } /* Fill in missing narSize fields (from old stores). */ if (info->narSize == 0) { - printError(format("updating size field on ‘%1%’ to %2%") % i % current.second); + printInfo("updating size field on '%s' to %s", printStorePath(i), current.second); info->narSize = current.second; update = true; } @@ -1188,9 +1296,9 @@ bool LocalStore::verifyStore(bool checkContents, bool repair) /* It's possible that the path got GC'ed, so ignore errors on invalid paths. */ if (isValidPath(i)) - printError(format("error: %1%") % e.msg()); + logError(e.info()); else - printError(format("warning: %1%") % e.msg()); + warn(e.msg()); errors = true; } } @@ -1200,44 +1308,49 @@ bool LocalStore::verifyStore(bool checkContents, bool repair) } -void LocalStore::verifyPath(const Path & path, const PathSet & store, - PathSet & done, PathSet & validPaths, bool repair, bool & errors) +void LocalStore::verifyPath(const Path & pathS, const StringSet & store, + PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) { checkInterrupt(); - if (done.find(path) != done.end()) return; - done.insert(path); + if (!done.insert(pathS).second) return; - if (!isStorePath(path)) { - printError(format("path ‘%1%’ is not in the Nix store") % path); - auto state(_state.lock()); - invalidatePath(*state, path); + if (!isStorePath(pathS)) { + logError({ + .name = "Nix path not found", + .hint = hintfmt("path '%s' is not in the Nix store", pathS) + }); return; } - if (store.find(baseNameOf(path)) == store.end()) { + auto path = parseStorePath(pathS); + + if (!store.count(std::string(path.to_string()))) { /* Check any referrers first. If we can invalidate them first, then we can invalidate this path as well. */ bool canInvalidate = true; - PathSet referrers; queryReferrers(path, referrers); + StorePathSet referrers; queryReferrers(path, referrers); for (auto & i : referrers) if (i != path) { - verifyPath(i, store, done, validPaths, repair, errors); - if (validPaths.find(i) != validPaths.end()) + verifyPath(printStorePath(i), store, done, validPaths, repair, errors); + if (validPaths.count(i)) canInvalidate = false; } if (canInvalidate) { - printError(format("path ‘%1%’ disappeared, removing from database...") % path); + printInfo("path '%s' disappeared, removing from database...", pathS); auto state(_state.lock()); invalidatePath(*state, path); } else { - printError(format("path ‘%1%’ disappeared, but it still has valid referrers!") % path); + logError({ + .name = "Missing path with referrers", + .hint = hintfmt("path '%s' disappeared, but it still has valid referrers!", pathS) + }); if (repair) try { repairPath(path); } catch (Error & e) { - printError(format("warning: %1%") % e.msg()); + logWarning(e.info()); errors = true; } else errors = true; @@ -1246,7 +1359,13 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, return; } - validPaths.insert(path); + validPaths.insert(std::move(path)); +} + + +unsigned int LocalStore::getProtocol() +{ + return PROTOCOL_VERSION; } @@ -1271,7 +1390,7 @@ static void makeMutable(const Path & path) AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC); if (fd == -1) { if (errno == ELOOP) return; // it's a symlink - throw SysError(format("opening file ‘%1%’") % path); + throw SysError("opening file '%1%'", path); } unsigned int flags = 0, old; @@ -1289,7 +1408,7 @@ static void makeMutable(const Path & path) void LocalStore::upgradeStore7() { if (getuid() != 0) return; - printError("removing immutable bits from the Nix store (this may take a while)..."); + printInfo("removing immutable bits from the Nix store (this may take a while)..."); makeMutable(realStoreDir); } @@ -1309,7 +1428,7 @@ void LocalStore::vacuumDB() } -void LocalStore::addSignatures(const Path & storePath, const StringSet & sigs) +void LocalStore::addSignatures(const StorePath & storePath, const StringSet & sigs) { retrySQLite([&]() { auto state(_state.lock()); @@ -1331,11 +1450,26 @@ void LocalStore::signPathInfo(ValidPathInfo & info) { // FIXME: keep secret keys in memory. - auto secretKeyFiles = settings.get("secret-key-files", Strings()); + auto secretKeyFiles = settings.secretKeyFiles; - for (auto & secretKeyFile : secretKeyFiles) { + for (auto & secretKeyFile : secretKeyFiles.get()) { SecretKey secretKey(readFile(secretKeyFile)); - info.sign(secretKey); + info.sign(*this, secretKey); + } +} + + +void LocalStore::createUser(const std::string & userName, uid_t userId) +{ + for (auto & dir : { + fmt("%s/profiles/per-user/%s", stateDir, userName), + fmt("%s/gcroots/per-user/%s", stateDir, userName) + }) { + createDirs(dir); + if (chmod(dir.c_str(), 0755) == -1) + throw SysError("changing permissions of directory '%s'", dir); + if (chown(dir.c_str(), userId, getgid()) == -1) + throw SysError("changing owner of directory '%s'", dir); } } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 511209d8404..e17cc45aead 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -7,6 +7,8 @@ #include "sync.hh" #include "util.hh" +#include +#include #include #include @@ -17,26 +19,18 @@ namespace nix { /* Nix store and database schema version. Version 1 (or 0) was Nix <= 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is - Nix 1.0. Version 7 is Nix 1.3. Version 10 is 1.12. */ + Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. */ const int nixSchemaVersion = 10; -extern string drvsLogDir; - - struct Derivation; struct OptimiseStats { - unsigned long filesLinked; - unsigned long long bytesFreed; - unsigned long long blocksFreed; - OptimiseStats() - { - filesLinked = 0; - bytesFreed = blocksFreed = 0; - } + unsigned long filesLinked = 0; + unsigned long long bytesFreed = 0; + unsigned long long blocksFreed = 0; }; @@ -67,29 +61,54 @@ private: SQLiteStmt stmtQueryValidPaths; /* The file to which we write our temporary roots. */ - Path fnTempRoots; AutoCloseFD fdTempRoots; + + /* The last time we checked whether to do an auto-GC, or an + auto-GC finished. */ + std::chrono::time_point lastGCCheck; + + /* Whether auto-GC is running. If so, get gcFuture to wait for + the GC to finish. */ + bool gcRunning = false; + std::shared_future gcFuture; + + /* How much disk space was available after the previous + auto-GC. If the current available disk space is below + minFree but not much below availAfterGC, then there is no + point in starting a new GC. */ + uint64_t availAfterGC = std::numeric_limits::max(); + + std::unique_ptr publicKeys; }; Sync _state; public: + PathSetting realStoreDir_; + const Path realStoreDir; const Path dbDir; const Path linksDir; const Path reservedPath; const Path schemaPath; const Path trashDir; + const Path tempRootsDir; + const Path fnTempRoots; private: - bool requireSigs; + Setting requireSigs{(Store*) this, + settings.requireSigs, + "require-sigs", "whether store paths should have a trusted signature on import"}; - PublicKeys publicKeys; + const PublicKeys & getPublicKeys(); public: + // Hack for build-remote.cc. + PathSet locksHeld; + /* Initialise the local store, upgrading the schema if necessary. */ LocalStore(const Params & params); @@ -100,57 +119,57 @@ public: std::string getUri() override; - bool isValidPathUncached(const Path & path) override; - - PathSet queryValidPaths(const PathSet & paths) override; + bool isValidPathUncached(const StorePath & path) override; - PathSet queryAllValidPaths() override; + StorePathSet queryValidPaths(const StorePathSet & paths, + SubstituteFlag maybeSubstitute = NoSubstitute) override; - void queryPathInfoUncached(const Path & path, - std::function)> success, - std::function failure) override; + StorePathSet queryAllValidPaths() override; - void queryReferrers(const Path & path, PathSet & referrers) override; + void queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept override; - PathSet queryValidDerivers(const Path & path) override; + void queryReferrers(const StorePath & path, StorePathSet & referrers) override; - PathSet queryDerivationOutputs(const Path & path) override; + StorePathSet queryValidDerivers(const StorePath & path) override; - StringSet queryDerivationOutputNames(const Path & path) override; + StorePathSet queryDerivationOutputs(const StorePath & path) override; - Path queryPathFromHashPart(const string & hashPart) override; + std::optional queryPathFromHashPart(const std::string & hashPart) override; - PathSet querySubstitutablePaths(const PathSet & paths) override; + StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const PathSet & paths, + void querySubstitutablePathInfos(const StorePathSet & paths, SubstitutablePathInfos & infos) override; - void addToStore(const ValidPathInfo & info, const ref & nar, - bool repair, bool dontCheckSigs, + void addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr accessor) override; - Path addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, - PathFilter & filter, bool repair) override; + StorePath addToStore(const string & name, const Path & srcPath, + FileIngestionMethod method, HashType hashAlgo, + PathFilter & filter, RepairFlag repair) override; /* Like addToStore(), but the contents of the path are contained in `dump', which is either a NAR serialisation (if recursive == true) or simply the contents of a regular file (if recursive == false). */ - Path addToStoreFromDump(const string & dump, const string & name, - bool recursive = true, HashType hashAlgo = htSHA256, bool repair = false); + StorePath addToStoreFromDump(const string & dump, const string & name, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; - Path addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair) override; + StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) override; - void buildPaths(const PathSet & paths, BuildMode buildMode) override; + void buildPaths( + const std::vector & paths, + BuildMode buildMode) override; - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; - void ensurePath(const Path & path) override; + void ensurePath(const StorePath & path) override; - void addTempRoot(const Path & path) override; + void addTempRoot(const StorePath & path) override; void addIndirectRoot(const Path & path) override; @@ -161,11 +180,11 @@ private: typedef std::shared_ptr FDPtr; typedef list FDs; - void readTempRoots(PathSet & tempRoots, FDs & fds); + void findTempRoots(FDs & fds, Roots & roots, bool censor); public: - Roots findRoots() override; + Roots findRoots(bool censor) override; void collectGarbage(const GCOptions & options, GCResults & results) override; @@ -178,7 +197,7 @@ public: /* Optimise a single store path. */ void optimisePath(const Path & path); - bool verifyStore(bool checkContents, bool repair) override; + bool verifyStore(bool checkContents, RepairFlag repair) override; /* Register the validity of a path, i.e., that `path' exists, that the paths referenced by it exists, and in the case of an output @@ -190,13 +209,19 @@ public: void registerValidPaths(const ValidPathInfos & infos); + unsigned int getProtocol() override; + void vacuumDB(); /* Repair the contents of the given path by redownloading it using a substituter (if available). */ - void repairPath(const Path & path); + void repairPath(const StorePath & path); - void addSignatures(const Path & storePath, const StringSet & sigs) override; + void addSignatures(const StorePath & storePath, const StringSet & sigs) override; + + /* If free disk space in /nix/store if below minFree, delete + garbage until it exceeds maxFree. */ + void autoGC(bool sync = true); private: @@ -206,17 +231,17 @@ private: void makeStoreWritable(); - uint64_t queryValidPathId(State & state, const Path & path); + uint64_t queryValidPathId(State & state, const StorePath & path); uint64_t addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs = true); - void invalidatePath(State & state, const Path & path); + void invalidatePath(State & state, const StorePath & path); /* Delete a path from the Nix store. */ - void invalidatePathChecked(const Path & path); + void invalidatePathChecked(const StorePath & path); - void verifyPath(const Path & path, const PathSet & store, - PathSet & done, PathSet & validPaths, bool repair, bool & errors); + void verifyPath(const Path & path, const StringSet & store, + PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); void updatePathInfo(State & state, const ValidPathInfo & info); @@ -231,34 +256,36 @@ private: void tryToDelete(GCState & state, const Path & path); - bool canReachRoot(GCState & state, PathSet & visited, const Path & path); + bool canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path); void deletePathRecursive(GCState & state, const Path & path); bool isActiveTempFile(const GCState & state, const Path & path, const string & suffix); - int openGCLock(LockType lockType); + AutoCloseFD openGCLock(LockType lockType); void findRoots(const Path & path, unsigned char type, Roots & roots); - void findRuntimeRoots(PathSet & roots); + void findRootsNoTemp(Roots & roots, bool censor); + + void findRuntimeRoots(Roots & roots, bool censor); void removeUnusedLinks(const GCState & state); Path createTempDirInStore(); - void checkDerivationOutputs(const Path & drvPath, const Derivation & drv); + void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv); typedef std::unordered_set InodeHash; InodeHash loadInodeHash(); Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash); - void optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash); + void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash); // Internal versions that are not wrapped in retry_sqlite. - bool isValidPath_(State & state, const Path & path); - void queryReferrers(State & state, const Path & path, PathSet & referrers); + bool isValidPath_(State & state, const StorePath & path); + void queryReferrers(State & state, const StorePath & path, StorePathSet & referrers); /* Add signatures to a ValidPathInfo using the secret keys specified by the ‘secret-key-files’ option. */ @@ -266,6 +293,8 @@ private: Path getRealStoreDir() override { return realStoreDir; } + void createUser(const std::string & userName, uid_t userId) override; + friend class DerivationGoal; friend class SubstitutionGoal; }; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index d17eb040d9d..91acef368fe 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -4,49 +4,35 @@ libstore_NAME = libnixstore libstore_DIR := $(d) -libstore_SOURCES := \ - $(d)/binary-cache-store.cc \ - $(d)/build.cc \ - $(d)/builtins.cc \ - $(d)/crypto.cc \ - $(d)/derivations.cc \ - $(d)/download.cc \ - $(d)/export-import.cc \ - $(d)/gc.cc \ - $(d)/globals.cc \ - $(d)/http-binary-cache-store.cc \ - $(d)/local-binary-cache-store.cc \ - $(d)/local-fs-store.cc \ - $(d)/local-store.cc \ - $(d)/misc.cc \ - $(d)/nar-accessor.cc \ - $(d)/nar-info.cc \ - $(d)/nar-info-disk-cache.cc \ - $(d)/optimise-store.cc \ - $(d)/pathlocks.cc \ - $(d)/profiles.cc \ - $(d)/references.cc \ - $(d)/remote-fs-accessor.cc \ - $(d)/remote-store.cc \ - $(d)/sqlite.cc \ - $(d)/ssh-store.cc \ - $(d)/store-api.cc \ - $(d)/ipfs-binary-cache-store.cc - -libstore_LIBS = libutil libformat +libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) + +libstore_LIBS = libutil libnixrust libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread +ifneq ($(OS), FreeBSD) + libstore_LDFLAGS += -ldl +endif + +ifeq ($(OS), Darwin) +libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb +endif + +$(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox))) ifeq ($(ENABLE_S3), 1) - libstore_LDFLAGS += -laws-cpp-sdk-s3 -laws-cpp-sdk-core - libstore_SOURCES += $(d)/s3-binary-cache-store.cc + libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core endif ifeq ($(OS), SunOS) libstore_LDFLAGS += -lsocket endif -libstore_CXXFLAGS = \ +ifeq ($(HAVE_SECCOMP), 1) + libstore_LDFLAGS += -lseccomp +endif + +libstore_CXXFLAGS += \ + -I src/libutil -I src/libstore \ -DNIX_PREFIX=\"$(prefix)\" \ -DNIX_STORE_DIR=\"$(storedir)\" \ -DNIX_DATA_DIR=\"$(datadir)\" \ @@ -55,14 +41,23 @@ libstore_CXXFLAGS = \ -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \ -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \ -DNIX_BIN_DIR=\"$(bindir)\" \ - -DBASH_PATH="\"$(bash)\"" + -DNIX_MAN_DIR=\"$(mandir)\" \ + -DLSOF=\"$(lsof)\" + +ifneq ($(sandbox_shell),) +libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\"" +endif + +$(d)/local-store.cc: $(d)/schema.sql.gen.hh -$(d)/local-store.cc: $(d)/schema.sql.hh +$(d)/build.cc: -%.sql.hh: %.sql - $(trace-gen) sed -e 's/"/\\"/g' -e 's/\(.*\)/"\1\\n"/' < $< > $@ || (rm $@ && exit 1) +%.gen.hh: % + @echo 'R"foo(' >> $@.tmp + $(trace-gen) cat $< >> $@.tmp + @echo ')foo"' >> $@.tmp + @mv $@.tmp $@ -clean-files += $(d)/schema.sql.hh +clean-files += $(d)/schema.sql.gen.hh $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644)) -$(eval $(call install-file-in, $(d)/sandbox-defaults.sb, $(datadir)/nix, 0644)) diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc new file mode 100644 index 00000000000..f848582dafd --- /dev/null +++ b/src/libstore/machines.cc @@ -0,0 +1,100 @@ +#include "machines.hh" +#include "util.hh" +#include "globals.hh" + +#include + +namespace nix { + +Machine::Machine(decltype(storeUri) storeUri, + decltype(systemTypes) systemTypes, + decltype(sshKey) sshKey, + decltype(maxJobs) maxJobs, + decltype(speedFactor) speedFactor, + decltype(supportedFeatures) supportedFeatures, + decltype(mandatoryFeatures) mandatoryFeatures, + decltype(sshPublicHostKey) sshPublicHostKey) : + storeUri( + // Backwards compatibility: if the URI is a hostname, + // prepend ssh://. + storeUri.find("://") != std::string::npos + || hasPrefix(storeUri, "local") + || hasPrefix(storeUri, "remote") + || hasPrefix(storeUri, "auto") + || hasPrefix(storeUri, "/") + ? storeUri + : "ssh://" + storeUri), + systemTypes(systemTypes), + sshKey(sshKey), + maxJobs(maxJobs), + speedFactor(std::max(1U, speedFactor)), + supportedFeatures(supportedFeatures), + mandatoryFeatures(mandatoryFeatures), + sshPublicHostKey(sshPublicHostKey) +{} + +bool Machine::allSupported(const std::set & features) const { + return std::all_of(features.begin(), features.end(), + [&](const string & feature) { + return supportedFeatures.count(feature) || + mandatoryFeatures.count(feature); + }); +} + +bool Machine::mandatoryMet(const std::set & features) const { + return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(), + [&](const string & feature) { + return features.count(feature); + }); +} + +void parseMachines(const std::string & s, Machines & machines) +{ + for (auto line : tokenizeString>(s, "\n;")) { + trim(line); + line.erase(std::find(line.begin(), line.end(), '#'), line.end()); + if (line.empty()) continue; + + if (line[0] == '@') { + auto file = trim(std::string(line, 1)); + try { + parseMachines(readFile(file), machines); + } catch (const SysError & e) { + if (e.errNo != ENOENT) + throw; + debug("cannot find machines file '%s'", file); + } + continue; + } + + auto tokens = tokenizeString>(line); + auto sz = tokens.size(); + if (sz < 1) + throw FormatError("bad machine specification '%s'", line); + + auto isSet = [&](size_t n) { + return tokens.size() > n && tokens[n] != "" && tokens[n] != "-"; + }; + + machines.emplace_back(tokens[0], + isSet(1) ? tokenizeString>(tokens[1], ",") : std::vector{settings.thisSystem}, + isSet(2) ? tokens[2] : "", + isSet(3) ? std::stoull(tokens[3]) : 1LL, + isSet(4) ? std::stoull(tokens[4]) : 1LL, + isSet(5) ? tokenizeString>(tokens[5], ",") : std::set{}, + isSet(6) ? tokenizeString>(tokens[6], ",") : std::set{}, + isSet(7) ? tokens[7] : ""); + } +} + +Machines getMachines() +{ + static auto machines = [&]() { + Machines machines; + parseMachines(settings.builders, machines); + return machines; + }(); + return machines; +} + +} diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh new file mode 100644 index 00000000000..de92eb924e4 --- /dev/null +++ b/src/libstore/machines.hh @@ -0,0 +1,39 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +struct Machine { + + const string storeUri; + const std::vector systemTypes; + const string sshKey; + const unsigned int maxJobs; + const unsigned int speedFactor; + const std::set supportedFeatures; + const std::set mandatoryFeatures; + const std::string sshPublicHostKey; + bool enabled = true; + + bool allSupported(const std::set & features) const; + + bool mandatoryMet(const std::set & features) const; + + Machine(decltype(storeUri) storeUri, + decltype(systemTypes) systemTypes, + decltype(sshKey) sshKey, + decltype(maxJobs) maxJobs, + decltype(speedFactor) speedFactor, + decltype(supportedFeatures) supportedFeatures, + decltype(mandatoryFeatures) mandatoryFeatures, + decltype(sshPublicHostKey) sshPublicHostKey); +}; + +typedef std::vector Machines; + +void parseMachines(const std::string & s, Machines & machines); + +Machines getMachines(); + +} diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 9a88cdc317b..9c47fe52454 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -1,4 +1,5 @@ #include "derivations.hh" +#include "parsed-derivations.hh" #include "globals.hh" #include "local-store.hh" #include "store-api.hh" @@ -8,13 +9,13 @@ namespace nix { -void Store::computeFSClosure(const PathSet & startPaths, - PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) +void Store::computeFSClosure(const StorePathSet & startPaths, + StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) { struct State { size_t pending; - PathSet & paths; + StorePathSet & paths; std::exception_ptr exc; }; @@ -28,44 +29,47 @@ void Store::computeFSClosure(const PathSet & startPaths, { auto state(state_.lock()); if (state->exc) return; - if (state->paths.count(path)) return; - state->paths.insert(path); + if (!state->paths.insert(parseStorePath(path)).second) return; state->pending++; } - queryPathInfo(path, - [&, path](ref info) { - // FIXME: calls to isValidPath() should be async + queryPathInfo(parseStorePath(path), {[&, pathS(path)](std::future> fut) { + // FIXME: calls to isValidPath() should be async + + try { + auto info = fut.get(); + + auto path = parseStorePath(pathS); if (flipDirection) { - PathSet referrers; + StorePathSet referrers; queryReferrers(path, referrers); for (auto & ref : referrers) if (ref != path) - enqueue(ref); + enqueue(printStorePath(ref)); if (includeOutputs) for (auto & i : queryValidDerivers(path)) - enqueue(i); + enqueue(printStorePath(i)); - if (includeDerivers && isDerivation(path)) + if (includeDerivers && path.isDerivation()) for (auto & i : queryDerivationOutputs(path)) if (isValidPath(i) && queryPathInfo(i)->deriver == path) - enqueue(i); + enqueue(printStorePath(i)); } else { for (auto & ref : info->references) if (ref != path) - enqueue(ref); + enqueue(printStorePath(ref)); - if (includeOutputs && isDerivation(path)) + if (includeOutputs && path.isDerivation()) for (auto & i : queryDerivationOutputs(path)) - if (isValidPath(i)) enqueue(i); + if (isValidPath(i)) enqueue(printStorePath(i)); - if (includeDerivers && isValidPath(info->deriver)) - enqueue(info->deriver); + if (includeDerivers && info->deriver && isValidPath(*info->deriver)) + enqueue(printStorePath(*info->deriver)); } @@ -75,18 +79,17 @@ void Store::computeFSClosure(const PathSet & startPaths, if (!--state->pending) done.notify_one(); } - }, - - [&, path](std::exception_ptr exc) { + } catch (...) { auto state(state_.lock()); - if (!state->exc) state->exc = exc; + if (!state->exc) state->exc = std::current_exception(); assert(state->pending); if (!--state->pending) done.notify_one(); - }); + }; + }}); }; for (auto & startPath : startPaths) - enqueue(startPath); + enqueue(printStorePath(startPath)); { auto state(state_.lock()); @@ -96,25 +99,29 @@ void Store::computeFSClosure(const PathSet & startPaths, } -void Store::computeFSClosure(const Path & startPath, - PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) +void Store::computeFSClosure(const StorePath & startPath, + StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) { - computeFSClosure(PathSet{startPath}, paths_, flipDirection, includeOutputs, includeDerivers); + StorePathSet paths; + paths.insert(startPath.clone()); + computeFSClosure(paths, paths_, flipDirection, includeOutputs, includeDerivers); } -void Store::queryMissing(const PathSet & targets, - PathSet & willBuild_, PathSet & willSubstitute_, PathSet & unknown_, +void Store::queryMissing(const std::vector & targets, + StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, unsigned long long & downloadSize_, unsigned long long & narSize_) { + Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths"); + downloadSize_ = narSize_ = 0; ThreadPool pool; struct State { - PathSet done; - PathSet & unknown, & willSubstitute, & willBuild; + std::unordered_set done; + StorePathSet & unknown, & willSubstitute, & willBuild; unsigned long long & downloadSize; unsigned long long & narSize; }; @@ -123,31 +130,36 @@ void Store::queryMissing(const PathSet & targets, { size_t left; bool done = false; - PathSet outPaths; + StorePathSet outPaths; DrvState(size_t left) : left(left) { } }; - Sync state_(State{PathSet(), unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_}); + Sync state_(State{{}, unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_}); - std::function doPath; + std::function doPath; - auto mustBuildDrv = [&](const Path & drvPath, const Derivation & drv) { + auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) { { auto state(state_.lock()); - state->willBuild.insert(drvPath); + state->willBuild.insert(drvPath.clone()); } for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, makeDrvPathWithOutputs(i.first, i.second))); + pool.enqueue(std::bind(doPath, StorePathWithOutputs(i.first, i.second))); }; auto checkOutput = [&]( - const Path & drvPath, ref drv, const Path & outPath, ref> drvState_) + const Path & drvPathS, ref drv, const Path & outPathS, ref> drvState_) { if (drvState_->lock()->done) return; + auto drvPath = parseStorePath(drvPathS); + auto outPath = parseStorePath(outPathS); + SubstitutablePathInfos infos; - querySubstitutablePathInfos({outPath}, infos); + StorePathSet paths; // FIXME + paths.insert(outPath.clone()); + querySubstitutablePathInfos(paths, infos); if (infos.empty()) { drvState_->lock()->done = true; @@ -158,74 +170,74 @@ void Store::queryMissing(const PathSet & targets, if (drvState->done) return; assert(drvState->left); drvState->left--; - drvState->outPaths.insert(outPath); + drvState->outPaths.insert(outPath.clone()); if (!drvState->left) { for (auto & path : drvState->outPaths) - pool.enqueue(std::bind(doPath, path)); + pool.enqueue(std::bind(doPath, StorePathWithOutputs(path.clone()))); } } } }; - doPath = [&](const Path & path) { + doPath = [&](const StorePathWithOutputs & path) { { auto state(state_.lock()); - if (state->done.count(path)) return; - state->done.insert(path); + if (!state->done.insert(path.to_string(*this)).second) return; } - DrvPathWithOutputs i2 = parseDrvPathWithOutputs(path); - - if (isDerivation(i2.first)) { - if (!isValidPath(i2.first)) { + if (path.path.isDerivation()) { + if (!isValidPath(path.path)) { // FIXME: we could try to substitute the derivation. auto state(state_.lock()); - state->unknown.insert(path); + state->unknown.insert(path.path.clone()); return; } - Derivation drv = derivationFromPath(i2.first); + auto drv = make_ref(derivationFromPath(path.path)); + ParsedDerivation parsedDrv(path.path.clone(), *drv); PathSet invalid; - for (auto & j : drv.outputs) - if (wantOutput(j.first, i2.second) + for (auto & j : drv->outputs) + if (wantOutput(j.first, path.outputs) && !isValidPath(j.second.path)) - invalid.insert(j.second.path); + invalid.insert(printStorePath(j.second.path)); if (invalid.empty()) return; - if (settings.useSubstitutes && drv.substitutesAllowed()) { + if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref>(DrvState(invalid.size())); for (auto & output : invalid) - pool.enqueue(std::bind(checkOutput, i2.first, make_ref(drv), output, drvState)); + pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState)); } else - mustBuildDrv(i2.first, drv); + mustBuildDrv(path.path, *drv); } else { - if (isValidPath(path)) return; + if (isValidPath(path.path)) return; SubstitutablePathInfos infos; - querySubstitutablePathInfos({path}, infos); + StorePathSet paths; // FIXME + paths.insert(path.path.clone()); + querySubstitutablePathInfos(paths, infos); if (infos.empty()) { auto state(state_.lock()); - state->unknown.insert(path); + state->unknown.insert(path.path.clone()); return; } - auto info = infos.find(path); + auto info = infos.find(path.path); assert(info != infos.end()); { auto state(state_.lock()); - state->willSubstitute.insert(path); + state->willSubstitute.insert(path.path.clone()); state->downloadSize += info->second.downloadSize; state->narSize += info->second.narSize; } for (auto & ref : info->second.references) - pool.enqueue(std::bind(doPath, ref)); + pool.enqueue(std::bind(doPath, StorePathWithOutputs(ref))); } }; @@ -236,40 +248,42 @@ void Store::queryMissing(const PathSet & targets, } -Paths Store::topoSortPaths(const PathSet & paths) +StorePaths Store::topoSortPaths(const StorePathSet & paths) { - Paths sorted; - PathSet visited, parents; + StorePaths sorted; + StorePathSet visited, parents; - std::function dfsVisit; + std::function dfsVisit; - dfsVisit = [&](const Path & path, const Path * parent) { - if (parents.find(path) != parents.end()) - throw BuildError(format("cycle detected in the references of ‘%1%’ from ‘%2%’") % path % *parent); + dfsVisit = [&](const StorePath & path, const StorePath * parent) { + if (parents.count(path)) + throw BuildError("cycle detected in the references of '%s' from '%s'", + printStorePath(path), printStorePath(*parent)); - if (visited.find(path) != visited.end()) return; - visited.insert(path); - parents.insert(path); + if (!visited.insert(path.clone()).second) return; + parents.insert(path.clone()); - PathSet references; + StorePathSet references; try { - references = queryPathInfo(path)->references; + references = cloneStorePathSet(queryPathInfo(path)->references); } catch (InvalidPath &) { } for (auto & i : references) /* Don't traverse into paths that don't exist. That can happen due to substitutes for non-existent paths. */ - if (i != path && paths.find(i) != paths.end()) + if (i != path && paths.count(i)) dfsVisit(i, &path); - sorted.push_front(path); + sorted.push_back(path.clone()); parents.erase(path); }; for (auto & i : paths) dfsVisit(i, nullptr); + std::reverse(sorted.begin(), sorted.end()); + return sorted; } diff --git a/src/libexpr/names.cc b/src/libstore/names.cc similarity index 92% rename from src/libexpr/names.cc rename to src/libstore/names.cc index 6d78d211612..d1c8a6101f8 100644 --- a/src/libexpr/names.cc +++ b/src/libstore/names.cc @@ -16,14 +16,14 @@ DrvName::DrvName() a letter. The `version' part is the rest (excluding the separating dash). E.g., `apache-httpd-2.0.48' is parsed to (`apache-httpd', '2.0.48'). */ -DrvName::DrvName(const string & s) : hits(0) +DrvName::DrvName(std::string_view s) : hits(0) { - name = fullName = s; + name = fullName = std::string(s); for (unsigned int i = 0; i < s.size(); ++i) { /* !!! isalpha/isdigit are affected by the locale. */ if (s[i] == '-' && i + 1 < s.size() && !isalpha(s[i + 1])) { - name = string(s, 0, i); - version = string(s, i + 1); + name = s.substr(0, i); + version = s.substr(i + 1); break; } } @@ -41,7 +41,7 @@ bool DrvName::matches(DrvName & n) } -static string nextComponent(string::const_iterator & p, +string nextComponent(string::const_iterator & p, const string::const_iterator end) { /* Skip any dots and dashes (component separators). */ diff --git a/src/libexpr/names.hh b/src/libstore/names.hh similarity index 78% rename from src/libexpr/names.hh rename to src/libstore/names.hh index 9667fc96fd0..00e14b8c797 100644 --- a/src/libexpr/names.hh +++ b/src/libstore/names.hh @@ -15,7 +15,7 @@ struct DrvName unsigned int hits; DrvName(); - DrvName(const string & s); + DrvName(std::string_view s); bool matches(DrvName & n); private: @@ -24,6 +24,8 @@ private: typedef list DrvNames; +string nextComponent(string::const_iterator & p, + const string::const_iterator end); int compareVersions(const string & v1, const string & v2); DrvNames drvNamesFromArgs(const Strings & opArgs); diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 4cb5de7449e..ca663d83759 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -1,136 +1,216 @@ #include "nar-accessor.hh" #include "archive.hh" +#include "json.hh" #include +#include +#include + +#include namespace nix { struct NarMember { - FSAccessor::Type type; + FSAccessor::Type type = FSAccessor::Type::tMissing; - bool isExecutable; + bool isExecutable = false; /* If this is a regular file, position of the contents of this file in the NAR. */ - size_t start, size; + size_t start = 0, size = 0; std::string target; + + /* If this is a directory, all the children of the directory. */ + std::map children; }; -struct NarIndexer : ParseSink, StringSource +struct NarAccessor : public FSAccessor { - // FIXME: should store this as a tree. Now we're vulnerable to - // O(nm) memory consumption (e.g. for x_0/.../x_n/{y_0..y_m}). - typedef std::map Members; - Members members; + std::shared_ptr nar; - Path currentPath; - std::string currentStart; - bool isExec = false; + GetNarBytes getNarBytes; - NarIndexer(const std::string & nar) : StringSource(nar) - { - } + NarMember root; - void createDirectory(const Path & path) override + struct NarIndexer : ParseSink, StringSource { - members.emplace(path, - NarMember{FSAccessor::Type::tDirectory, false, 0, 0}); - } + NarAccessor & acc; - void createRegularFile(const Path & path) override - { - currentPath = path; - } + std::stack parents; - void isExecutable() override - { - isExec = true; - } + std::string currentStart; + bool isExec = false; - void preallocateContents(unsigned long long size) override - { - currentStart = string(s, pos, 16); - assert(size <= std::numeric_limits::max()); - members.emplace(currentPath, - NarMember{FSAccessor::Type::tRegular, isExec, pos, (size_t) size}); - } + NarIndexer(NarAccessor & acc, const std::string & nar) + : StringSource(nar), acc(acc) + { } - void receiveContents(unsigned char * data, unsigned int len) override - { - // Sanity check - if (!currentStart.empty()) { - assert(len < 16 || currentStart == string((char *) data, 16)); - currentStart.clear(); + void createMember(const Path & path, NarMember member) { + size_t level = std::count(path.begin(), path.end(), '/'); + while (parents.size() > level) parents.pop(); + + if (parents.empty()) { + acc.root = std::move(member); + parents.push(&acc.root); + } else { + if (parents.top()->type != FSAccessor::Type::tDirectory) + throw Error("NAR file missing parent directory of path '%s'", path); + auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member)); + parents.push(&result.first->second); + } } - } - void createSymlink(const Path & path, const string & target) override + void createDirectory(const Path & path) override + { + createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0}); + } + + void createRegularFile(const Path & path) override + { + createMember(path, {FSAccessor::Type::tRegular, false, 0, 0}); + } + + void isExecutable() override + { + parents.top()->isExecutable = true; + } + + void preallocateContents(unsigned long long size) override + { + currentStart = string(s, pos, 16); + assert(size <= std::numeric_limits::max()); + parents.top()->size = (size_t)size; + parents.top()->start = pos; + } + + void receiveContents(unsigned char * data, unsigned int len) override + { + // Sanity check + if (!currentStart.empty()) { + assert(len < 16 || currentStart == string((char *) data, 16)); + currentStart.clear(); + } + } + + void createSymlink(const Path & path, const string & target) override + { + createMember(path, + NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); + } + }; + + NarAccessor(ref nar) : nar(nar) { - members.emplace(path, - NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); + NarIndexer indexer(*this, *nar); + parseDump(indexer, indexer); } - Members::iterator find(const Path & path) + NarAccessor(const std::string & listing, GetNarBytes getNarBytes) + : getNarBytes(getNarBytes) { - auto i = members.find(path); - if (i == members.end()) - throw Error(format("NAR file does not contain path ‘%1%’") % path); - return i; - } -}; + using json = nlohmann::json; -struct NarAccessor : public FSAccessor -{ - ref nar; - NarIndexer indexer; + std::function recurse; + + recurse = [&](NarMember & member, json & v) { + std::string type = v["type"]; - NarAccessor(ref nar) : nar(nar), indexer(*nar) + if (type == "directory") { + member.type = FSAccessor::Type::tDirectory; + for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) { + std::string name = i.key(); + recurse(member.children[name], i.value()); + } + } else if (type == "regular") { + member.type = FSAccessor::Type::tRegular; + member.size = v["size"]; + member.isExecutable = v.value("executable", false); + member.start = v["narOffset"]; + } else if (type == "symlink") { + member.type = FSAccessor::Type::tSymlink; + member.target = v.value("target", ""); + } else return; + }; + + json v = json::parse(listing); + recurse(root, v); + } + + NarMember * find(const Path & path) { - parseDump(indexer, indexer); + Path canon = path == "" ? "" : canonPath(path); + NarMember * current = &root; + auto end = path.end(); + for (auto it = path.begin(); it != end; ) { + // because it != end, the remaining component is non-empty so we need + // a directory + if (current->type != FSAccessor::Type::tDirectory) return nullptr; + + // skip slash (canonPath above ensures that this is always a slash) + assert(*it == '/'); + it += 1; + + // lookup current component + auto next = std::find(it, end, '/'); + auto child = current->children.find(std::string(it, next)); + if (child == current->children.end()) return nullptr; + current = &child->second; + + it = next; + } + + return current; + } + + NarMember & get(const Path & path) { + auto result = find(path); + if (result == nullptr) + throw Error("NAR file does not contain path '%1%'", path); + return *result; } Stat stat(const Path & path) override { - auto i = indexer.members.find(path); - if (i == indexer.members.end()) + auto i = find(path); + if (i == nullptr) return {FSAccessor::Type::tMissing, 0, false}; - return {i->second.type, i->second.size, i->second.isExecutable}; + return {i->type, i->size, i->isExecutable, i->start}; } StringSet readDirectory(const Path & path) override { - auto i = indexer.find(path); + auto i = get(path); - if (i->second.type != FSAccessor::Type::tDirectory) - throw Error(format("path ‘%1%’ inside NAR file is not a directory") % path); + if (i.type != FSAccessor::Type::tDirectory) + throw Error("path '%1%' inside NAR file is not a directory", path); - ++i; StringSet res; - while (i != indexer.members.end() && isInDir(i->first, path)) { - // FIXME: really bad performance. - if (i->first.find('/', path.size() + 1) == std::string::npos) - res.insert(std::string(i->first, path.size() + 1)); - ++i; - } + for (auto & child : i.children) + res.insert(child.first); + return res; } std::string readFile(const Path & path) override { - auto i = indexer.find(path); - if (i->second.type != FSAccessor::Type::tRegular) - throw Error(format("path ‘%1%’ inside NAR file is not a regular file") % path); - return std::string(*nar, i->second.start, i->second.size); + auto i = get(path); + if (i.type != FSAccessor::Type::tRegular) + throw Error("path '%1%' inside NAR file is not a regular file", path); + + if (getNarBytes) return getNarBytes(i.start, i.size); + + assert(nar); + return std::string(*nar, i.start, i.size); } std::string readLink(const Path & path) override { - auto i = indexer.find(path); - if (i->second.type != FSAccessor::Type::tSymlink) - throw Error(format("path ‘%1%’ inside NAR file is not a symlink") % path); - return i->second.target; + auto i = get(path); + if (i.type != FSAccessor::Type::tSymlink) + throw Error("path '%1%' inside NAR file is not a symlink", path); + return i.target; } }; @@ -139,4 +219,48 @@ ref makeNarAccessor(ref nar) return make_ref(nar); } +ref makeLazyNarAccessor(const std::string & listing, + GetNarBytes getNarBytes) +{ + return make_ref(listing, getNarBytes); +} + +void listNar(JSONPlaceholder & res, ref accessor, + const Path & path, bool recurse) +{ + auto st = accessor->stat(path); + + auto obj = res.object(); + + switch (st.type) { + case FSAccessor::Type::tRegular: + obj.attr("type", "regular"); + obj.attr("size", st.fileSize); + if (st.isExecutable) + obj.attr("executable", true); + if (st.narOffset) + obj.attr("narOffset", st.narOffset); + break; + case FSAccessor::Type::tDirectory: + obj.attr("type", "directory"); + { + auto res2 = obj.object("entries"); + for (auto & name : accessor->readDirectory(path)) { + if (recurse) { + auto res3 = res2.placeholder(name); + listNar(res3, accessor, path + "/" + name, true); + } else + res2.object(name); + } + } + break; + case FSAccessor::Type::tSymlink: + obj.attr("type", "symlink"); + obj.attr("target", accessor->readLink(path)); + break; + default: + throw Error("path '%s' does not exist in NAR", path); + } +} + } diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 83c570be4c7..2871199de16 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -1,5 +1,7 @@ #pragma once +#include + #include "fs-accessor.hh" namespace nix { @@ -8,4 +10,21 @@ namespace nix { file. */ ref makeNarAccessor(ref nar); +/* Create a NAR accessor from a NAR listing (in the format produced by + listNar()). The callback getNarBytes(offset, length) is used by the + readFile() method of the accessor to get the contents of files + inside the NAR. */ +typedef std::function GetNarBytes; + +ref makeLazyNarAccessor( + const std::string & listing, + GetNarBytes getNarBytes); + +class JSONPlaceholder; + +/* Write a JSON representation of the contents of a NAR (except file + contents). */ +void listNar(JSONPlaceholder & res, ref accessor, + const Path & path, bool recurse); + } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index d28ff42c7f2..e8cf1d17724 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -1,6 +1,7 @@ #include "nar-info-disk-cache.hh" #include "sync.hh" #include "sqlite.hh" +#include "globals.hh" #include @@ -30,19 +31,16 @@ create table if not exists NARs ( refs text, deriver text, sigs text, + ca text, timestamp integer not null, present integer not null, primary key (cache, hashPart), foreign key (cache) references BinaryCaches(id) on delete cascade ); -create table if not exists NARExistence ( - cache integer not null, - storePath text not null, - exist integer not null, - timestamp integer not null, - primary key (cache, storePath), - foreign key (cache) references BinaryCaches(id) on delete cascade +create table if not exists LastPurge ( + dummy text primary key, + value integer ); )sql"; @@ -51,8 +49,8 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache { public: - /* How long negative lookups are valid. */ - const int ttlNegative = 3600; + /* How often to purge expired entries from the cache. */ + const int purgeInterval = 24 * 3600; struct Cache { @@ -65,7 +63,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache struct State { SQLite db; - SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR; + SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR, purgeCache; std::map caches; }; @@ -75,17 +73,12 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache { auto state(_state.lock()); - Path dbPath = getCacheDir() + "/nix/binary-cache-v5.sqlite"; + Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite"; createDirs(dirOf(dbPath)); state->db = SQLite(dbPath); - if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK) - throwSQLiteError(state->db, "setting timeout"); - - // We can always reproduce the cache. - state->db.exec("pragma synchronous = off"); - state->db.exec("pragma main.journal_mode = truncate"); + state->db.isCache(); state->db.exec(schema); @@ -97,13 +90,36 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache state->insertNAR.create(state->db, "insert or replace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, " - "narSize, refs, deriver, sigs, timestamp, present) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)"); + "narSize, refs, deriver, sigs, ca, timestamp, present) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)"); state->insertMissingNAR.create(state->db, "insert or replace into NARs(cache, hashPart, timestamp, present) values (?, ?, ?, 0)"); state->queryNAR.create(state->db, - "select * from NARs where cache = ? and hashPart = ?"); + "select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))"); + + /* Periodically purge expired entries from the database. */ + retrySQLite([&]() { + auto now = time(0); + + SQLiteStmt queryLastPurge(state->db, "select value from LastPurge"); + auto queryLastPurge_(queryLastPurge.use()); + + if (!queryLastPurge_.next() || queryLastPurge_.getInt(0) < now - purgeInterval) { + SQLiteStmt(state->db, + "delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))") + .use() + (now - settings.ttlNegativeNarInfoCache) + (now - settings.ttlPositiveNarInfoCache) + .exec(); + + debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db)); + + SQLiteStmt(state->db, + "insert or replace into LastPurge(dummy, value) values ('', ?)") + .use()(now).exec(); + } + }); } Cache & getCache(State & state, const std::string & uri) @@ -115,121 +131,129 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override { - auto state(_state.lock()); + retrySQLite([&]() { + auto state(_state.lock()); - // FIXME: race + // FIXME: race - state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec(); - assert(sqlite3_changes(state->db) == 1); - state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority}; + state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec(); + assert(sqlite3_changes(state->db) == 1); + state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority}; + }); } - bool cacheExists(const std::string & uri, - bool & wantMassQuery, int & priority) override + std::optional cacheExists(const std::string & uri) override { - auto state(_state.lock()); - - auto i = state->caches.find(uri); - if (i == state->caches.end()) { - auto queryCache(state->queryCache.use()(uri)); - if (!queryCache.next()) return false; - state->caches.emplace(uri, - Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)}); - } - - auto & cache(getCache(*state, uri)); - - wantMassQuery = cache.wantMassQuery; - priority = cache.priority; - - return true; + return retrySQLite>([&]() -> std::optional { + auto state(_state.lock()); + + auto i = state->caches.find(uri); + if (i == state->caches.end()) { + auto queryCache(state->queryCache.use()(uri)); + if (!queryCache.next()) + return std::nullopt; + state->caches.emplace(uri, + Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)}); + } + + auto & cache(getCache(*state, uri)); + + return CacheInfo { + .wantMassQuery = cache.wantMassQuery, + .priority = cache.priority + }; + }); } std::pair> lookupNarInfo( const std::string & uri, const std::string & hashPart) override { - auto state(_state.lock()); - - auto & cache(getCache(*state, uri)); - - auto queryNAR(state->queryNAR.use()(cache.id)(hashPart)); - - if (!queryNAR.next()) - // FIXME: check NARExistence - return {oUnknown, 0}; - - if (!queryNAR.getInt(13)) - return {oInvalid, 0}; + return retrySQLite>>( + [&]() -> std::pair> { + auto state(_state.lock()); - auto narInfo = make_ref(); + auto & cache(getCache(*state, uri)); - // FIXME: implement TTL. + auto now = time(0); - auto namePart = queryNAR.getStr(2); - narInfo->path = cache.storeDir + "/" + - hashPart + (namePart.empty() ? "" : "-" + namePart); - narInfo->url = queryNAR.getStr(3); - narInfo->compression = queryNAR.getStr(4); - if (!queryNAR.isNull(5)) - narInfo->fileHash = parseHash(queryNAR.getStr(5)); - narInfo->fileSize = queryNAR.getInt(6); - narInfo->narHash = parseHash(queryNAR.getStr(7)); - narInfo->narSize = queryNAR.getInt(8); - for (auto & r : tokenizeString(queryNAR.getStr(9), " ")) - narInfo->references.insert(cache.storeDir + "/" + r); - if (!queryNAR.isNull(10)) - narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(10); - for (auto & sig : tokenizeString(queryNAR.getStr(11), " ")) - narInfo->sigs.insert(sig); - - return {oValid, narInfo}; + auto queryNAR(state->queryNAR.use() + (cache.id) + (hashPart) + (now - settings.ttlNegativeNarInfoCache) + (now - settings.ttlPositiveNarInfoCache)); + + if (!queryNAR.next()) + return {oUnknown, 0}; + + if (!queryNAR.getInt(0)) + return {oInvalid, 0}; + + auto namePart = queryNAR.getStr(1); + auto narInfo = make_ref(StorePath::fromBaseName(hashPart + "-" + namePart)); + narInfo->url = queryNAR.getStr(2); + narInfo->compression = queryNAR.getStr(3); + if (!queryNAR.isNull(4)) + narInfo->fileHash = Hash(queryNAR.getStr(4)); + narInfo->fileSize = queryNAR.getInt(5); + narInfo->narHash = Hash(queryNAR.getStr(6)); + narInfo->narSize = queryNAR.getInt(7); + for (auto & r : tokenizeString(queryNAR.getStr(8), " ")) + narInfo->references.insert(StorePath::fromBaseName(r)); + if (!queryNAR.isNull(9)) + narInfo->deriver = StorePath::fromBaseName(queryNAR.getStr(9)); + for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) + narInfo->sigs.insert(sig); + narInfo->ca = queryNAR.getStr(11); + + return {oValid, narInfo}; + }); } void upsertNarInfo( const std::string & uri, const std::string & hashPart, - std::shared_ptr info) override + std::shared_ptr info) override { - auto state(_state.lock()); - - auto & cache(getCache(*state, uri)); - - if (info) { - - auto narInfo = std::dynamic_pointer_cast(info); - - assert(hashPart == storePathToHash(info->path)); - - state->insertNAR.use() - (cache.id) - (hashPart) - (storePathToName(info->path)) - (narInfo ? narInfo->url : "", narInfo != 0) - (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash) - (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash.to_string()) - (info->narSize) - (concatStringsSep(" ", info->shortRefs())) - (info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "") - (concatStringsSep(" ", info->sigs)) - (time(0)).exec(); - - } else { - state->insertMissingNAR.use() - (cache.id) - (hashPart) - (time(0)).exec(); - } + retrySQLite([&]() { + auto state(_state.lock()); + + auto & cache(getCache(*state, uri)); + + if (info) { + + auto narInfo = std::dynamic_pointer_cast(info); + + //assert(hashPart == storePathToHash(info->path)); + + state->insertNAR.use() + (cache.id) + (hashPart) + (std::string(info->path.name())) + (narInfo ? narInfo->url : "", narInfo != 0) + (narInfo ? narInfo->compression : "", narInfo != 0) + (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string(Base32, true) : "", narInfo && narInfo->fileHash) + (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) + (info->narHash.to_string(Base32, true)) + (info->narSize) + (concatStringsSep(" ", info->shortRefs())) + (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) + (concatStringsSep(" ", info->sigs)) + (info->ca) + (time(0)).exec(); + + } else { + state->insertMissingNAR.use() + (cache.id) + (hashPart) + (time(0)).exec(); + } + }); } }; ref getNarInfoDiskCache() { - static Sync> cache; - - auto cache_(cache.lock()); - if (!*cache_) *cache_ = std::make_shared(); - return ref(*cache_); + static ref cache = make_ref(); + return cache; } } diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index 88d909732db..04de2c5eb3c 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -10,18 +10,25 @@ class NarInfoDiskCache public: typedef enum { oValid, oInvalid, oUnknown } Outcome; + virtual ~NarInfoDiskCache() { } + virtual void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) = 0; - virtual bool cacheExists(const std::string & uri, - bool & wantMassQuery, int & priority) = 0; + struct CacheInfo + { + bool wantMassQuery; + int priority; + }; + + virtual std::optional cacheExists(const std::string & uri) = 0; virtual std::pair> lookupNarInfo( const std::string & uri, const std::string & hashPart) = 0; virtual void upsertNarInfo( const std::string & uri, const std::string & hashPart, - std::shared_ptr info) = 0; + std::shared_ptr info) = 0; }; /* Return a singleton cache object that can be used concurrently by diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 201cac671a5..2322847232b 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -4,20 +4,23 @@ namespace nix { NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) + : ValidPathInfo(StorePath::dummy.clone()) // FIXME: hack { auto corrupt = [&]() { - throw Error(format("NAR info file ‘%1%’ is corrupt") % whence); + throw Error("NAR info file '%1%' is corrupt", whence); }; auto parseHashField = [&](const string & s) { try { - return parseHash(s); + return Hash(s); } catch (BadHash &) { corrupt(); return Hash(); // never reached } }; + bool havePath = false; + size_t pos = 0; while (pos < s.size()) { @@ -32,8 +35,8 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & std::string value(s, colon + 2, eol - colon - 2); if (name == "StorePath") { - if (!store.isStorePath(value)) corrupt(); - path = value; + path = store.parseStorePath(value); + havePath = true; } else if (name == "URL") url = value; @@ -52,16 +55,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "References") { auto refs = tokenizeString(value, " "); if (!references.empty()) corrupt(); - for (auto & r : refs) { - auto r2 = store.storeDir + "/" + r; - if (!store.isStorePath(r2)) corrupt(); - references.insert(r2); - } + for (auto & r : refs) + references.insert(StorePath::fromBaseName(r)); } else if (name == "Deriver") { - auto p = store.storeDir + "/" + value; - if (!store.isStorePath(p)) corrupt(); - deriver = p; + if (value != "unknown-deriver") + deriver = StorePath::fromBaseName(value); } else if (name == "System") system = value; @@ -77,27 +76,27 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & if (compression == "") compression = "bzip2"; - if (path.empty() || url.empty() || narSize == 0 || !narHash) corrupt(); + if (!havePath || url.empty() || narSize == 0 || !narHash) corrupt(); } -std::string NarInfo::to_string() const +std::string NarInfo::to_string(const Store & store) const { std::string res; - res += "StorePath: " + path + "\n"; + res += "StorePath: " + store.printStorePath(path) + "\n"; res += "URL: " + url + "\n"; assert(compression != ""); res += "Compression: " + compression + "\n"; assert(fileHash.type == htSHA256); - res += "FileHash: sha256:" + printHash32(fileHash) + "\n"; + res += "FileHash: " + fileHash.to_string(Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; assert(narHash.type == htSHA256); - res += "NarHash: sha256:" + printHash32(narHash) + "\n"; + res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; - if (!deriver.empty()) - res += "Deriver: " + baseNameOf(deriver) + "\n"; + if (deriver) + res += "Deriver: " + std::string(deriver->to_string()) + "\n"; if (!system.empty()) res += "System: " + system + "\n"; diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 4995061fbb6..373c3342767 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -14,11 +14,12 @@ struct NarInfo : ValidPathInfo uint64_t fileSize = 0; std::string system; - NarInfo() { } + NarInfo() = delete; + NarInfo(StorePath && path) : ValidPathInfo(std::move(path)) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); - std::string to_string() const; + std::string to_string(const Store & store) const; }; } diff --git a/src/libstore/nix-store.pc.in b/src/libstore/nix-store.pc.in index 3f1a2d83d2f..6d67b1e0380 100644 --- a/src/libstore/nix-store.pc.in +++ b/src/libstore/nix-store.pc.in @@ -5,5 +5,5 @@ includedir=@includedir@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ -Libs: -L${libdir} -lnixstore -lnixutil -lnixformat -Cflags: -I${includedir}/nix +Libs: -L${libdir} -lnixstore -lnixutil +Cflags: -I${includedir}/nix -std=c++17 diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index b71c7e905ff..b2b2412a318 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -1,5 +1,3 @@ -#include "config.h" - #include "util.hh" #include "local-store.hh" #include "globals.hh" @@ -11,6 +9,7 @@ #include #include #include +#include namespace nix { @@ -20,9 +19,9 @@ static void makeWritable(const Path & path) { struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path ‘%1%’") % path); + throw SysError("getting attributes of path '%1%'", path); if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) - throw SysError(format("changing writability of ‘%1%’") % path); + throw SysError("changing writability of '%1%'", path); } @@ -48,7 +47,7 @@ LocalStore::InodeHash LocalStore::loadInodeHash() InodeHash inodeHash; AutoCloseDir dir(opendir(linksDir.c_str())); - if (!dir) throw SysError(format("opening directory ‘%1%’") % linksDir); + if (!dir) throw SysError("opening directory '%1%'", linksDir); struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { /* sic */ @@ -56,7 +55,7 @@ LocalStore::InodeHash LocalStore::loadInodeHash() // We don't care if we hit non-hash files, anything goes inodeHash.insert(dirent->d_ino); } - if (errno) throw SysError(format("reading directory ‘%1%’") % linksDir); + if (errno) throw SysError("reading directory '%1%'", linksDir); printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size()); @@ -69,14 +68,14 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa Strings names; AutoCloseDir dir(opendir(path.c_str())); - if (!dir) throw SysError(format("opening directory ‘%1%’") % path); + if (!dir) throw SysError("opening directory '%1%'", path); struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { /* sic */ checkInterrupt(); if (inodeHash.count(dirent->d_ino)) { - debug(format("‘%1%’ is already linked") % dirent->d_name); + debug(format("'%1%' is already linked") % dirent->d_name); continue; } @@ -84,24 +83,38 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa if (name == "." || name == "..") continue; names.push_back(name); } - if (errno) throw SysError(format("reading directory ‘%1%’") % path); + if (errno) throw SysError("reading directory '%1%'", path); return names; } -void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash) +void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, + const Path & path, InodeHash & inodeHash) { checkInterrupt(); struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path ‘%1%’") % path); + throw SysError("getting attributes of path '%1%'", path); + +#if __APPLE__ + /* HFS/macOS has some undocumented security feature disabling hardlinking for + special files within .app dirs. *.app/Contents/PkgInfo and + *.app/Contents/Resources/\*.lproj seem to be the only paths affected. See + https://github.com/NixOS/nix/issues/1443 for more discussion. */ + + if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) + { + debug(format("'%1%' is not allowed to be linked in macOS") % path); + return; + } +#endif if (S_ISDIR(st.st_mode)) { Strings names = readDirectoryIgnoringInodes(path, inodeHash); for (auto & i : names) - optimisePath_(stats, path + "/" + i, inodeHash); + optimisePath_(act, stats, path + "/" + i, inodeHash); return; } @@ -117,13 +130,16 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa NixOS (example: $fontconfig/var/cache being modified). Skip those files. FIXME: check the modification time. */ if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { - printError(format("skipping suspicious writable file ‘%1%’") % path); + logWarning({ + .name = "Suspicious file", + .hint = hintfmt("skipping suspicious writable file '%1%'", path) + }); return; } /* This can still happen on top-level files. */ if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) { - debug(format("‘%1%’ is already linked, with %2% other file(s)") % path % (st.st_nlink - 2)); + debug(format("'%1%' is already linked, with %2% other file(s)") % path % (st.st_nlink - 2)); return; } @@ -137,10 +153,10 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ Hash hash = hashPath(htSHA256, path).first; - debug(format("‘%1%’ has hash ‘%2%’") % path % printHash(hash)); + debug(format("'%1%' has hash '%2%'") % path % hash.to_string(Base32, true)); /* Check if this is a known hash. */ - Path linkPath = linksDir + "/" + printHash32(hash); + Path linkPath = linksDir + "/" + hash.to_string(Base32, false); retry: if (!pathExists(linkPath)) { @@ -161,11 +177,11 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa full. When that happens, it's fine to ignore it: we just effectively disable deduplication of this file. */ - printInfo("cannot link ‘%s’ to ‘%s’: %s", linkPath, path, strerror(errno)); + printInfo("cannot link '%s' to '%s': %s", linkPath, path, strerror(errno)); return; default: - throw SysError("cannot link ‘%1%’ to ‘%2%’", linkPath, path); + throw SysError("cannot link '%1%' to '%2%'", linkPath, path); } } @@ -173,20 +189,23 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa current file with a hard link to that file. */ struct stat stLink; if (lstat(linkPath.c_str(), &stLink)) - throw SysError(format("getting attributes of path ‘%1%’") % linkPath); + throw SysError("getting attributes of path '%1%'", linkPath); if (st.st_ino == stLink.st_ino) { - debug(format("‘%1%’ is already linked to ‘%2%’") % path % linkPath); + debug(format("'%1%' is already linked to '%2%'") % path % linkPath); return; } if (st.st_size != stLink.st_size) { - printError(format("removing corrupted link ‘%1%’") % linkPath); + logWarning({ + .name = "Corrupted link", + .hint = hintfmt("removing corrupted link '%1%'", linkPath) + }); unlink(linkPath.c_str()); goto retry; } - printMsg(lvlTalkative, format("linking ‘%1%’ to ‘%2%’") % path % linkPath); + printMsg(lvlTalkative, format("linking '%1%' to '%2%'") % path % linkPath); /* Make the containing directory writable, but only if it's not the store itself (we don't want or need to mess with its @@ -199,7 +218,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); Path tempLink = (format("%1%/.tmp-link-%2%-%3%") - % realStoreDir % getpid() % rand()).str(); + % realStoreDir % getpid() % random()).str(); if (link(linkPath.c_str(), tempLink.c_str()) == -1) { if (errno == EMLINK) { @@ -207,44 +226,59 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa systems). This is likely to happen with empty files. Just shrug and ignore. */ if (st.st_size) - printInfo(format("‘%1%’ has maximum number of links") % linkPath); + printInfo(format("'%1%' has maximum number of links") % linkPath); return; } - throw SysError("cannot link ‘%1%’ to ‘%2%’", tempLink, linkPath); + throw SysError("cannot link '%1%' to '%2%'", tempLink, linkPath); } /* Atomically replace the old file with the new hard link. */ if (rename(tempLink.c_str(), path.c_str()) == -1) { if (unlink(tempLink.c_str()) == -1) - printError(format("unable to unlink ‘%1%’") % tempLink); + logError({ + .name = "Unlink error", + .hint = hintfmt("unable to unlink '%1%'", tempLink) + }); if (errno == EMLINK) { /* Some filesystems generate too many links on the rename, rather than on the original link. (Probably it temporarily increases the st_nlink field before decreasing it again.) */ - if (st.st_size) - printInfo(format("‘%1%’ has maximum number of links") % linkPath); + debug("'%s' has reached maximum number of links", linkPath); return; } - throw SysError(format("cannot rename ‘%1%’ to ‘%2%’") % tempLink % path); + throw SysError("cannot rename '%1%' to '%2%'", tempLink, path); } stats.filesLinked++; stats.bytesFreed += st.st_size; stats.blocksFreed += st.st_blocks; + + if (act) + act->result(resFileLinked, st.st_size, st.st_blocks); } void LocalStore::optimiseStore(OptimiseStats & stats) { - PathSet paths = queryAllValidPaths(); + Activity act(*logger, actOptimiseStore); + + auto paths = queryAllValidPaths(); InodeHash inodeHash = loadInodeHash(); + act.progress(0, paths.size()); + + uint64_t done = 0; + for (auto & i : paths) { addTempRoot(i); if (!isValidPath(i)) continue; /* path was GC'ed, probably */ - Activity act(*logger, lvlChatty, format("hashing files in ‘%1%’") % i); - optimisePath_(stats, realStoreDir + "/" + baseNameOf(i), inodeHash); + { + Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", printStorePath(i))); + optimisePath_(&act, stats, realStoreDir + "/" + std::string(i.to_string()), inodeHash); + } + done++; + act.progress(done, paths.size()); } } @@ -259,7 +293,7 @@ void LocalStore::optimiseStore() optimiseStore(stats); - printError( + printInfo( format("%1% freed by hard-linking %2% files") % showBytes(stats.bytesFreed) % stats.filesLinked); @@ -270,7 +304,7 @@ void LocalStore::optimisePath(const Path & path) OptimiseStats stats; InodeHash inodeHash; - if (settings.autoOptimiseStore) optimisePath_(stats, path, inodeHash); + if (settings.autoOptimiseStore) optimisePath_(nullptr, stats, path, inodeHash); } diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc new file mode 100644 index 00000000000..45c033c667d --- /dev/null +++ b/src/libstore/parsed-derivations.cc @@ -0,0 +1,120 @@ +#include "parsed-derivations.hh" + +#include + +namespace nix { + +ParsedDerivation::ParsedDerivation(StorePath && drvPath, BasicDerivation & drv) + : drvPath(std::move(drvPath)), drv(drv) +{ + /* Parse the __json attribute, if any. */ + auto jsonAttr = drv.env.find("__json"); + if (jsonAttr != drv.env.end()) { + try { + structuredAttrs = std::make_unique(nlohmann::json::parse(jsonAttr->second)); + } catch (std::exception & e) { + throw Error("cannot process __json attribute of '%s': %s", drvPath.to_string(), e.what()); + } + } +} + +ParsedDerivation::~ParsedDerivation() { } + +std::optional ParsedDerivation::getStringAttr(const std::string & name) const +{ + if (structuredAttrs) { + auto i = structuredAttrs->find(name); + if (i == structuredAttrs->end()) + return {}; + else { + if (!i->is_string()) + throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath.to_string()); + return i->get(); + } + } else { + auto i = drv.env.find(name); + if (i == drv.env.end()) + return {}; + else + return i->second; + } +} + +bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const +{ + if (structuredAttrs) { + auto i = structuredAttrs->find(name); + if (i == structuredAttrs->end()) + return def; + else { + if (!i->is_boolean()) + throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath.to_string()); + return i->get(); + } + } else { + auto i = drv.env.find(name); + if (i == drv.env.end()) + return def; + else + return i->second == "1"; + } +} + +std::optional ParsedDerivation::getStringsAttr(const std::string & name) const +{ + if (structuredAttrs) { + auto i = structuredAttrs->find(name); + if (i == structuredAttrs->end()) + return {}; + else { + if (!i->is_array()) + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath.to_string()); + Strings res; + for (auto j = i->begin(); j != i->end(); ++j) { + if (!j->is_string()) + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath.to_string()); + res.push_back(j->get()); + } + return res; + } + } else { + auto i = drv.env.find(name); + if (i == drv.env.end()) + return {}; + else + return tokenizeString(i->second); + } +} + +StringSet ParsedDerivation::getRequiredSystemFeatures() const +{ + StringSet res; + for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings())) + res.insert(i); + return res; +} + +bool ParsedDerivation::canBuildLocally() const +{ + if (drv.platform != settings.thisSystem.get() + && !settings.extraPlatforms.get().count(drv.platform) + && !drv.isBuiltin()) + return false; + + for (auto & feature : getRequiredSystemFeatures()) + if (!settings.systemFeatures.get().count(feature)) return false; + + return true; +} + +bool ParsedDerivation::willBuildLocally() const +{ + return getBoolAttr("preferLocalBuild") && canBuildLocally(); +} + +bool ParsedDerivation::substitutesAllowed() const +{ + return getBoolAttr("allowSubstitutes", true); +} + +} diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh new file mode 100644 index 00000000000..f4df5dd5434 --- /dev/null +++ b/src/libstore/parsed-derivations.hh @@ -0,0 +1,39 @@ +#include "derivations.hh" + +#include + +namespace nix { + +class ParsedDerivation +{ + StorePath drvPath; + BasicDerivation & drv; + std::unique_ptr structuredAttrs; + +public: + + ParsedDerivation(StorePath && drvPath, BasicDerivation & drv); + + ~ParsedDerivation(); + + const nlohmann::json * getStructuredAttrs() const + { + return structuredAttrs.get(); + } + + std::optional getStringAttr(const std::string & name) const; + + bool getBoolAttr(const std::string & name, bool def = false) const; + + std::optional getStringsAttr(const std::string & name) const; + + StringSet getRequiredSystemFeatures() const; + + bool canBuildLocally() const; + + bool willBuildLocally() const; + + bool substitutesAllowed() const; +}; + +} diff --git a/src/libstore/path.cc b/src/libstore/path.cc new file mode 100644 index 00000000000..9a28aa96ad4 --- /dev/null +++ b/src/libstore/path.cc @@ -0,0 +1,131 @@ +#include "store-api.hh" + +namespace nix { + +extern "C" { + rust::Result ffi_StorePath_new(rust::StringSlice path, rust::StringSlice storeDir); + rust::Result ffi_StorePath_new2(unsigned char hash[20], rust::StringSlice storeDir); + rust::Result ffi_StorePath_fromBaseName(rust::StringSlice baseName); + rust::String ffi_StorePath_to_string(const StorePath & _this); + StorePath ffi_StorePath_clone(const StorePath & _this); + rust::StringSlice ffi_StorePath_name(const StorePath & _this); +} + +StorePath StorePath::make(std::string_view path, std::string_view storeDir) +{ + return ffi_StorePath_new((rust::StringSlice) path, (rust::StringSlice) storeDir).unwrap(); +} + +StorePath StorePath::make(unsigned char hash[20], std::string_view name) +{ + return ffi_StorePath_new2(hash, (rust::StringSlice) name).unwrap(); +} + +StorePath StorePath::fromBaseName(std::string_view baseName) +{ + return ffi_StorePath_fromBaseName((rust::StringSlice) baseName).unwrap(); +} + +rust::String StorePath::to_string() const +{ + return ffi_StorePath_to_string(*this); +} + +StorePath StorePath::clone() const +{ + return ffi_StorePath_clone(*this); +} + +bool StorePath::isDerivation() const +{ + return hasSuffix(name(), drvExtension); +} + +std::string_view StorePath::name() const +{ + return ffi_StorePath_name(*this); +} + +StorePath StorePath::dummy( + StorePath::make( + (unsigned char *) "xxxxxxxxxxxxxxxxxxxx", "x")); + +StorePath Store::parseStorePath(std::string_view path) const +{ + return StorePath::make(path, storeDir); +} + +std::optional Store::maybeParseStorePath(std::string_view path) const +{ + try { + return parseStorePath(path); + } catch (Error &) { + return {}; + } +} + +bool Store::isStorePath(std::string_view path) const +{ + return (bool) maybeParseStorePath(path); +} + +StorePathSet Store::parseStorePathSet(const PathSet & paths) const +{ + StorePathSet res; + for (auto & i : paths) res.insert(parseStorePath(i)); + return res; +} + +std::string Store::printStorePath(const StorePath & path) const +{ + auto s = storeDir + "/"; + s += (std::string_view) path.to_string(); + return s; +} + +PathSet Store::printStorePathSet(const StorePathSet & paths) const +{ + PathSet res; + for (auto & i : paths) res.insert(printStorePath(i)); + return res; +} + +StorePathSet cloneStorePathSet(const StorePathSet & paths) +{ + StorePathSet res; + for (auto & p : paths) + res.insert(p.clone()); + return res; +} + +StorePathSet storePathsToSet(const StorePaths & paths) +{ + StorePathSet res; + for (auto & p : paths) + res.insert(p.clone()); + return res; +} + +StorePathSet singleton(const StorePath & path) +{ + StorePathSet res; + res.insert(path.clone()); + return res; +} + +std::pair parsePathWithOutputs(std::string_view s) +{ + size_t n = s.find("!"); + return n == s.npos + ? std::make_pair(s, std::set()) + : std::make_pair(((std::string_view) s).substr(0, n), + tokenizeString>(((std::string_view) s).substr(n + 1), ",")); +} + +StorePathWithOutputs Store::parsePathWithOutputs(const std::string & s) +{ + auto [path, outputs] = nix::parsePathWithOutputs(s); + return {parseStorePath(path), std::move(outputs)}; +} + +} diff --git a/src/libstore/path.hh b/src/libstore/path.hh new file mode 100644 index 00000000000..5122e742290 --- /dev/null +++ b/src/libstore/path.hh @@ -0,0 +1,114 @@ +#pragma once + +#include "rust-ffi.hh" + +namespace nix { + +/* See path.rs. */ +struct StorePath; + +class Store; + +extern "C" { + void ffi_StorePath_drop(void *); + bool ffi_StorePath_less_than(const StorePath & a, const StorePath & b); + bool ffi_StorePath_eq(const StorePath & a, const StorePath & b); + unsigned char * ffi_StorePath_hash_data(const StorePath & p); +} + +struct StorePath : rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop> +{ + StorePath() = delete; + + static StorePath make(std::string_view path, std::string_view storeDir); + + static StorePath make(unsigned char hash[20], std::string_view name); + + static StorePath fromBaseName(std::string_view baseName); + + rust::String to_string() const; + + bool operator < (const StorePath & other) const + { + return ffi_StorePath_less_than(*this, other); + } + + bool operator == (const StorePath & other) const + { + return ffi_StorePath_eq(*this, other); + } + + bool operator != (const StorePath & other) const + { + return !(*this == other); + } + + StorePath clone() const; + + /* Check whether a file name ends with the extension for + derivations. */ + bool isDerivation() const; + + std::string_view name() const; + + unsigned char * hashData() const + { + return ffi_StorePath_hash_data(*this); + } + + static StorePath dummy; +}; + +typedef std::set StorePathSet; +typedef std::vector StorePaths; + +StorePathSet cloneStorePathSet(const StorePathSet & paths); +StorePathSet storePathsToSet(const StorePaths & paths); + +StorePathSet singleton(const StorePath & path); + +/* Size of the hash part of store paths, in base-32 characters. */ +const size_t storePathHashLen = 32; // i.e. 160 bits + +/* Extension of derivations in the Nix store. */ +const std::string drvExtension = ".drv"; + +enum struct FileIngestionMethod : uint8_t { + Flat = false, + Recursive = true +}; + +struct StorePathWithOutputs +{ + StorePath path; + std::set outputs; + + StorePathWithOutputs(const StorePath & path, const std::set & outputs = {}) + : path(path.clone()), outputs(outputs) + { } + + StorePathWithOutputs(StorePath && path, std::set && outputs) + : path(std::move(path)), outputs(std::move(outputs)) + { } + + StorePathWithOutputs(const StorePathWithOutputs & other) + : path(other.path.clone()), outputs(other.outputs) + { } + + std::string to_string(const Store & store) const; +}; + +std::pair parsePathWithOutputs(std::string_view s); + +} + +namespace std { + +template<> struct hash { + std::size_t operator()(const nix::StorePath & path) const noexcept + { + return * (std::size_t *) path.hashData(); + } +}; + +} diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 620c9a6b752..926f4ea1e44 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -5,23 +5,24 @@ #include #include +#include #include #include -#include +#include namespace nix { -int openLockFile(const Path & path, bool create) +AutoCloseFD openLockFile(const Path & path, bool create) { AutoCloseFD fd; fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600); if (!fd && (create || errno != ENOENT)) - throw SysError(format("opening lock file ‘%1%’") % path); + throw SysError("opening lock file '%1%'", path); - return fd.release(); + return fd; } @@ -40,29 +41,26 @@ void deleteLockFile(const Path & path, int fd) bool lockFile(int fd, LockType lockType, bool wait) { - struct flock lock; - if (lockType == ltRead) lock.l_type = F_RDLCK; - else if (lockType == ltWrite) lock.l_type = F_WRLCK; - else if (lockType == ltNone) lock.l_type = F_UNLCK; + int type; + if (lockType == ltRead) type = LOCK_SH; + else if (lockType == ltWrite) type = LOCK_EX; + else if (lockType == ltNone) type = LOCK_UN; else abort(); - lock.l_whence = SEEK_SET; - lock.l_start = 0; - lock.l_len = 0; /* entire file */ if (wait) { - while (fcntl(fd, F_SETLKW, &lock) != 0) { + while (flock(fd, type) != 0) { checkInterrupt(); if (errno != EINTR) - throw SysError(format("acquiring/releasing lock")); + throw SysError("acquiring/releasing lock"); else return false; } } else { - while (fcntl(fd, F_SETLK, &lock) != 0) { + while (flock(fd, type | LOCK_NB) != 0) { checkInterrupt(); - if (errno == EACCES || errno == EAGAIN) return false; + if (errno == EWOULDBLOCK) return false; if (errno != EINTR) - throw SysError(format("acquiring/releasing lock")); + throw SysError("acquiring/releasing lock"); } } @@ -70,14 +68,6 @@ bool lockFile(int fd, LockType lockType, bool wait) } -/* This enables us to check whether are not already holding a lock on - a file ourselves. POSIX locks (fcntl) suck in this respect: if we - close a descriptor, the previous lock will be closed as well. And - there is no way to query whether we already have a lock (F_GETLK - only works on locks held by other processes). */ -static Sync lockedPaths_; - - PathLocks::PathLocks() : deletePaths(false) { @@ -91,7 +81,7 @@ PathLocks::PathLocks(const PathSet & paths, const string & waitMsg) } -bool PathLocks::lockPaths(const PathSet & _paths, +bool PathLocks::lockPaths(const PathSet & paths, const string & waitMsg, bool wait) { assert(fds.empty()); @@ -99,72 +89,54 @@ bool PathLocks::lockPaths(const PathSet & _paths, /* Note that `fds' is built incrementally so that the destructor will only release those locks that we have already acquired. */ - /* Sort the paths. This assures that locks are always acquired in - the same order, thus preventing deadlocks. */ - Paths paths(_paths.begin(), _paths.end()); - paths.sort(); - - /* Acquire the lock for each path. */ + /* Acquire the lock for each path in sorted order. This ensures + that locks are always acquired in the same order, thus + preventing deadlocks. */ for (auto & path : paths) { checkInterrupt(); Path lockPath = path + ".lock"; - debug(format("locking path ‘%1%’") % path); - - { - auto lockedPaths(lockedPaths_.lock()); - if (lockedPaths->count(lockPath)) - throw Error("deadlock: trying to re-acquire self-held lock ‘%s’", lockPath); - lockedPaths->insert(lockPath); - } - - try { + debug(format("locking path '%1%'") % path); - AutoCloseFD fd; + AutoCloseFD fd; - while (1) { + while (1) { - /* Open/create the lock file. */ - fd = openLockFile(lockPath, true); + /* Open/create the lock file. */ + fd = openLockFile(lockPath, true); - /* Acquire an exclusive lock. */ - if (!lockFile(fd.get(), ltWrite, false)) { - if (wait) { - if (waitMsg != "") printError(waitMsg); - lockFile(fd.get(), ltWrite, true); - } else { - /* Failed to lock this path; release all other - locks. */ - unlock(); - return false; - } + /* Acquire an exclusive lock. */ + if (!lockFile(fd.get(), ltWrite, false)) { + if (wait) { + if (waitMsg != "") printError(waitMsg); + lockFile(fd.get(), ltWrite, true); + } else { + /* Failed to lock this path; release all other + locks. */ + unlock(); + return false; } - - debug(format("lock acquired on ‘%1%’") % lockPath); - - /* Check that the lock file hasn't become stale (i.e., - hasn't been unlinked). */ - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError(format("statting lock file ‘%1%’") % lockPath); - if (st.st_size != 0) - /* This lock file has been unlinked, so we're holding - a lock on a deleted file. This means that other - processes may create and acquire a lock on - `lockPath', and proceed. So we must retry. */ - debug(format("open lock file ‘%1%’ has become stale") % lockPath); - else - break; } - /* Use borrow so that the descriptor isn't closed. */ - fds.push_back(FDPair(fd.release(), lockPath)); - - } catch (...) { - lockedPaths_.lock()->erase(lockPath); - throw; + debug(format("lock acquired on '%1%'") % lockPath); + + /* Check that the lock file hasn't become stale (i.e., + hasn't been unlinked). */ + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("statting lock file '%1%'", lockPath); + if (st.st_size != 0) + /* This lock file has been unlinked, so we're holding + a lock on a deleted file. This means that other + processes may create and acquire a lock on + `lockPath', and proceed. So we must retry. */ + debug(format("open lock file '%1%' has become stale") % lockPath); + else + break; } + /* Use borrow so that the descriptor isn't closed. */ + fds.push_back(FDPair(fd.release(), lockPath)); } return true; @@ -186,13 +158,12 @@ void PathLocks::unlock() for (auto & i : fds) { if (deletePaths) deleteLockFile(i.second, i.first); - lockedPaths_.lock()->erase(i.second); - if (close(i.first) == -1) printError( - format("error (ignored): cannot close lock file on ‘%1%’") % i.second); + "error (ignored): cannot close lock file on '%1%'", + i.second); - debug(format("lock released on ‘%1%’") % i.second); + debug(format("lock released on '%1%'") % i.second); } fds.clear(); @@ -205,11 +176,4 @@ void PathLocks::setDeletion(bool deletePaths) } -bool pathIsLockedByMe(const Path & path) -{ - Path lockPath = path + ".lock"; - return lockedPaths_.lock()->count(lockPath); -} - - } diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 40103c393f6..411da022295 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -1,15 +1,13 @@ #pragma once -#include "types.hh" - +#include "util.hh" namespace nix { - /* Open (possibly create) a lock file and return the file descriptor. -1 is returned if create is false and the lock could not be opened because it doesn't exist. Any other error throws an exception. */ -int openLockFile(const Path & path, bool create); +AutoCloseFD openLockFile(const Path & path, bool create); /* Delete an open lock file. */ void deleteLockFile(const Path & path, int fd); @@ -18,8 +16,7 @@ enum LockType { ltRead, ltWrite, ltNone }; bool lockFile(int fd, LockType lockType, bool wait); - -class PathLocks +class PathLocks { private: typedef std::pair FDPair; @@ -38,9 +35,4 @@ public: void setDeletion(bool deletePaths); }; - -// FIXME: not thread-safe! -bool pathIsLockedByMe(const Path & path); - - } diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index f24daa8862a..6cfe393a45e 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -40,7 +40,7 @@ Generations findGenerations(Path profile, int & curGen) Generations gens; Path profileDir = dirOf(profile); - string profileName = baseNameOf(profile); + auto profileName = std::string(baseNameOf(profile)); for (auto & i : readDirectory(profileDir)) { int n; @@ -50,7 +50,7 @@ Generations findGenerations(Path profile, int & curGen) gen.number = n; struct stat st; if (lstat(gen.path.c_str(), &st) != 0) - throw SysError(format("statting ‘%1%’") % gen.path); + throw SysError("statting '%1%'", gen.path); gen.creationTime = st.st_mtime; gens.push_back(gen); } @@ -108,7 +108,7 @@ Path createGeneration(ref store, Path profile, Path outPath) user environment etc. we've just built. */ Path generation; makeName(profile, num + 1, generation); - store->addPermRoot(outPath, generation, false, true); + store->addPermRoot(store->parseStorePath(outPath), generation, false, true); return generation; } @@ -117,7 +117,7 @@ Path createGeneration(ref store, Path profile, Path outPath) static void removeFile(const Path & path) { if (remove(path.c_str()) == -1) - throw SysError(format("cannot unlink ‘%1%’") % path); + throw SysError("cannot unlink '%1%'", path); } @@ -149,7 +149,7 @@ void deleteGenerations(const Path & profile, const std::set & gens Generations gens = findGenerations(profile, curGen); if (gensToDelete.find(curGen) != gensToDelete.end()) - throw Error(format("cannot delete current generation of profile %1%’") % profile); + throw Error("cannot delete current generation of profile %1%'", profile); for (auto & i : gens) { if (gensToDelete.find(i.number) == gensToDelete.end()) continue; @@ -157,6 +157,29 @@ void deleteGenerations(const Path & profile, const std::set & gens } } +void deleteGenerationsGreaterThan(const Path & profile, int max, bool dryRun) +{ + PathLocks lock; + lockProfile(lock, profile); + + int curGen; + bool fromCurGen = false; + Generations gens = findGenerations(profile, curGen); + for (auto i = gens.rbegin(); i != gens.rend(); ++i) { + if (i->number == curGen) { + fromCurGen = true; + max--; + continue; + } + if (fromCurGen) { + if (max) { + max--; + continue; + } + deleteGeneration2(profile, i->number, dryRun); + } + } +} void deleteOldGenerations(const Path & profile, bool dryRun) { @@ -203,7 +226,7 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b int days; if (!string2Int(strDays, days) || days < 1) - throw Error(format("invalid number of days specifier ‘%1%’") % timeSpec); + throw Error("invalid number of days specifier '%1%'", timeSpec); time_t oldTime = curTime - days * 24 * 3600; @@ -222,7 +245,7 @@ void switchLink(Path link, Path target) void lockProfile(PathLocks & lock, const Path & profile) { - lock.lockPaths({profile}, (format("waiting for lock on profile ‘%1%’") % profile).str()); + lock.lockPaths({profile}, (format("waiting for lock on profile '%1%'") % profile).str()); lock.setDeletion(true); } @@ -233,4 +256,22 @@ string optimisticLockProfile(const Path & profile) } +Path getDefaultProfile() +{ + Path profileLink = getHome() + "/.nix-profile"; + try { + if (!pathExists(profileLink)) { + replaceSymlink( + getuid() == 0 + ? settings.nixStateDir + "/profiles/default" + : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), + profileLink); + } + return absPath(readLink(profileLink), dirOf(profileLink)); + } catch (Error &) { + return profileLink; + } +} + + } diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 1d4e6d3037d..78645d8b6a4 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -39,6 +39,8 @@ void deleteGeneration(const Path & profile, unsigned int gen); void deleteGenerations(const Path & profile, const std::set & gensToDelete, bool dryRun); +void deleteGenerationsGreaterThan(const Path & profile, const int max, bool dryRun); + void deleteOldGenerations(const Path & profile, bool dryRun); void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun); @@ -62,4 +64,8 @@ void lockProfile(PathLocks & lock, const Path & profile); rebuilt. */ string optimisticLockProfile(const Path & profile); +/* Resolve ~/.nix-profile. If ~/.nix-profile doesn't exist yet, create + it. */ +Path getDefaultProfile(); + } diff --git a/src/libstore/references.cc b/src/libstore/references.cc index 33eab5a240b..a10d536a355 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -13,7 +13,7 @@ namespace nix { static unsigned int refLength = 32; /* characters */ -static void search(const unsigned char * s, unsigned int len, +static void search(const unsigned char * s, size_t len, StringSet & hashes, StringSet & seen) { static bool initialised = false; @@ -25,7 +25,7 @@ static void search(const unsigned char * s, unsigned int len, initialised = true; } - for (unsigned int i = 0; i + refLength <= len; ) { + for (size_t i = 0; i + refLength <= len; ) { int j; bool match = true; for (j = refLength - 1; j >= 0; --j) @@ -36,11 +36,10 @@ static void search(const unsigned char * s, unsigned int len, } if (!match) continue; string ref((const char *) s + i, refLength); - if (hashes.find(ref) != hashes.end()) { - debug(format("found reference to ‘%1%’ at offset ‘%2%’") + if (hashes.erase(ref)) { + debug(format("found reference to '%1%' at offset '%2%'") % ref % i); seen.insert(ref); - hashes.erase(ref); } ++i; } @@ -73,7 +72,7 @@ void RefScanSink::operator () (const unsigned char * data, size_t len) search(data, len, hashes, seen); - unsigned int tailLen = len <= refLength ? len : refLength; + size_t tailLen = len <= refLength ? len : refLength; tail = string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)) + string((const char *) data + len - tailLen, tailLen); @@ -90,10 +89,10 @@ PathSet scanForReferences(const string & path, hash part of the file name. (This assumes that all references have the form `HASH-bla'). */ for (auto & i : refs) { - string baseName = baseNameOf(i); + auto baseName = std::string(baseNameOf(i)); string::size_type pos = baseName.find('-'); if (pos == string::npos) - throw Error(format("bad reference ‘%1%’") % i); + throw Error("bad reference '%1%'", i); string s = string(baseName, 0, pos); assert(s.size() == refLength); assert(backMap.find(s) == backMap.end()); @@ -119,4 +118,66 @@ PathSet scanForReferences(const string & path, } +RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink) + : from(from), to(to), nextSink(nextSink) +{ + assert(from.size() == to.size()); +} + +void RewritingSink::operator () (const unsigned char * data, size_t len) +{ + std::string s(prev); + s.append((const char *) data, len); + + size_t j = 0; + while ((j = s.find(from, j)) != string::npos) { + matches.push_back(pos + j); + s.replace(j, from.size(), to); + } + + prev = s.size() < from.size() ? s : std::string(s, s.size() - from.size() + 1, from.size() - 1); + + auto consumed = s.size() - prev.size(); + + pos += consumed; + + if (consumed) nextSink((unsigned char *) s.data(), consumed); +} + +void RewritingSink::flush() +{ + if (prev.empty()) return; + pos += prev.size(); + nextSink((unsigned char *) prev.data(), prev.size()); + prev.clear(); +} + +HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus) + : hashSink(ht) + , rewritingSink(modulus, std::string(modulus.size(), 0), hashSink) +{ +} + +void HashModuloSink::operator () (const unsigned char * data, size_t len) +{ + rewritingSink(data, len); +} + +HashResult HashModuloSink::finish() +{ + rewritingSink.flush(); + + /* Hash the positions of the self-references. This ensures that a + NAR with self-references and a NAR with some of the + self-references already zeroed out do not produce a hash + collision. FIXME: proof. */ + for (auto & pos : rewritingSink.matches) { + auto s = fmt("|%d", pos); + hashSink((unsigned char *) s.data(), s.size()); + } + + auto h = hashSink.finish(); + return {h.first, rewritingSink.pos}; +} + } diff --git a/src/libstore/references.hh b/src/libstore/references.hh index 013809d122f..c38bdd720d3 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -7,5 +7,32 @@ namespace nix { PathSet scanForReferences(const Path & path, const PathSet & refs, HashResult & hash); - + +struct RewritingSink : Sink +{ + std::string from, to, prev; + Sink & nextSink; + uint64_t pos = 0; + + std::vector matches; + + RewritingSink(const std::string & from, const std::string & to, Sink & nextSink); + + void operator () (const unsigned char * data, size_t len) override; + + void flush(); +}; + +struct HashModuloSink : AbstractHashSink +{ + HashSink hashSink; + RewritingSink rewritingSink; + + HashModuloSink(HashType ht, const std::string & modulus); + + void operator () (const unsigned char * data, size_t len) override; + + HashResult finish() override; +}; + } diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index ca14057c2e2..9277a8e6bc9 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -1,12 +1,46 @@ #include "remote-fs-accessor.hh" #include "nar-accessor.hh" +#include "json.hh" -namespace nix { +#include +#include +#include +namespace nix { -RemoteFSAccessor::RemoteFSAccessor(ref store) +RemoteFSAccessor::RemoteFSAccessor(ref store, const Path & cacheDir) : store(store) + , cacheDir(cacheDir) +{ + if (cacheDir != "") + createDirs(cacheDir); +} + +Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext) +{ + assert(cacheDir != ""); + return fmt("%s/%s.%s", cacheDir, storePathToHash(storePath), ext); +} + +void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar, + ref narAccessor) { + nars.emplace(storePath, narAccessor); + + if (cacheDir != "") { + try { + std::ostringstream str; + JSONPlaceholder jsonRoot(str); + listNar(jsonRoot, narAccessor, "", true); + writeFile(makeCacheFile(storePath, "ls"), str.str()); + + /* FIXME: do this asynchronously. */ + writeFile(makeCacheFile(storePath, "nar"), nar); + + } catch (...) { + ignoreException(); + } + } } std::pair, Path> RemoteFSAccessor::fetch(const Path & path_) @@ -16,18 +50,56 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_) auto storePath = store->toStorePath(path); std::string restPath = std::string(path, storePath.size()); - if (!store->isValidPath(storePath)) - throw InvalidPath(format("path ‘%1%’ is not a valid store path") % storePath); + if (!store->isValidPath(store->parseStorePath(storePath))) + throw InvalidPath("path '%1%' is not a valid store path", storePath); auto i = nars.find(storePath); if (i != nars.end()) return {i->second, restPath}; StringSink sink; - store->narFromPath(storePath, sink); + std::string listing; + Path cacheFile; + + if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) { + + try { + listing = nix::readFile(makeCacheFile(storePath, "ls")); + + auto narAccessor = makeLazyNarAccessor(listing, + [cacheFile](uint64_t offset, uint64_t length) { + + AutoCloseFD fd = open(cacheFile.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening NAR cache file '%s'", cacheFile); + + if (lseek(fd.get(), offset, SEEK_SET) != (off_t) offset) + throw SysError("seeking in '%s'", cacheFile); + + std::string buf(length, 0); + readFull(fd.get(), (unsigned char *) buf.data(), length); + + return buf; + }); + + nars.emplace(storePath, narAccessor); + return {narAccessor, restPath}; + + } catch (SysError &) { } + + try { + *sink.s = nix::readFile(cacheFile); + + auto narAccessor = makeNarAccessor(sink.s); + nars.emplace(storePath, narAccessor); + return {narAccessor, restPath}; + + } catch (SysError &) { } + } - auto accessor = makeNarAccessor(sink.s); - nars.emplace(storePath, accessor); - return {accessor, restPath}; + store->narFromPath(store->parseStorePath(storePath), sink); + auto narAccessor = makeNarAccessor(sink.s); + addToCache(storePath, *sink.s, narAccessor); + return {narAccessor, restPath}; } FSAccessor::Stat RemoteFSAccessor::stat(const Path & path) diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index 28f36c8296e..4afb3be9573 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -12,10 +12,21 @@ class RemoteFSAccessor : public FSAccessor std::map> nars; + Path cacheDir; + std::pair, Path> fetch(const Path & path_); + + friend class BinaryCacheStore; + + Path makeCacheFile(const Path & storePath, const std::string & ext); + + void addToCache(const Path & storePath, const std::string & nar, + ref narAccessor); + public: - RemoteFSAccessor(ref store); + RemoteFSAccessor(ref store, + const /* FIXME: use std::optional */ Path & cacheDir = ""); Stat stat(const Path & path) override; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 816d95ba607..99fee81506b 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -7,6 +7,7 @@ #include "globals.hh" #include "derivations.hh" #include "pool.hh" +#include "finally.hh" #include #include @@ -21,46 +22,78 @@ namespace nix { -Path readStorePath(Store & store, Source & from) +template<> StorePathSet readStorePaths(const Store & store, Source & from) { - Path path = readString(from); - store.assertStorePath(path); - return path; + StorePathSet paths; + for (auto & i : readStrings(from)) + paths.insert(store.parseStorePath(i)); + return paths; } -template T readStorePaths(Store & store, Source & from) +void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths) { - T paths = readStrings(from); - for (auto & i : paths) store.assertStorePath(i); - return paths; + out << paths.size(); + for (auto & i : paths) + out << store.printStorePath(i); } -template PathSet readStorePaths(Store & store, Source & from); /* TODO: Separate these store impls into different files, give them better names */ -RemoteStore::RemoteStore(const Params & params, size_t maxConnections) +RemoteStore::RemoteStore(const Params & params) : Store(params) , connections(make_ref>( - maxConnections, - [this]() { return openConnection(); }, - [](const ref & r) { return r->to.good() && r->from.good(); } + std::max(1, (int) maxConnections), + [this]() { return openConnectionWrapper(); }, + [this](const ref & r) { + return + r->to.good() + && r->from.good() + && std::chrono::duration_cast( + std::chrono::steady_clock::now() - r->startTime).count() < maxConnectionAge; + } )) { } -UDSRemoteStore::UDSRemoteStore(const Params & params, size_t maxConnections) +ref RemoteStore::openConnectionWrapper() +{ + if (failed) + throw Error("opening a connection to remote store '%s' previously failed", getUri()); + try { + return openConnection(); + } catch (...) { + failed = true; + throw; + } +} + + +UDSRemoteStore::UDSRemoteStore(const Params & params) : Store(params) , LocalFSStore(params) - , RemoteStore(params, maxConnections) + , RemoteStore(params) +{ +} + + +UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params & params) + : Store(params) + , LocalFSStore(params) + , RemoteStore(params) + , path(socket_path) { } std::string UDSRemoteStore::getUri() { - return "daemon"; + if (path) { + return std::string("unix://") + *path; + } else { + return "daemon"; + } } @@ -78,20 +111,22 @@ ref UDSRemoteStore::openConnection() throw SysError("cannot create Unix domain socket"); closeOnExec(conn->fd.get()); - string socketPath = settings.nixDaemonSocketFile; + string socketPath = path ? *path : settings.nixDaemonSocketFile; struct sockaddr_un addr; addr.sun_family = AF_UNIX; if (socketPath.size() + 1 >= sizeof(addr.sun_path)) - throw Error(format("socket path ‘%1%’ is too long") % socketPath); + throw Error("socket path '%1%' is too long", socketPath); strcpy(addr.sun_path, socketPath.c_str()); - if (connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError(format("cannot connect to daemon at ‘%1%’") % socketPath); + if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot connect to daemon at '%1%'", socketPath); conn->from.fd = conn->fd.get(); conn->to.fd = conn->fd.get(); + conn->startTime = std::chrono::steady_clock::now(); + initConnection(*conn); return conn; @@ -107,7 +142,7 @@ void RemoteStore::initConnection(Connection & conn) unsigned int magic = readInt(conn.from); if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch"); - conn.daemonVersion = readInt(conn.from); + conn.from >> conn.daemonVersion; if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) throw Error("Nix daemon protocol version not supported"); if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10) @@ -115,7 +150,7 @@ void RemoteStore::initConnection(Connection & conn) conn.to << PROTOCOL_VERSION; if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) { - int cpu = settings.lockCPU ? lockToCurrentCPU() : -1; + int cpu = sameMachine() && settings.lockCPU ? lockToCurrentCPU() : -1; if (cpu != -1) conn.to << 1 << cpu; else @@ -125,10 +160,11 @@ void RemoteStore::initConnection(Connection & conn) if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11) conn.to << false; - conn.processStderr(); + auto ex = conn.processStderr(); + if (ex) std::rethrow_exception(ex); } catch (Error & e) { - throw Error(format("cannot start daemon worker: %1%") % e.msg()); + throw Error("cannot open connection to remote store '%s': %s", getUri(), e.what()); } setOptions(conn); @@ -144,7 +180,7 @@ void RemoteStore::setOptions(Connection & conn) << verbosity << settings.maxBuildJobs << settings.maxSilentTime - << settings.useBuildHook + << true << (settings.verboseBuild ? lvlError : lvlVomit) << 0 // obsolete log type << 0 /* obsolete print build trace */ @@ -152,106 +188,162 @@ void RemoteStore::setOptions(Connection & conn) << settings.useSubstitutes; if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { - Settings::SettingsMap overrides = settings.getOverrides(); - if (overrides["ssh-auth-sock"] == "") - overrides["ssh-auth-sock"] = getEnv("SSH_AUTH_SOCK"); + std::map overrides; + globalConfig.getSettings(overrides, true); + overrides.erase(settings.keepFailed.name); + overrides.erase(settings.keepGoing.name); + overrides.erase(settings.tryFallback.name); + overrides.erase(settings.maxBuildJobs.name); + overrides.erase(settings.maxSilentTime.name); + overrides.erase(settings.buildCores.name); + overrides.erase(settings.useSubstitutes.name); + overrides.erase(settings.showTrace.name); conn.to << overrides.size(); for (auto & i : overrides) - conn.to << i.first << i.second; + conn.to << i.first << i.second.value; } - conn.processStderr(); + auto ex = conn.processStderr(); + if (ex) std::rethrow_exception(ex); } -bool RemoteStore::isValidPathUncached(const Path & path) +/* A wrapper around Pool::Handle that marks + the connection as bad (causing it to be closed) if a non-daemon + exception is thrown before the handle is closed. Such an exception + causes a deviation from the expected protocol and therefore a + desynchronization between the client and daemon. */ +struct ConnectionHandle { - auto conn(connections->get()); - conn->to << wopIsValidPath << path; - conn->processStderr(); - unsigned int reply = readInt(conn->from); - return reply != 0; + Pool::Handle handle; + bool daemonException = false; + + ConnectionHandle(Pool::Handle && handle) + : handle(std::move(handle)) + { } + + ConnectionHandle(ConnectionHandle && h) + : handle(std::move(h.handle)) + { } + + ~ConnectionHandle() + { + if (!daemonException && std::uncaught_exception()) { + handle.markBad(); + debug("closing daemon connection because of an exception"); + } + } + + RemoteStore::Connection * operator -> () { return &*handle; } + + void processStderr(Sink * sink = 0, Source * source = 0) + { + auto ex = handle->processStderr(sink, source); + if (ex) { + daemonException = true; + std::rethrow_exception(ex); + } + } +}; + + +ConnectionHandle RemoteStore::getConnection() +{ + return ConnectionHandle(connections->get()); } -PathSet RemoteStore::queryValidPaths(const PathSet & paths) +bool RemoteStore::isValidPathUncached(const StorePath & path) { - auto conn(connections->get()); + auto conn(getConnection()); + conn->to << wopIsValidPath << printStorePath(path); + conn.processStderr(); + return readInt(conn->from); +} + + +StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) +{ + auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - PathSet res; + StorePathSet res; for (auto & i : paths) - if (isValidPath(i)) res.insert(i); + if (isValidPath(i)) res.insert(i.clone()); return res; } else { - conn->to << wopQueryValidPaths << paths; - conn->processStderr(); - return readStorePaths(*this, conn->from); + conn->to << wopQueryValidPaths; + writeStorePaths(*this, conn->to, paths); + conn.processStderr(); + return readStorePaths(*this, conn->from); } } -PathSet RemoteStore::queryAllValidPaths() +StorePathSet RemoteStore::queryAllValidPaths() { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopQueryAllValidPaths; - conn->processStderr(); - return readStorePaths(*this, conn->from); + conn.processStderr(); + return readStorePaths(*this, conn->from); } -PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths) +StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) { - auto conn(connections->get()); + auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - PathSet res; + StorePathSet res; for (auto & i : paths) { - conn->to << wopHasSubstitutes << i; - conn->processStderr(); - if (readInt(conn->from)) res.insert(i); + conn->to << wopHasSubstitutes << printStorePath(i); + conn.processStderr(); + if (readInt(conn->from)) res.insert(i.clone()); } return res; } else { - conn->to << wopQuerySubstitutablePaths << paths; - conn->processStderr(); - return readStorePaths(*this, conn->from); + conn->to << wopQuerySubstitutablePaths; + writeStorePaths(*this, conn->to, paths); + conn.processStderr(); + return readStorePaths(*this, conn->from); } } -void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, +void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, SubstitutablePathInfos & infos) { if (paths.empty()) return; - auto conn(connections->get()); + auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { for (auto & i : paths) { SubstitutablePathInfo info; - conn->to << wopQuerySubstitutablePathInfo << i; - conn->processStderr(); + conn->to << wopQuerySubstitutablePathInfo << printStorePath(i); + conn.processStderr(); unsigned int reply = readInt(conn->from); if (reply == 0) continue; - info.deriver = readString(conn->from); - if (info.deriver != "") assertStorePath(info.deriver); - info.references = readStorePaths(*this, conn->from); + auto deriver = readString(conn->from); + if (deriver != "") + info.deriver = parseStorePath(deriver); + info.references = readStorePaths(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); - infos[i] = info; + infos.insert_or_assign(i.clone(), std::move(info)); } } else { - conn->to << wopQuerySubstitutablePathInfos << paths; - conn->processStderr(); - unsigned int count = readInt(conn->from); - for (unsigned int n = 0; n < count; n++) { - Path path = readStorePath(*this, conn->from); - SubstitutablePathInfo & info(infos[path]); - info.deriver = readString(conn->from); - if (info.deriver != "") assertStorePath(info.deriver); - info.references = readStorePaths(*this, conn->from); + conn->to << wopQuerySubstitutablePathInfos; + writeStorePaths(*this, conn->to, paths); + conn.processStderr(); + size_t count = readNum(conn->from); + for (size_t n = 0; n < count; n++) { + SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); + auto deriver = readString(conn->from); + if (deriver != "") + info.deriver = parseStorePath(deriver); + info.references = readStorePaths(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -260,211 +352,211 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, } -void RemoteStore::queryPathInfoUncached(const Path & path, - std::function)> success, - std::function failure) +void RemoteStore::queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept { - sync2async>(success, failure, [&]() { - auto conn(connections->get()); - conn->to << wopQueryPathInfo << path; - try { - conn->processStderr(); - } catch (Error & e) { - // Ugly backwards compatibility hack. - if (e.msg().find("is not valid") != std::string::npos) - throw InvalidPath(e.what()); - throw; - } - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { - bool valid = readInt(conn->from) != 0; - if (!valid) throw InvalidPath(format("path ‘%s’ is not valid") % path); - } - auto info = std::make_shared(); - info->path = path; - info->deriver = readString(conn->from); - if (info->deriver != "") assertStorePath(info->deriver); - info->narHash = parseHash(htSHA256, readString(conn->from)); - info->references = readStorePaths(*this, conn->from); - info->registrationTime = readInt(conn->from); - info->narSize = readLongLong(conn->from); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { - info->ultimate = readInt(conn->from) != 0; - info->sigs = readStrings(conn->from); - info->ca = readString(conn->from); + try { + std::shared_ptr info; + { + auto conn(getConnection()); + conn->to << wopQueryPathInfo << printStorePath(path); + try { + conn.processStderr(); + } catch (Error & e) { + // Ugly backwards compatibility hack. + if (e.msg().find("is not valid") != std::string::npos) + throw InvalidPath(e.info()); + throw; + } + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { + bool valid; conn->from >> valid; + if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); + } + info = std::make_shared(path.clone()); + auto deriver = readString(conn->from); + if (deriver != "") info->deriver = parseStorePath(deriver); + info->narHash = Hash(readString(conn->from), htSHA256); + info->references = readStorePaths(*this, conn->from); + conn->from >> info->registrationTime >> info->narSize; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { + conn->from >> info->ultimate; + info->sigs = readStrings(conn->from); + conn->from >> info->ca; + } } - return info; - }); -} - - -void RemoteStore::queryReferrers(const Path & path, - PathSet & referrers) -{ - auto conn(connections->get()); - conn->to << wopQueryReferrers << path; - conn->processStderr(); - PathSet referrers2 = readStorePaths(*this, conn->from); - referrers.insert(referrers2.begin(), referrers2.end()); + callback(std::move(info)); + } catch (...) { callback.rethrow(); } } -PathSet RemoteStore::queryValidDerivers(const Path & path) +void RemoteStore::queryReferrers(const StorePath & path, + StorePathSet & referrers) { - auto conn(connections->get()); - conn->to << wopQueryValidDerivers << path; - conn->processStderr(); - return readStorePaths(*this, conn->from); + auto conn(getConnection()); + conn->to << wopQueryReferrers << printStorePath(path); + conn.processStderr(); + for (auto & i : readStorePaths(*this, conn->from)) + referrers.insert(i.clone()); } -PathSet RemoteStore::queryDerivationOutputs(const Path & path) +StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) { - auto conn(connections->get()); - conn->to << wopQueryDerivationOutputs << path; - conn->processStderr(); - return readStorePaths(*this, conn->from); + auto conn(getConnection()); + conn->to << wopQueryValidDerivers << printStorePath(path); + conn.processStderr(); + return readStorePaths(*this, conn->from); } -PathSet RemoteStore::queryDerivationOutputNames(const Path & path) +StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) { - auto conn(connections->get()); - conn->to << wopQueryDerivationOutputNames << path; - conn->processStderr(); - return readStrings(conn->from); + auto conn(getConnection()); + conn->to << wopQueryDerivationOutputs << printStorePath(path); + conn.processStderr(); + return readStorePaths(*this, conn->from); } -Path RemoteStore::queryPathFromHashPart(const string & hashPart) +std::optional RemoteStore::queryPathFromHashPart(const std::string & hashPart) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopQueryPathFromHashPart << hashPart; - conn->processStderr(); + conn.processStderr(); Path path = readString(conn->from); - if (!path.empty()) assertStorePath(path); - return path; + if (path.empty()) return {}; + return parseStorePath(path); } -void RemoteStore::addToStore(const ValidPathInfo & info, const ref & nar, - bool repair, bool dontCheckSigs, std::shared_ptr accessor) +void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr accessor) { - auto conn(connections->get()); + auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) { conn->to << wopImportPaths; - StringSink sink; - sink << 1 // == path follows - ; - assert(nar->size() % 8 == 0); - sink((unsigned char *) nar->data(), nar->size()); - sink - << exportMagic - << info.path - << info.references - << info.deriver - << 0 // == no legacy signature - << 0 // == no path follows - ; - - StringSource source(*sink.s); - conn->processStderr(0, &source); - - auto importedPaths = readStorePaths(*this, conn->from); + auto source2 = sinkToSource([&](Sink & sink) { + sink << 1 // == path follows + ; + copyNAR(source, sink); + sink + << exportMagic + << printStorePath(info.path); + writeStorePaths(*this, sink, info.references); + sink + << (info.deriver ? printStorePath(*info.deriver) : "") + << 0 // == no legacy signature + << 0 // == no path follows + ; + }); + + conn.processStderr(0, source2.get()); + + auto importedPaths = readStorePaths(*this, conn->from); assert(importedPaths.size() <= 1); } else { conn->to << wopAddToStoreNar - << info.path << info.deriver << printHash(info.narHash) - << info.references << info.registrationTime << info.narSize - << info.ultimate << info.sigs << *nar << repair << dontCheckSigs; - // FIXME: don't send nar as a string - conn->processStderr(); + << printStorePath(info.path) + << (info.deriver ? printStorePath(*info.deriver) : "") + << info.narHash.to_string(Base16, false); + writeStorePaths(*this, conn->to, info.references); + conn->to << info.registrationTime << info.narSize + << info.ultimate << info.sigs << info.ca + << repair << !checkSigs; + bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; + if (!tunnel) copyNAR(source, conn->to); + conn.processStderr(0, tunnel ? &source : nullptr); } } -Path RemoteStore::addToStore(const string & name, const Path & _srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) +StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - auto conn(connections->get()); + auto conn(getConnection()); Path srcPath(absPath(_srcPath)); - conn->to << wopAddToStore << name - << ((hashAlgo == htSHA256 && recursive) ? 0 : 1) /* backwards compatibility hack */ - << (recursive ? 1 : 0) - << printHashType(hashAlgo); + conn->to + << wopAddToStore + << name + << ((hashAlgo == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ + << (method == FileIngestionMethod::Recursive ? 1 : 0) + << printHashType(hashAlgo); try { conn->to.written = 0; conn->to.warn = true; - dumpPath(srcPath, conn->to, filter); + connections->incCapacity(); + { + Finally cleanup([&]() { connections->decCapacity(); }); + dumpPath(srcPath, conn->to, filter); + } conn->to.warn = false; - conn->processStderr(); + conn.processStderr(); } catch (SysError & e) { /* Daemon closed while we were sending the path. Probably OOM or I/O error. */ if (e.errNo == EPIPE) try { - conn->processStderr(); + conn.processStderr(); } catch (EndOfFile & e) { } throw; } - return readStorePath(*this, conn->from); + return parseStorePath(readString(conn->from)); } -Path RemoteStore::addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair) +StorePath RemoteStore::addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - auto conn(connections->get()); - conn->to << wopAddTextToStore << name << s << references; + auto conn(getConnection()); + conn->to << wopAddTextToStore << name << s; + writeStorePaths(*this, conn->to, references); - conn->processStderr(); - return readStorePath(*this, conn->from); + conn.processStderr(); + return parseStorePath(readString(conn->from)); } -void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) +void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopBuildPaths; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13) { - conn->to << drvPaths; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) - conn->to << buildMode; - else - /* Old daemons did not take a 'buildMode' parameter, so we - need to validate it here on the client side. */ - if (buildMode != bmNormal) - throw Error("repairing or checking is not supported when building through the Nix daemon"); - } else { - /* For backwards compatibility with old daemons, strip output - identifiers. */ - PathSet drvPaths2; - for (auto & i : drvPaths) - drvPaths2.insert(string(i, 0, i.find('!'))); - conn->to << drvPaths2; - } - conn->processStderr(); + assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); + Strings ss; + for (auto & p : drvPaths) + ss.push_back(p.to_string(*this)); + conn->to << ss; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) + conn->to << buildMode; + else + /* Old daemons did not take a 'buildMode' parameter, so we + need to validate it here on the client side. */ + if (buildMode != bmNormal) + throw Error("repairing or checking is not supported when building through the Nix daemon"); + conn.processStderr(); readInt(conn->from); } -BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv, +BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { - auto conn(connections->get()); - conn->to << wopBuildDerivation << drvPath << drv << buildMode; - conn->processStderr(); + auto conn(getConnection()); + conn->to << wopBuildDerivation << printStorePath(drvPath); + writeDerivation(conn->to, *this, drv); + conn->to << buildMode; + conn.processStderr(); BuildResult res; unsigned int status; conn->from >> status >> res.errorMsg; @@ -473,53 +565,53 @@ BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDeriva } -void RemoteStore::ensurePath(const Path & path) +void RemoteStore::ensurePath(const StorePath & path) { - auto conn(connections->get()); - conn->to << wopEnsurePath << path; - conn->processStderr(); + auto conn(getConnection()); + conn->to << wopEnsurePath << printStorePath(path); + conn.processStderr(); readInt(conn->from); } -void RemoteStore::addTempRoot(const Path & path) +void RemoteStore::addTempRoot(const StorePath & path) { - auto conn(connections->get()); - conn->to << wopAddTempRoot << path; - conn->processStderr(); + auto conn(getConnection()); + conn->to << wopAddTempRoot << printStorePath(path); + conn.processStderr(); readInt(conn->from); } void RemoteStore::addIndirectRoot(const Path & path) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopAddIndirectRoot << path; - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } void RemoteStore::syncWithGC() { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopSyncWithGC; - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } -Roots RemoteStore::findRoots() +Roots RemoteStore::findRoots(bool censor) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopFindRoots; - conn->processStderr(); - unsigned int count = readInt(conn->from); + conn.processStderr(); + size_t count = readNum(conn->from); Roots result; while (count--) { Path link = readString(conn->from); - Path target = readStorePath(*this, conn->from); - result[link] = target; + auto target = parseStorePath(readString(conn->from)); + result[std::move(target)].emplace(link); } return result; } @@ -527,15 +619,17 @@ Roots RemoteStore::findRoots() void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to - << wopCollectGarbage << options.action << options.pathsToDelete << options.ignoreLiveness + << wopCollectGarbage << options.action; + writeStorePaths(*this, conn->to, options.pathsToDelete); + conn->to << options.ignoreLiveness << options.maxFreed /* removed options */ << 0 << 0 << 0; - conn->processStderr(); + conn.processStderr(); results.paths = readStrings(conn->from); results.bytesFreed = readLongLong(conn->from); @@ -550,31 +644,79 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) void RemoteStore::optimiseStore() { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopOptimiseStore; - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } -bool RemoteStore::verifyStore(bool checkContents, bool repair) +bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopVerifyStore << checkContents << repair; - conn->processStderr(); - return readInt(conn->from) != 0; + conn.processStderr(); + return readInt(conn->from); } -void RemoteStore::addSignatures(const Path & storePath, const StringSet & sigs) +void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & sigs) { - auto conn(connections->get()); - conn->to << wopAddSignatures << storePath << sigs; - conn->processStderr(); + auto conn(getConnection()); + conn->to << wopAddSignatures << printStorePath(storePath) << sigs; + conn.processStderr(); readInt(conn->from); } +void RemoteStore::queryMissing(const std::vector & targets, + StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, + unsigned long long & downloadSize, unsigned long long & narSize) +{ + { + auto conn(getConnection()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19) + // Don't hold the connection handle in the fallback case + // to prevent a deadlock. + goto fallback; + conn->to << wopQueryMissing; + Strings ss; + for (auto & p : targets) + ss.push_back(p.to_string(*this)); + conn->to << ss; + conn.processStderr(); + willBuild = readStorePaths(*this, conn->from); + willSubstitute = readStorePaths(*this, conn->from); + unknown = readStorePaths(*this, conn->from); + conn->from >> downloadSize >> narSize; + return; + } + + fallback: + return Store::queryMissing(targets, willBuild, willSubstitute, + unknown, downloadSize, narSize); +} + + +void RemoteStore::connect() +{ + auto conn(getConnection()); +} + + +unsigned int RemoteStore::getProtocol() +{ + auto conn(connections->get()); + return conn->daemonVersion; +} + + +void RemoteStore::flushBadConnections() +{ + connections->flushBad(); +} + + RemoteStore::Connection::~Connection() { try { @@ -585,35 +727,94 @@ RemoteStore::Connection::~Connection() } -void RemoteStore::Connection::processStderr(Sink * sink, Source * source) +static Logger::Fields readFields(Source & from) +{ + Logger::Fields fields; + size_t size = readInt(from); + for (size_t n = 0; n < size; n++) { + auto type = (decltype(Logger::Field::type)) readInt(from); + if (type == Logger::Field::tInt) + fields.push_back(readNum(from)); + else if (type == Logger::Field::tString) + fields.push_back(readString(from)); + else + throw Error("got unsupported field type %x from Nix daemon", (int) type); + } + return fields; +} + + +std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source) { to.flush(); - unsigned int msg; - while ((msg = readInt(from)) == STDERR_NEXT - || msg == STDERR_READ || msg == STDERR_WRITE) { + + while (true) { + + auto msg = readNum(from); + if (msg == STDERR_WRITE) { string s = readString(from); if (!sink) throw Error("no sink"); (*sink)(s); } + else if (msg == STDERR_READ) { if (!source) throw Error("no source"); - size_t len = readInt(from); + size_t len = readNum(from); auto buf = std::make_unique(len); writeString(buf.get(), source->read(buf.get(), len), to); to.flush(); } - else + + else if (msg == STDERR_ERROR) { + string error = readString(from); + unsigned int status = readInt(from); + return std::make_exception_ptr(Error(status, error)); + } + + else if (msg == STDERR_NEXT) printError(chomp(readString(from))); + + else if (msg == STDERR_START_ACTIVITY) { + auto act = readNum(from); + auto lvl = (Verbosity) readInt(from); + auto type = (ActivityType) readInt(from); + auto s = readString(from); + auto fields = readFields(from); + auto parent = readNum(from); + logger->startActivity(act, lvl, type, s, fields, parent); + } + + else if (msg == STDERR_STOP_ACTIVITY) { + auto act = readNum(from); + logger->stopActivity(act); + } + + else if (msg == STDERR_RESULT) { + auto act = readNum(from); + auto type = (ResultType) readInt(from); + auto fields = readFields(from); + logger->result(act, type, fields); + } + + else if (msg == STDERR_LAST) + break; + + else + throw Error("got unknown message type %x from Nix daemon", msg); } - if (msg == STDERR_ERROR) { - string error = readString(from); - unsigned int status = readInt(from); - throw Error(status, error); - } - else if (msg != STDERR_LAST) - throw Error("protocol error processing standard error"); + + return nullptr; } +static std::string uriScheme = "unix://"; + +static RegisterStoreImplementation regStore([]( + const std::string & uri, const Store::Params & params) + -> std::shared_ptr +{ + if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; + return std::make_shared(std::string(uri, uriScheme.size()), params); +}); } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 40f17da300d..80c8e9f1168 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -14,6 +14,7 @@ class Pid; struct FdSink; struct FdSource; template class Pool; +struct ConnectionHandle; /* FIXME: RemoteStore is a misnomer - should be something like @@ -22,109 +23,136 @@ class RemoteStore : public virtual Store { public: - RemoteStore(const Params & params, size_t maxConnections = std::numeric_limits::max()); + const Setting maxConnections{(Store*) this, 1, + "max-connections", "maximum number of concurrent connections to the Nix daemon"}; - /* Implementations of abstract store API methods. */ + const Setting maxConnectionAge{(Store*) this, std::numeric_limits::max(), + "max-connection-age", "number of seconds to reuse a connection"}; + + virtual bool sameMachine() = 0; - bool isValidPathUncached(const Path & path) override; + RemoteStore(const Params & params); - PathSet queryValidPaths(const PathSet & paths) override; + /* Implementations of abstract store API methods. */ - PathSet queryAllValidPaths() override; + bool isValidPathUncached(const StorePath & path) override; - void queryPathInfoUncached(const Path & path, - std::function)> success, - std::function failure) override; + StorePathSet queryValidPaths(const StorePathSet & paths, + SubstituteFlag maybeSubstitute = NoSubstitute) override; - void queryReferrers(const Path & path, PathSet & referrers) override; + StorePathSet queryAllValidPaths() override; - PathSet queryValidDerivers(const Path & path) override; + void queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept override; - PathSet queryDerivationOutputs(const Path & path) override; + void queryReferrers(const StorePath & path, StorePathSet & referrers) override; - StringSet queryDerivationOutputNames(const Path & path) override; + StorePathSet queryValidDerivers(const StorePath & path) override; - Path queryPathFromHashPart(const string & hashPart) override; + StorePathSet queryDerivationOutputs(const StorePath & path) override; - PathSet querySubstitutablePaths(const PathSet & paths) override; + std::optional queryPathFromHashPart(const std::string & hashPart) override; - void querySubstitutablePathInfos(const PathSet & paths, + StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; + + void querySubstitutablePathInfos(const StorePathSet & paths, SubstitutablePathInfos & infos) override; - void addToStore(const ValidPathInfo & info, const ref & nar, - bool repair, bool dontCheckSigs, + void addToStore(const ValidPathInfo & info, Source & nar, + RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr accessor) override; - Path addToStore(const string & name, const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, bool repair = false) override; + StorePath addToStore(const string & name, const Path & srcPath, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override; - Path addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair = false) override; + StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) override; - void buildPaths(const PathSet & paths, BuildMode buildMode) override; + void buildPaths(const std::vector & paths, BuildMode buildMode) override; - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; - void ensurePath(const Path & path) override; + void ensurePath(const StorePath & path) override; - void addTempRoot(const Path & path) override; + void addTempRoot(const StorePath & path) override; void addIndirectRoot(const Path & path) override; void syncWithGC() override; - Roots findRoots() override; + Roots findRoots(bool censor) override; void collectGarbage(const GCOptions & options, GCResults & results) override; void optimiseStore() override; - bool verifyStore(bool checkContents, bool repair) override; + bool verifyStore(bool checkContents, RepairFlag repair) override; + + void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - void addSignatures(const Path & storePath, const StringSet & sigs) override; + void queryMissing(const std::vector & targets, + StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, + unsigned long long & downloadSize, unsigned long long & narSize) override; + + void connect() override; + + unsigned int getProtocol() override; + + void flushBadConnections(); protected: struct Connection { + AutoCloseFD fd; FdSink to; FdSource from; unsigned int daemonVersion; + std::chrono::time_point startTime; virtual ~Connection(); - void processStderr(Sink * sink = 0, Source * source = 0); + std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0); }; + ref openConnectionWrapper(); + virtual ref openConnection() = 0; void initConnection(Connection & conn); ref> connections; + virtual void setOptions(Connection & conn); + + ConnectionHandle getConnection(); + + friend struct ConnectionHandle; + private: - void setOptions(Connection & conn); + std::atomic_bool failed{false}; + }; class UDSRemoteStore : public LocalFSStore, public RemoteStore { public: - UDSRemoteStore(const Params & params, size_t maxConnections = std::numeric_limits::max()); + UDSRemoteStore(const Params & params); + UDSRemoteStore(std::string path, const Params & params); std::string getUri() override; -private: + bool sameMachine() override + { return true; } - struct Connection : RemoteStore::Connection - { - AutoCloseFD fd; - }; +private: ref openConnection() override; + std::optional path; }; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index ccb71f1eefe..427dd48ce36 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -1,39 +1,41 @@ -#include "config.h" - #if ENABLE_S3 -#if __linux__ +#include "s3.hh" #include "s3-binary-cache-store.hh" #include "nar-info.hh" #include "nar-info-disk-cache.hh" #include "globals.hh" +#include "compression.hh" +#include "filetransfer.hh" +#include "istringstream_nocopy.hh" #include +#include +#include +#include #include +#include +#include +#include +#include #include -#include -#include #include #include #include #include +#include -namespace nix { +using namespace Aws::Transfer; -struct istringstream_nocopy : public std::stringstream -{ - istringstream_nocopy(const std::string & s) - { - rdbuf()->pubsetbuf( - (char *) s.data(), s.size()); - } -}; +namespace nix { struct S3Error : public Error { Aws::S3::S3Errors err; - S3Error(Aws::S3::S3Errors err, const FormatOrString & fs) - : Error(fs), err(err) { }; + + template + S3Error(Aws::S3::S3Errors err, const Args & ... args) + : Error(args...), err(err) { }; }; /* Helper: given an Outcome, return R in case of success, or @@ -48,6 +50,16 @@ R && checkAws(const FormatOrString & fs, Aws::Utils::Outcome && outcome) return outcome.GetResultWithOwnership(); } +class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem +{ + using Aws::Utils::Logging::FormattedLogSystem::FormattedLogSystem; + + void ProcessFormattedStatement(Aws::String && statement) override + { + debug("AWS: %s", chomp(statement)); + } +}; + static void initAWS() { static std::once_flag flag; @@ -58,25 +70,134 @@ static void initAWS() shared.cc), so don't let aws-sdk-cpp override it. */ options.cryptoOptions.initAndCleanupOpenSSL = false; + if (verbosity >= lvlDebug) { + options.loggingOptions.logLevel = + verbosity == lvlDebug + ? Aws::Utils::Logging::LogLevel::Debug + : Aws::Utils::Logging::LogLevel::Trace; + options.loggingOptions.logger_create_fn = [options]() { + return std::make_shared(options.loggingOptions.logLevel); + }; + } + Aws::InitAPI(options); }); } +S3Helper::S3Helper(const string & profile, const string & region, const string & scheme, const string & endpoint) + : config(makeConfig(region, scheme, endpoint)) + , client(make_ref( + profile == "" + ? std::dynamic_pointer_cast( + std::make_shared()) + : std::dynamic_pointer_cast( + std::make_shared(profile.c_str())), + *config, + // FIXME: https://github.com/aws/aws-sdk-cpp/issues/759 +#if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3 + false, +#else + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, +#endif + endpoint.empty())) +{ +} + +/* Log AWS retries. */ +class RetryStrategy : public Aws::Client::DefaultRetryStrategy +{ + bool ShouldRetry(const Aws::Client::AWSError& error, long attemptedRetries) const override + { + auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries); + if (retry) + printError("AWS error '%s' (%s), will retry in %d ms", + error.GetExceptionName(), + error.GetMessage(), + CalculateDelayBeforeNextRetry(error, attemptedRetries)); + return retry; + } +}; + +ref S3Helper::makeConfig(const string & region, const string & scheme, const string & endpoint) +{ + initAWS(); + auto res = make_ref(); + res->region = region; + if (!scheme.empty()) { + res->scheme = Aws::Http::SchemeMapper::FromString(scheme.c_str()); + } + if (!endpoint.empty()) { + res->endpointOverride = endpoint; + } + res->requestTimeoutMs = 600 * 1000; + res->connectTimeoutMs = 5 * 1000; + res->retryStrategy = std::make_shared(); + res->caFile = settings.caFile; + return res; +} + +S3Helper::FileTransferResult S3Helper::getObject( + const std::string & bucketName, const std::string & key) +{ + debug("fetching 's3://%s/%s'...", bucketName, key); + + auto request = + Aws::S3::Model::GetObjectRequest() + .WithBucket(bucketName) + .WithKey(key); + + request.SetResponseStreamFactory([&]() { + return Aws::New("STRINGSTREAM"); + }); + + FileTransferResult res; + + auto now1 = std::chrono::steady_clock::now(); + + try { + + auto result = checkAws(fmt("AWS error fetching '%s'", key), + client->GetObject(request)); + + res.data = decompress(result.GetContentEncoding(), + dynamic_cast(result.GetBody()).str()); + + } catch (S3Error & e) { + if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw; + } + + auto now2 = std::chrono::steady_clock::now(); + + res.durationMs = std::chrono::duration_cast(now2 - now1).count(); + + return res; +} + struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore { - std::string bucketName; + const Setting profile{this, "", "profile", "The name of the AWS configuration profile to use."}; + const Setting region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; + const Setting scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."}; + const Setting endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; + const Setting narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"}; + const Setting lsCompression{this, "", "ls-compression", "compression method for .ls files"}; + const Setting logCompression{this, "", "log-compression", "compression method for log/* files"}; + const Setting multipartUpload{ + this, false, "multipart-upload", "whether to use multi-part uploads"}; + const Setting bufferSize{ + this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; - ref config; - ref client; + std::string bucketName; Stats stats; + S3Helper s3Helper; + S3BinaryCacheStoreImpl( const Params & params, const std::string & bucketName) : S3BinaryCacheStore(params) , bucketName(bucketName) - , config(makeConfig()) - , client(make_ref(*config)) + , s3Helper(profile, region, scheme, endpoint) { diskCache = getNarInfoDiskCache(); } @@ -86,42 +207,14 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore return "s3://" + bucketName; } - ref makeConfig() - { - initAWS(); - auto res = make_ref(); - res->region = Aws::Region::US_EAST_1; // FIXME: make configurable - res->requestTimeoutMs = 600 * 1000; - return res; - } - void init() override { - if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) { - - /* Create the bucket if it doesn't already exists. */ - // FIXME: HeadBucket would be more appropriate, but doesn't return - // an easily parsed 404 message. - auto res = client->GetBucketLocation( - Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName)); - - if (!res.IsSuccess()) { - if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET) - throw Error(format("AWS error checking bucket ‘%s’: %s") % bucketName % res.GetError().GetMessage()); - - checkAws(format("AWS error creating bucket ‘%s’") % bucketName, - client->CreateBucket( - Aws::S3::Model::CreateBucketRequest() - .WithBucket(bucketName) - .WithCreateBucketConfiguration( - Aws::S3::Model::CreateBucketConfiguration() - /* .WithLocationConstraint( - Aws::S3::Model::BucketLocationConstraint::US) */ ))); - } - + if (auto cacheInfo = diskCache->cacheExists(getUri())) { + wantMassQuery.setDefault(cacheInfo->wantMassQuery ? "true" : "false"); + priority.setDefault(fmt("%d", cacheInfo->priority)); + } else { BinaryCacheStore::init(); - - diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority); + diskCache->createCache(getUri(), storeDir, wantMassQuery, priority); } } @@ -134,7 +227,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore fetches the .narinfo file, rather than first checking for its existence via a HEAD request. Since .narinfos are small, doing a GET is unlikely to be slower than HEAD. */ - bool isValidPathUncached(const Path & storePath) override + bool isValidPathUncached(const StorePath & storePath) override { try { queryPathInfo(storePath); @@ -148,107 +241,162 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore { stats.head++; - auto res = client->HeadObject( + auto res = s3Helper.client->HeadObject( Aws::S3::Model::HeadObjectRequest() .WithBucket(bucketName) .WithKey(path)); if (!res.IsSuccess()) { auto & error = res.GetError(); - if (error.GetErrorType() == Aws::S3::S3Errors::UNKNOWN // FIXME - && error.GetMessage().find("404") != std::string::npos) + if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND + || error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY + // If bucket listing is disabled, 404s turn into 403s + || error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED) return false; - throw Error(format("AWS error fetching ‘%s’: %s") % path % error.GetMessage()); + throw Error("AWS error fetching '%s': %s", path, error.GetMessage()); } return true; } - void upsertFile(const std::string & path, const std::string & data) override - { - auto request = - Aws::S3::Model::PutObjectRequest() - .WithBucket(bucketName) - .WithKey(path); + std::shared_ptr transferManager; + std::once_flag transferManagerCreated; + void uploadFile(const std::string & path, const std::string & data, + const std::string & mimeType, + const std::string & contentEncoding) + { auto stream = std::make_shared(data); - request.SetBody(stream); - - stats.put++; - stats.putBytes += data.size(); + auto maxThreads = std::thread::hardware_concurrency(); + + static std::shared_ptr + executor = std::make_shared(maxThreads); + + std::call_once(transferManagerCreated, [&]() + { + if (multipartUpload) { + TransferManagerConfiguration transferConfig(executor.get()); + + transferConfig.s3Client = s3Helper.client; + transferConfig.bufferSize = bufferSize; + + transferConfig.uploadProgressCallback = + [](const TransferManager *transferManager, + const std::shared_ptr + &transferHandle) + { + //FIXME: find a way to properly abort the multipart upload. + //checkInterrupt(); + debug("upload progress ('%s'): '%d' of '%d' bytes", + transferHandle->GetKey(), + transferHandle->GetBytesTransferred(), + transferHandle->GetBytesTotalSize()); + }; + + transferManager = TransferManager::Create(transferConfig); + } + }); auto now1 = std::chrono::steady_clock::now(); - auto result = checkAws(format("AWS error uploading ‘%s’") % path, - client->PutObject(request)); + if (transferManager) { - auto now2 = std::chrono::steady_clock::now(); + if (contentEncoding != "") + throw Error("setting a content encoding is not supported with S3 multi-part uploads"); - auto duration = std::chrono::duration_cast(now2 - now1).count(); + std::shared_ptr transferHandle = + transferManager->UploadFile( + stream, bucketName, path, mimeType, + Aws::Map(), + nullptr /*, contentEncoding */); - printInfo(format("uploaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms") - % bucketName % path % data.size() % duration); + transferHandle->WaitUntilFinished(); - stats.putTimeMs += duration; - } + if (transferHandle->GetStatus() == TransferStatus::FAILED) + throw Error("AWS error: failed to upload 's3://%s/%s': %s", + bucketName, path, transferHandle->GetLastError().GetMessage()); - void getFile(const std::string & path, - std::function)> success, - std::function failure) override - { - sync2async>(success, failure, [&]() { - debug(format("fetching ‘s3://%1%/%2%’...") % bucketName % path); + if (transferHandle->GetStatus() != TransferStatus::COMPLETED) + throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state", + bucketName, path); + + } else { auto request = - Aws::S3::Model::GetObjectRequest() + Aws::S3::Model::PutObjectRequest() .WithBucket(bucketName) .WithKey(path); - request.SetResponseStreamFactory([&]() { - return Aws::New("STRINGSTREAM"); - }); + request.SetContentType(mimeType); - stats.get++; + if (contentEncoding != "") + request.SetContentEncoding(contentEncoding); - try { + auto stream = std::make_shared(data); + + request.SetBody(stream); + + auto result = checkAws(fmt("AWS error uploading '%s'", path), + s3Helper.client->PutObject(request)); + } + + auto now2 = std::chrono::steady_clock::now(); - auto now1 = std::chrono::steady_clock::now(); + auto duration = + std::chrono::duration_cast(now2 - now1) + .count(); - auto result = checkAws(format("AWS error fetching ‘%s’") % path, - client->GetObject(request)); + printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") % + bucketName % path % data.size() % duration); - auto now2 = std::chrono::steady_clock::now(); + stats.putTimeMs += duration; + stats.putBytes += data.size(); + stats.put++; + } - auto res = dynamic_cast(result.GetBody()).str(); + void upsertFile(const std::string & path, const std::string & data, + const std::string & mimeType) override + { + if (narinfoCompression != "" && hasSuffix(path, ".narinfo")) + uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression); + else if (lsCompression != "" && hasSuffix(path, ".ls")) + uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression); + else if (logCompression != "" && hasPrefix(path, "log/")) + uploadFile(path, *compress(logCompression, data), mimeType, logCompression); + else + uploadFile(path, data, mimeType, ""); + } - auto duration = std::chrono::duration_cast(now2 - now1).count(); + void getFile(const std::string & path, Sink & sink) override + { + stats.get++; - printMsg(lvlTalkative, format("downloaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms") - % bucketName % path % res.size() % duration); + // FIXME: stream output to sink. + auto res = s3Helper.getObject(bucketName, path); - stats.getBytes += res.size(); - stats.getTimeMs += duration; + stats.getBytes += res.data ? res.data->size() : 0; + stats.getTimeMs += res.durationMs; - return std::make_shared(res); + if (res.data) { + printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", + bucketName, path, res.data->size(), res.durationMs); - } catch (S3Error & e) { - if (e.err == Aws::S3::S3Errors::NO_SUCH_KEY) return std::shared_ptr(); - throw; - } - }); + sink((unsigned char *) res.data->data(), res.data->size()); + } else + throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); } - PathSet queryAllValidPaths() override + StorePathSet queryAllValidPaths() override { - PathSet paths; + StorePathSet paths; std::string marker; do { - debug(format("listing bucket ‘s3://%s’ from key ‘%s’...") % bucketName % marker); + debug(format("listing bucket 's3://%s' from key '%s'...") % bucketName % marker); - auto res = checkAws(format("AWS error listing bucket ‘%s’") % bucketName, - client->ListObjects( + auto res = checkAws(format("AWS error listing bucket '%s'") % bucketName, + s3Helper.client->ListObjects( Aws::S3::Model::ListObjectsRequest() .WithBucket(bucketName) .WithDelimiter("/") @@ -256,13 +404,13 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore auto & contents = res.GetContents(); - debug(format("got %d keys, next marker ‘%s’") + debug(format("got %d keys, next marker '%s'") % contents.size() % res.GetNextMarker()); for (auto object : contents) { auto & key = object.GetKey(); if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue; - paths.insert(storeDir + "/" + key.substr(0, key.size() - 8)); + paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-unknown")); } marker = res.GetNextMarker(); @@ -286,4 +434,3 @@ static RegisterStoreImplementation regStore([]( } #endif -#endif diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh new file mode 100644 index 00000000000..2042bffcf94 --- /dev/null +++ b/src/libstore/s3.hh @@ -0,0 +1,33 @@ +#pragma once + +#if ENABLE_S3 + +#include "ref.hh" + +namespace Aws { namespace Client { class ClientConfiguration; } } +namespace Aws { namespace S3 { class S3Client; } } + +namespace nix { + +struct S3Helper +{ + ref config; + ref client; + + S3Helper(const std::string & profile, const std::string & region, const std::string & scheme, const std::string & endpoint); + + ref makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint); + + struct FileTransferResult + { + std::shared_ptr data; + unsigned int durationMs; + }; + + FileTransferResult getObject( + const std::string & bucketName, const std::string & key); +}; + +} + +#endif diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb new file mode 100644 index 00000000000..351037822fd --- /dev/null +++ b/src/libstore/sandbox-defaults.sb @@ -0,0 +1,97 @@ +(define TMPDIR (param "_GLOBAL_TMP_DIR")) + +(deny default) + +; Disallow creating setuid/setgid binaries, since that +; would allow breaking build user isolation. +(deny file-write-setugid) + +; Allow forking. +(allow process-fork) + +; Allow reading system information like #CPUs, etc. +(allow sysctl-read) + +; Allow POSIX semaphores and shared memory. +(allow ipc-posix*) + +; Allow socket creation. +(allow system-socket) + +; Allow sending signals within the sandbox. +(allow signal (target same-sandbox)) + +; Allow getpwuid. +(allow mach-lookup (global-name "com.apple.system.opendirectoryd.libinfo")) + +; Access to /tmp. +; The network-outbound/network-inbound ones are for unix domain sockets, which +; we allow access to in TMPDIR (but if we allow them more broadly, you could in +; theory escape the sandbox) +(allow file* process-exec network-outbound network-inbound + (literal "/tmp") (subpath TMPDIR)) + +; Some packages like to read the system version. +(allow file-read* (literal "/System/Library/CoreServices/SystemVersion.plist")) + +; Without this line clang cannot write to /dev/null, breaking some configure tests. +(allow file-read-metadata (literal "/dev")) + +; Many packages like to do local networking in their test suites, but let's only +; allow it if the package explicitly asks for it. +(if (param "_ALLOW_LOCAL_NETWORKING") + (begin + (allow network* (local ip) (local tcp) (local udp)) + + ; Allow access to /etc/resolv.conf (which is a symlink to + ; /private/var/run/resolv.conf). + ; TODO: deduplicate with sandbox-network.sb + (allow file-read-metadata + (literal "/var") + (literal "/etc") + (literal "/etc/resolv.conf") + (literal "/private/etc/resolv.conf")) + + (allow file-read* + (literal "/private/var/run/resolv.conf")) + + ; Allow DNS lookups. This is even needed for localhost, which lots of tests rely on + (allow file-read-metadata (literal "/etc/hosts")) + (allow file-read* (literal "/private/etc/hosts")) + (allow network-outbound (remote unix-socket (path-literal "/private/var/run/mDNSResponder"))))) + +; Standard devices. +(allow file* + (literal "/dev/null") + (literal "/dev/random") + (literal "/dev/stdin") + (literal "/dev/stdout") + (literal "/dev/tty") + (literal "/dev/urandom") + (literal "/dev/zero") + (subpath "/dev/fd")) + +; Allow pseudo-terminals. +(allow file* + (literal "/dev/ptmx") + (regex #"^/dev/pty[a-z]+") + (regex #"^/dev/ttys[0-9]+")) + +; Does nothing, but reduces build noise. +(allow file* (literal "/dev/dtracehelper")) + +; Allow access to zoneinfo since libSystem needs it. +(allow file-read* (subpath "/usr/share/zoneinfo")) + +(allow file-read* (subpath "/usr/share/locale")) + +; This is mostly to get more specific log messages when builds try to +; access something in /etc or /var. +(allow file-read-metadata + (literal "/etc") + (literal "/var") + (literal "/private/var/tmp")) + +; This is used by /bin/sh on macOS 10.15 and later. +(allow file* + (literal "/private/var/select/sh")) diff --git a/src/libstore/sandbox-defaults.sb.in b/src/libstore/sandbox-defaults.sb.in deleted file mode 100644 index b5e80085fbe..00000000000 --- a/src/libstore/sandbox-defaults.sb.in +++ /dev/null @@ -1,63 +0,0 @@ -(allow file-read* file-write-data (literal "/dev/null")) -(allow ipc-posix*) -(allow mach-lookup (global-name "com.apple.SecurityServer")) - -(allow file-read* - (literal "/dev/dtracehelper") - (literal "/dev/tty") - (literal "/dev/autofs_nowait") - (literal "/System/Library/CoreServices/SystemVersion.plist") - (literal "/private/var/run/systemkeychaincheck.done") - (literal "/private/etc/protocols") - (literal "/private/var/tmp") - (literal "/private/var/db") - (subpath "/private/var/db/mds")) - -(allow file-read* - (subpath "/usr/share/icu") - (subpath "/usr/share/locale") - (subpath "/usr/share/zoneinfo")) - -(allow file-write* - (literal "/dev/tty") - (literal "/dev/dtracehelper") - (literal "/mds")) - -(allow file-ioctl (literal "/dev/dtracehelper")) - -(allow file-read-metadata - (literal "/var") - (literal "/tmp") - ; symlinks - (literal "@sysconfdir@") - (literal "@sysconfdir@/nix") - (literal "@sysconfdir@/nix/nix.conf") - (literal "/etc/resolv.conf") - (literal "/private/etc/resolv.conf")) - -(allow file-read* - (literal "/private@sysconfdir@/nix/nix.conf") - (literal "/private/var/run/resolv.conf")) - -; some builders use filehandles other than stdin/stdout -(allow file* - (subpath "/dev/fd") - (literal "/dev/ptmx") - (regex #"^/dev/[pt]ty.*$")) - -; allow everything inside TMP -(allow file* process-exec - (subpath (param "_GLOBAL_TMP_DIR")) - (subpath "/private/tmp")) - -(allow process-fork) -(allow sysctl-read) -(allow signal (target same-sandbox)) - -; allow getpwuid (for git and other packages) -(allow mach-lookup - (global-name "com.apple.system.notification_center") - (global-name "com.apple.system.opendirectoryd.libinfo")) - -; allow local networking -(allow network* (local ip) (remote unix-socket)) diff --git a/src/libstore/sandbox-minimal.sb b/src/libstore/sandbox-minimal.sb new file mode 100644 index 00000000000..65f5108b399 --- /dev/null +++ b/src/libstore/sandbox-minimal.sb @@ -0,0 +1,5 @@ +(allow default) + +; Disallow creating setuid/setgid binaries, since that +; would allow breaking build user isolation. +(deny file-write-setugid) diff --git a/src/libstore/sandbox-network.sb b/src/libstore/sandbox-network.sb new file mode 100644 index 00000000000..56beec761fa --- /dev/null +++ b/src/libstore/sandbox-network.sb @@ -0,0 +1,16 @@ +; Allow local and remote network traffic. +(allow network* (local ip) (remote ip)) + +; Allow access to /etc/resolv.conf (which is a symlink to +; /private/var/run/resolv.conf). +(allow file-read-metadata + (literal "/var") + (literal "/etc") + (literal "/etc/resolv.conf") + (literal "/private/etc/resolv.conf")) + +(allow file-read* + (literal "/private/var/run/resolv.conf")) + +; Allow DNS lookups. +(allow network-outbound (remote unix-socket (path-literal "/private/var/run/mDNSResponder"))) diff --git a/src/nix-store/serve-protocol.hh b/src/libstore/serve-protocol.hh similarity index 87% rename from src/nix-store/serve-protocol.hh rename to src/libstore/serve-protocol.hh index f8cc9a4b6eb..9fae6d5349f 100644 --- a/src/nix-store/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -5,7 +5,7 @@ namespace nix { #define SERVE_MAGIC_1 0x390c9deb #define SERVE_MAGIC_2 0x5452eecb -#define SERVE_PROTOCOL_VERSION 0x203 +#define SERVE_PROTOCOL_VERSION 0x205 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -18,6 +18,7 @@ typedef enum { cmdBuildPaths = 6, cmdQueryClosure = 7, cmdBuildDerivation = 8, + cmdAddToStoreNar = 9, } ServeCommand; } diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 0197b091cd1..76c822c4e9c 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -3,43 +3,38 @@ #include +#include + namespace nix { -[[noreturn]] void throwSQLiteError(sqlite3 * db, const format & f) +[[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs) { int err = sqlite3_errcode(db); + int exterr = sqlite3_extended_errcode(db); + + auto path = sqlite3_db_filename(db, nullptr); + if (!path) path = "(in-memory)"; + if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { - if (err == SQLITE_PROTOCOL) - printError("warning: SQLite database is busy (SQLITE_PROTOCOL)"); - else { - static bool warned = false; - if (!warned) { - printError("warning: SQLite database is busy"); - warned = true; - } - } - /* Sleep for a while since retrying the transaction right away - is likely to fail again. */ - checkInterrupt(); -#if HAVE_NANOSLEEP - struct timespec t; - t.tv_sec = 0; - t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ - nanosleep(&t, 0); -#else - sleep(1); -#endif - throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); + throw SQLiteBusy( + err == SQLITE_PROTOCOL + ? fmt("SQLite database '%s' is busy (SQLITE_PROTOCOL)", path) + : fmt("SQLite database '%s' is busy", path)); } else - throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); + throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path); } -SQLite::SQLite(const Path & path) +SQLite::SQLite(const Path & path, bool create) { if (sqlite3_open_v2(path.c_str(), &db, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) - throw Error(format("cannot open SQLite database ‘%s’") % path); + SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) + throw Error("cannot open SQLite database '%s'", path); + + if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) + throwSQLiteError(db, "setting timeout"); + + exec("pragma foreign_keys = 1"); } SQLite::~SQLite() @@ -52,26 +47,35 @@ SQLite::~SQLite() } } +void SQLite::isCache() +{ + exec("pragma synchronous = off"); + exec("pragma main.journal_mode = truncate"); +} + void SQLite::exec(const std::string & stmt) { - if (sqlite3_exec(db, stmt.c_str(), 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, format("executing SQLite statement ‘%s’") % stmt); + retrySQLite([&]() { + if (sqlite3_exec(db, stmt.c_str(), 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, format("executing SQLite statement '%s'") % stmt); + }); } -void SQLiteStmt::create(sqlite3 * db, const string & s) +void SQLiteStmt::create(sqlite3 * db, const string & sql) { checkInterrupt(); assert(!stmt); - if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) - throwSQLiteError(db, "creating statement"); + if (sqlite3_prepare_v2(db, sql.c_str(), -1, &stmt, 0) != SQLITE_OK) + throwSQLiteError(db, fmt("creating statement '%s'", sql)); this->db = db; + this->sql = sql; } SQLiteStmt::~SQLiteStmt() { try { if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) - throwSQLiteError(db, "finalizing statement"); + throwSQLiteError(db, fmt("finalizing statement '%s'", sql)); } catch (...) { ignoreException(); } @@ -101,6 +105,16 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool return *this; } +SQLiteStmt::Use & SQLiteStmt::Use::operator () (const unsigned char * data, size_t len, bool notNull) +{ + if (notNull) { + if (sqlite3_bind_blob(stmt, curArg++, data, len, SQLITE_TRANSIENT) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; +} + SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull) { if (notNull) { @@ -128,14 +142,14 @@ void SQLiteStmt::Use::exec() int r = step(); assert(r != SQLITE_ROW); if (r != SQLITE_DONE) - throwSQLiteError(stmt.db, "executing SQLite statement"); + throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", stmt.sql)); } bool SQLiteStmt::Use::next() { int r = step(); if (r != SQLITE_DONE && r != SQLITE_ROW) - throwSQLiteError(stmt.db, "executing SQLite query"); + throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", stmt.sql)); return r == SQLITE_ROW; } @@ -182,4 +196,27 @@ SQLiteTxn::~SQLiteTxn() } } +void handleSQLiteBusy(const SQLiteBusy & e) +{ + static std::atomic lastWarned{0}; + + time_t now = time(0); + + if (now > lastWarned + 10) { + lastWarned = now; + logWarning({ + .name = "Sqlite busy", + .hint = hintfmt(e.what()) + }); + } + + /* Sleep for a while since retrying the transaction right away + is likely to fail again. */ + checkInterrupt(); + struct timespec t; + t.tv_sec = 0; + t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ + nanosleep(&t, 0); +} + } diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 7c1ed538215..dd81ab05135 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -3,10 +3,10 @@ #include #include -#include "types.hh" +#include "error.hh" -class sqlite3; -class sqlite3_stmt; +struct sqlite3; +struct sqlite3_stmt; namespace nix { @@ -15,13 +15,16 @@ struct SQLite { sqlite3 * db = 0; SQLite() { } - SQLite(const Path & path); + SQLite(const Path & path, bool create = true); SQLite(const SQLite & from) = delete; SQLite& operator = (const SQLite & from) = delete; SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; } ~SQLite(); operator sqlite3 * () { return db; } + /* Disable synchronous mode, set truncate journal mode. */ + void isCache(); + void exec(const std::string & stmt); }; @@ -30,7 +33,9 @@ struct SQLiteStmt { sqlite3 * db = 0; sqlite3_stmt * stmt = 0; + std::string sql; SQLiteStmt() { } + SQLiteStmt(sqlite3 * db, const std::string & sql) { create(db, sql); } void create(sqlite3 * db, const std::string & s); ~SQLiteStmt(); operator sqlite3_stmt * () { return stmt; } @@ -50,6 +55,7 @@ struct SQLiteStmt /* Bind the next parameter. */ Use & operator () (const std::string & value, bool notNull = true); + Use & operator () (const unsigned char * data, size_t len, bool notNull = true); Use & operator () (int64_t value, bool notNull = true); Use & bind(); // null @@ -91,17 +97,20 @@ struct SQLiteTxn MakeError(SQLiteError, Error); MakeError(SQLiteBusy, SQLiteError); -[[noreturn]] void throwSQLiteError(sqlite3 * db, const format & f); +[[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs); + +void handleSQLiteBusy(const SQLiteBusy & e); /* Convenience function for retrying a SQLite transaction when the database is busy. */ -template -T retrySQLite(std::function fun) +template +T retrySQLite(F && fun) { while (true) { try { return fun(); } catch (SQLiteBusy & e) { + handleSQLiteBusy(e); } } } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 3d01594009a..caae6b596c7 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -4,18 +4,43 @@ #include "archive.hh" #include "worker-protocol.hh" #include "pool.hh" +#include "ssh.hh" namespace nix { +static std::string uriScheme = "ssh-ng://"; + class SSHStore : public RemoteStore { public: - SSHStore(string uri, const Params & params, size_t maxConnections = std::numeric_limits::max()); + const Setting sshKey{(Store*) this, "", "ssh-key", "path to an SSH private key"}; + const Setting compress{(Store*) this, false, "compress", "whether to compress the connection"}; + const Setting remoteProgram{(Store*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"}; + const Setting remoteStore{(Store*) this, "", "remote-store", "URI of the store on the remote system"}; + + SSHStore(const std::string & host, const Params & params) + : Store(params) + , RemoteStore(params) + , host(host) + , master( + host, + sshKey, + // Use SSH master only if using more than 1 connection. + connections->capacity() > 1, + compress) + { + } + + std::string getUri() override + { + return uriScheme + host; + } - std::string getUri() override; + bool sameMachine() override + { return false; } - void narFromPath(const Path & path, Sink & sink) override; + void narFromPath(const StorePath & path, Sink & sink) override; ref getFSAccessor() override; @@ -23,63 +48,32 @@ class SSHStore : public RemoteStore struct Connection : RemoteStore::Connection { - Pid sshPid; - AutoCloseFD out; - AutoCloseFD in; + std::unique_ptr sshConn; }; ref openConnection() override; - AutoDelete tmpDir; - - Path socketPath; - - Pid sshMaster; - - string uri; - - Path key; -}; - -SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections) - : Store(params) - , RemoteStore(params, maxConnections) - , tmpDir(createTempDir("", "nix", true, true, 0700)) - , socketPath((Path) tmpDir + "/ssh.sock") - , uri(std::move(uri)) - , key(get(params, "ssh-key", "")) -{ - /* open a connection and perform the handshake to verify all is well */ - connections->get(); -} + std::string host; -string SSHStore::getUri() -{ - return "ssh://" + uri; -} + SSHMaster master; -class ForwardSource : public Source -{ - Source & readSource; - Sink & writeSink; -public: - ForwardSource(Source & readSource, Sink & writeSink) : readSource(readSource), writeSink(writeSink) {} - size_t read(unsigned char * data, size_t len) override + void setOptions(RemoteStore::Connection & conn) override { - auto res = readSource.read(data, len); - writeSink(data, len); - return res; - } + /* TODO Add a way to explicitly ask for some options to be + forwarded. One option: A way to query the daemon for its + settings, and then a series of params to SSHStore like + forward-cores or forward-overridden-cores that only + override the requested settings. + */ + }; }; -void SSHStore::narFromPath(const Path & path, Sink & sink) +void SSHStore::narFromPath(const StorePath & path, Sink & sink) { auto conn(connections->get()); - conn->to << wopNarFromPath << path; + conn->to << wopNarFromPath << printStorePath(path); conn->processStderr(); - ParseSink ps; - auto fwd = ForwardSource(conn->from, sink); - parseDump(ps, fwd); + copyNAR(conn->from, sink); } ref SSHStore::getFSAccessor() @@ -89,34 +83,12 @@ ref SSHStore::getFSAccessor() ref SSHStore::openConnection() { - if ((pid_t) sshMaster == -1) { - sshMaster = startProcess([&]() { - if (key.empty()) - execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), uri.c_str(), NULL); - else - execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), "-i", key.c_str(), uri.c_str(), NULL); - throw SysError("starting ssh master"); - }); - } - auto conn = make_ref(); - Pipe in, out; - in.create(); - out.create(); - conn->sshPid = startProcess([&]() { - if (dup2(in.readSide.get(), STDIN_FILENO) == -1) - throw SysError("duping over STDIN"); - if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) - throw SysError("duping over STDOUT"); - execlp("ssh", "ssh", "-S", socketPath.c_str(), uri.c_str(), "nix-daemon", "--stdio", NULL); - throw SysError("executing nix-daemon --stdio over ssh"); - }); - in.readSide = -1; - out.writeSide = -1; - conn->out = std::move(out.readSide); - conn->in = std::move(in.writeSide); - conn->to = FdSink(conn->in.get()); - conn->from = FdSource(conn->out.get()); + conn->sshConn = master.startCommand( + fmt("%s --stdio", remoteProgram) + + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); + conn->to = FdSink(conn->sshConn->in.get()); + conn->from = FdSource(conn->sshConn->out.get()); initConnection(*conn); return conn; } @@ -125,8 +97,8 @@ static RegisterStoreImplementation regStore([]( const std::string & uri, const Store::Params & params) -> std::shared_ptr { - if (std::string(uri, 0, 6) != "ssh://") return 0; - return std::make_shared(uri.substr(6), params); + if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; + return std::make_shared(std::string(uri, uriScheme.size()), params); }); } diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc new file mode 100644 index 00000000000..84548a6e4eb --- /dev/null +++ b/src/libstore/ssh.cc @@ -0,0 +1,134 @@ +#include "ssh.hh" + +namespace nix { + +SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD) + : host(host) + , fakeSSH(host == "localhost") + , keyFile(keyFile) + , useMaster(useMaster && !fakeSSH) + , compress(compress) + , logFD(logFD) +{ + if (host == "" || hasPrefix(host, "-")) + throw Error("invalid SSH host name '%s'", host); +} + +void SSHMaster::addCommonSSHOpts(Strings & args) +{ + for (auto & i : tokenizeString(getEnv("NIX_SSHOPTS").value_or(""))) + args.push_back(i); + if (!keyFile.empty()) + args.insert(args.end(), {"-i", keyFile}); + if (compress) + args.push_back("-C"); +} + +std::unique_ptr SSHMaster::startCommand(const std::string & command) +{ + Path socketPath = startMaster(); + + Pipe in, out; + in.create(); + out.create(); + + auto conn = std::make_unique(); + ProcessOptions options; + options.dieWithParent = false; + + conn->sshPid = startProcess([&]() { + restoreSignals(); + + close(in.writeSide.get()); + close(out.readSide.get()); + + if (dup2(in.readSide.get(), STDIN_FILENO) == -1) + throw SysError("duping over stdin"); + if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("duping over stdout"); + if (logFD != -1 && dup2(logFD, STDERR_FILENO) == -1) + throw SysError("duping over stderr"); + + Strings args; + + if (fakeSSH) { + args = { "bash", "-c" }; + } else { + args = { "ssh", host.c_str(), "-x", "-a" }; + addCommonSSHOpts(args); + if (socketPath != "") + args.insert(args.end(), {"-S", socketPath}); + if (verbosity >= lvlChatty) + args.push_back("-v"); + } + + args.push_back(command); + execvp(args.begin()->c_str(), stringsToCharPtrs(args).data()); + + // could not exec ssh/bash + throw SysError("unable to execute '%s'", args.front()); + }, options); + + + in.readSide = -1; + out.writeSide = -1; + + conn->out = std::move(out.readSide); + conn->in = std::move(in.writeSide); + + return conn; +} + +Path SSHMaster::startMaster() +{ + if (!useMaster) return ""; + + auto state(state_.lock()); + + if (state->sshMaster != -1) return state->socketPath; + + state->tmpDir = std::make_unique(createTempDir("", "nix", true, true, 0700)); + + state->socketPath = (Path) *state->tmpDir + "/ssh.sock"; + + Pipe out; + out.create(); + + ProcessOptions options; + options.dieWithParent = false; + + state->sshMaster = startProcess([&]() { + restoreSignals(); + + close(out.readSide.get()); + + if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("duping over stdout"); + + Strings args = + { "ssh", host.c_str(), "-M", "-N", "-S", state->socketPath + , "-o", "LocalCommand=echo started" + , "-o", "PermitLocalCommand=yes" + }; + if (verbosity >= lvlChatty) + args.push_back("-v"); + addCommonSSHOpts(args); + execvp(args.begin()->c_str(), stringsToCharPtrs(args).data()); + + throw SysError("unable to execute '%s'", args.front()); + }, options); + + out.writeSide = -1; + + std::string reply; + try { + reply = readLine(out.readSide.get()); + } catch (EndOfFile & e) { } + + if (reply != "started") + throw Error("failed to start SSH master connection to '%s'", host); + + return state->socketPath; +} + +} diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh new file mode 100644 index 00000000000..4f0f0bd29f9 --- /dev/null +++ b/src/libstore/ssh.hh @@ -0,0 +1,45 @@ +#pragma once + +#include "util.hh" +#include "sync.hh" + +namespace nix { + +class SSHMaster +{ +private: + + const std::string host; + bool fakeSSH; + const std::string keyFile; + const bool useMaster; + const bool compress; + const int logFD; + + struct State + { + Pid sshMaster; + std::unique_ptr tmpDir; + Path socketPath; + }; + + Sync state_; + + void addCommonSSHOpts(Strings & args); + +public: + + SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD = -1); + + struct Connection + { + Pid sshPid; + AutoCloseFD out, in; + }; + + std::unique_ptr startCommand(const std::string & command); + + Path startMaster(); +}; + +} diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8fdd6277155..e23a9ca50f9 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -4,6 +4,9 @@ #include "util.hh" #include "nar-info-disk-cache.hh" #include "thread-pool.hh" +#include "json.hh" +#include "derivations.hh" +#include "url.hh" #include @@ -17,25 +20,10 @@ bool Store::isInStore(const Path & path) const } -bool Store::isStorePath(const Path & path) const -{ - return isInStore(path) - && path.size() >= storeDir.size() + 1 + storePathHashLen - && path.find('/', storeDir.size() + 1) == Path::npos; -} - - -void Store::assertStorePath(const Path & path) const -{ - if (!isStorePath(path)) - throw Error(format("path ‘%1%’ is not in the Nix store") % path); -} - - Path Store::toStorePath(const Path & path) const { if (!isInStore(path)) - throw Error(format("path ‘%1%’ is not in the Nix store") % path); + throw Error("path '%1%' is not in the Nix store", path); Path::size_type slash = path.find('/', storeDir.size() + 1); if (slash == Path::npos) return path; @@ -44,31 +32,30 @@ Path Store::toStorePath(const Path & path) const } -Path Store::followLinksToStore(const Path & _path) const +Path Store::followLinksToStore(std::string_view _path) const { - Path path = absPath(_path); + Path path = absPath(std::string(_path)); while (!isInStore(path)) { if (!isLink(path)) break; string target = readLink(path); path = absPath(target, dirOf(path)); } if (!isInStore(path)) - throw Error(format("path ‘%1%’ is not in the Nix store") % path); + throw NotInStore("path '%1%' is not in the Nix store", path); return path; } -Path Store::followLinksToStorePath(const Path & path) const +StorePath Store::followLinksToStorePath(std::string_view path) const { - return toStorePath(followLinksToStore(path)); + return parseStorePath(toStorePath(followLinksToStore(path))); } -string storePathToName(const Path & path) +StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const { - auto base = baseNameOf(path); - assert(base.size() == storePathHashLen || (base.size() > storePathHashLen && base[storePathHashLen] == '-')); - return base.size() == storePathHashLen ? "" : string(base, storePathHashLen + 1); + auto [path2, outputs] = nix::parsePathWithOutputs(path); + return StorePathWithOutputs(followLinksToStorePath(path2), std::move(outputs)); } @@ -80,25 +67,6 @@ string storePathToHash(const Path & path) } -void checkStoreName(const string & name) -{ - string validChars = "+-._?="; - /* Disallow names starting with a dot for possible security - reasons (e.g., "." and ".."). */ - if (string(name, 0, 1) == ".") - throw Error(format("illegal name: ‘%1%’") % name); - for (auto & i : name) - if (!((i >= 'A' && i <= 'Z') || - (i >= 'a' && i <= 'z') || - (i >= '0' && i <= '9') || - validChars.find(i) != string::npos)) - { - throw Error(format("invalid character ‘%1%’ in name ‘%2%’") - % i % name); - } -} - - /* Store paths have the following form: /- @@ -165,81 +133,96 @@ void checkStoreName(const string & name) collisions (for security). For instance, it shouldn't be feasible to come up with a derivation whose output path collides with the path for a copied source. The former would have a starting with - "output:out:", while the latter would have a <2> starting with + "output:out:", while the latter would have a starting with "source:". */ -Path Store::makeStorePath(const string & type, - const Hash & hash, const string & name) const +StorePath Store::makeStorePath(const string & type, + const Hash & hash, std::string_view name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ - string s = type + ":sha256:" + printHash(hash) + ":" - + storeDir + ":" + name; + string s = type + ":" + hash.to_string(Base16, true) + ":" + storeDir + ":" + std::string(name); + auto h = compressHash(hashString(htSHA256, s), 20); + return StorePath::make(h.hash, name); +} - checkStoreName(name); - return storeDir + "/" - + printHash32(compressHash(hashString(htSHA256, s), 20)) - + "-" + name; +StorePath Store::makeOutputPath(const string & id, + const Hash & hash, std::string_view name) const +{ + return makeStorePath("output:" + id, hash, + std::string(name) + (id == "out" ? "" : "-" + id)); } -Path Store::makeOutputPath(const string & id, - const Hash & hash, const string & name) const +static std::string makeType( + const Store & store, + string && type, + const StorePathSet & references, + bool hasSelfReference = false) { - return makeStorePath("output:" + id, hash, - name + (id == "out" ? "" : "-" + id)); + for (auto & i : references) { + type += ":"; + type += store.printStorePath(i); + } + if (hasSelfReference) type += ":self"; + return std::move(type); } -Path Store::makeFixedOutputPath(bool recursive, - const Hash & hash, const string & name) const +StorePath Store::makeFixedOutputPath( + FileIngestionMethod recursive, + const Hash & hash, + std::string_view name, + const StorePathSet & references, + bool hasSelfReference) const { - return hash.type == htSHA256 && recursive - ? makeStorePath("source", hash, name) - : makeStorePath("output:out", hashString(htSHA256, - "fixed:out:" + (recursive ? (string) "r:" : "") + - printHashType(hash.type) + ":" + printHash(hash) + ":"), + if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); + } else { + assert(references.empty()); + return makeStorePath("output:out", + hashString(htSHA256, + "fixed:out:" + + (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "") + + hash.to_string(Base16, true) + ":"), name); + } } -Path Store::makeTextPath(const string & name, const Hash & hash, - const PathSet & references) const +StorePath Store::makeTextPath(std::string_view name, const Hash & hash, + const StorePathSet & references) const { assert(hash.type == htSHA256); /* Stuff the references (if any) into the type. This is a bit hacky, but we can't put them in `s' since that would be ambiguous. */ - string type = "text"; - for (auto & i : references) { - type += ":"; - type += i; - } - return makeStorePath(type, hash, name); + return makeStorePath(makeType(*this, "text", references), hash, name); } -std::pair Store::computeStorePathForPath(const Path & srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter) const +std::pair Store::computeStorePathForPath(std::string_view name, + const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const { - Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - string name = baseNameOf(srcPath); - Path dstPath = makeFixedOutputPath(recursive, h, name); - return std::pair(dstPath, h); + Hash h = method == FileIngestionMethod::Recursive + ? hashPath(hashAlgo, srcPath, filter).first + : hashFile(hashAlgo, srcPath); + return std::make_pair(makeFixedOutputPath(method, h, name), h); } -Path Store::computeStorePathForText(const string & name, const string & s, - const PathSet & references) const +StorePath Store::computeStorePathForText(const string & name, const string & s, + const StorePathSet & references) const { return makeTextPath(name, hashString(htSHA256, s), references); } Store::Store(const Params & params) - : storeDir(get(params, "store", settings.nixStore)) + : Config(params) + , state({(size_t) pathInfoCacheSize}) { } @@ -249,17 +232,25 @@ std::string Store::getUri() return ""; } +bool Store::PathInfoCacheValue::isKnownNow() +{ + std::chrono::duration ttl = didExist() + ? std::chrono::seconds(settings.ttlPositiveNarInfoCache) + : std::chrono::seconds(settings.ttlNegativeNarInfoCache); + + return std::chrono::steady_clock::now() < time_point + ttl; +} -bool Store::isValidPath(const Path & storePath) +bool Store::isValidPath(const StorePath & storePath) { - auto hashPart = storePathToHash(storePath); + auto hashPart = storePathToHash(printStorePath(storePath)); { auto state_(state.lock()); auto res = state_->pathInfoCache.get(hashPart); - if (res) { + if (res && res->isKnownNow()) { stats.narInfoReadAverted++; - return *res != 0; + return res->didExist(); } } @@ -269,7 +260,7 @@ bool Store::isValidPath(const Path & storePath) stats.narInfoReadAverted++; auto state_(state.lock()); state_->pathInfoCache.upsert(hashPart, - res.first == NarInfoDiskCache::oInvalid ? 0 : res.second); + res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue { .value = res.second }); return res.first == NarInfoDiskCache::oValid; } } @@ -284,37 +275,51 @@ bool Store::isValidPath(const Path & storePath) } -ref Store::queryPathInfo(const Path & storePath) +/* Default implementation for stores that only implement + queryPathInfoUncached(). */ +bool Store::isValidPathUncached(const StorePath & path) { - std::promise> promise; + try { + queryPathInfo(path); + return true; + } catch (InvalidPath &) { + return false; + } +} + + +ref Store::queryPathInfo(const StorePath & storePath) +{ + std::promise> promise; queryPathInfo(storePath, - [&](ref info) { - promise.set_value(info); - }, - [&](std::exception_ptr exc) { - promise.set_exception(exc); - }); + {[&](std::future> result) { + try { + promise.set_value(result.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }}); return promise.get_future().get(); } -void Store::queryPathInfo(const Path & storePath, - std::function)> success, - std::function failure) +void Store::queryPathInfo(const StorePath & storePath, + Callback> callback) noexcept { - auto hashPart = storePathToHash(storePath); + std::string hashPart; try { + hashPart = storePathToHash(printStorePath(storePath)); { auto res = state.lock()->pathInfoCache.get(hashPart); - if (res) { + if (res && res->isKnownNow()) { stats.narInfoReadAverted++; - if (!*res) - throw InvalidPath(format("path ‘%s’ is not valid") % storePath); - return success(ref(*res)); + if (!res->didExist()) + throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); + return callback(ref(res->value)); } } @@ -325,83 +330,85 @@ void Store::queryPathInfo(const Path & storePath, { auto state_(state.lock()); state_->pathInfoCache.upsert(hashPart, - res.first == NarInfoDiskCache::oInvalid ? 0 : res.second); + res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second }); if (res.first == NarInfoDiskCache::oInvalid || - (res.second->path != storePath && storePathToName(storePath) != "")) - throw InvalidPath(format("path ‘%s’ is not valid") % storePath); + res.second->path != storePath) + throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); } - return success(ref(res.second)); + return callback(ref(res.second)); } } - } catch (std::exception & e) { - return callFailure(failure); - } + } catch (...) { return callback.rethrow(); } + + auto callbackPtr = std::make_shared(std::move(callback)); queryPathInfoUncached(storePath, - [this, storePath, hashPart, success, failure](std::shared_ptr info) { + {[this, storePath{printStorePath(storePath)}, hashPart, callbackPtr](std::future> fut) { - if (diskCache) - diskCache->upsertNarInfo(getUri(), hashPart, info); + try { + auto info = fut.get(); - { - auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, info); - } + if (diskCache) + diskCache->upsertNarInfo(getUri(), hashPart, info); - if (!info - || (info->path != storePath && storePathToName(storePath) != "")) - { - stats.narInfoMissing++; - return failure(std::make_exception_ptr(InvalidPath(format("path ‘%s’ is not valid") % storePath))); - } + { + auto state_(state.lock()); + state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info }); + } - callSuccess(success, failure, ref(info)); + if (!info || info->path != parseStorePath(storePath)) { + stats.narInfoMissing++; + throw InvalidPath("path '%s' is not valid", storePath); + } - }, failure); + (*callbackPtr)(ref(info)); + } catch (...) { callbackPtr->rethrow(); } + }}); } -PathSet Store::queryValidPaths(const PathSet & paths) +StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) { struct State { size_t left; - PathSet valid; + StorePathSet valid; std::exception_ptr exc; }; - Sync state_(State{paths.size(), PathSet()}); + Sync state_(State{paths.size(), StorePathSet()}); std::condition_variable wakeup; + ThreadPool pool; + + auto doQuery = [&](const Path & path) { + checkInterrupt(); + queryPathInfo(parseStorePath(path), {[path, this, &state_, &wakeup](std::future> fut) { + auto state(state_.lock()); + try { + auto info = fut.get(); + state->valid.insert(parseStorePath(path)); + } catch (InvalidPath &) { + } catch (...) { + state->exc = std::current_exception(); + } + assert(state->left); + if (!--state->left) + wakeup.notify_one(); + }}); + }; for (auto & path : paths) - queryPathInfo(path, - [path, &state_, &wakeup](ref info) { - auto state(state_.lock()); - state->valid.insert(path); - assert(state->left); - if (!--state->left) - wakeup.notify_one(); - }, - [path, &state_, &wakeup](std::exception_ptr exc) { - auto state(state_.lock()); - try { - std::rethrow_exception(exc); - } catch (InvalidPath &) { - } catch (...) { - state->exc = exc; - } - assert(state->left); - if (!--state->left) - wakeup.notify_one(); - }); + pool.enqueue(std::bind(doQuery, printStorePath(path))); // FIXME + + pool.process(); while (true) { auto state(state_.lock()); if (!state->left) { if (state->exc) std::rethrow_exception(state->exc); - return state->valid; + return std::move(state->valid); } state.wait(wakeup); } @@ -411,34 +418,124 @@ PathSet Store::queryValidPaths(const PathSet & paths) /* Return a string accepted by decodeValidPathInfo() that registers the specified paths as valid. Note: it's the responsibility of the caller to provide a closure. */ -string Store::makeValidityRegistration(const PathSet & paths, +string Store::makeValidityRegistration(const StorePathSet & paths, bool showDerivers, bool showHash) { string s = ""; for (auto & i : paths) { - s += i + "\n"; + s += printStorePath(i) + "\n"; auto info = queryPathInfo(i); if (showHash) { - s += printHash(info->narHash) + "\n"; + s += info->narHash.to_string(Base16, false) + "\n"; s += (format("%1%\n") % info->narSize).str(); } - Path deriver = showDerivers ? info->deriver : ""; + auto deriver = showDerivers && info->deriver ? printStorePath(*info->deriver) : ""; s += deriver + "\n"; s += (format("%1%\n") % info->references.size()).str(); for (auto & j : info->references) - s += j + "\n"; + s += printStorePath(j) + "\n"; } return s; } +void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, + bool includeImpureInfo, bool showClosureSize, + Base hashBase, + AllowInvalidFlag allowInvalid) +{ + auto jsonList = jsonOut.list(); + + for (auto & storePath : storePaths) { + auto jsonPath = jsonList.object(); + jsonPath.attr("path", printStorePath(storePath)); + + try { + auto info = queryPathInfo(storePath); + + jsonPath + .attr("narHash", info->narHash.to_string(hashBase, true)) + .attr("narSize", info->narSize); + + { + auto jsonRefs = jsonPath.list("references"); + for (auto & ref : info->references) + jsonRefs.elem(printStorePath(ref)); + } + + if (info->ca != "") + jsonPath.attr("ca", info->ca); + + std::pair closureSizes; + + if (showClosureSize) { + closureSizes = getClosureSize(info->path); + jsonPath.attr("closureSize", closureSizes.first); + } + + if (includeImpureInfo) { + + if (info->deriver) + jsonPath.attr("deriver", printStorePath(*info->deriver)); + + if (info->registrationTime) + jsonPath.attr("registrationTime", info->registrationTime); + + if (info->ultimate) + jsonPath.attr("ultimate", info->ultimate); + + if (!info->sigs.empty()) { + auto jsonSigs = jsonPath.list("signatures"); + for (auto & sig : info->sigs) + jsonSigs.elem(sig); + } + + auto narInfo = std::dynamic_pointer_cast( + std::shared_ptr(info)); + + if (narInfo) { + if (!narInfo->url.empty()) + jsonPath.attr("url", narInfo->url); + if (narInfo->fileHash) + jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true)); + if (narInfo->fileSize) + jsonPath.attr("downloadSize", narInfo->fileSize); + if (showClosureSize) + jsonPath.attr("closureDownloadSize", closureSizes.second); + } + } + + } catch (InvalidPath &) { + jsonPath.attr("valid", false); + } + } +} + + +std::pair Store::getClosureSize(const StorePath & storePath) +{ + uint64_t totalNarSize = 0, totalDownloadSize = 0; + StorePathSet closure; + computeFSClosure(storePath, closure, false, false); + for (auto & p : closure) { + auto info = queryPathInfo(p); + totalNarSize += info->narSize; + auto narInfo = std::dynamic_pointer_cast( + std::shared_ptr(info)); + if (narInfo) + totalDownloadSize += narInfo->fileSize; + } + return {totalNarSize, totalDownloadSize}; +} + + const Store::Stats & Store::getStats() { { @@ -449,115 +546,260 @@ const Store::Stats & Store::getStats() } +void Store::buildPaths(const std::vector & paths, BuildMode buildMode) +{ + StorePathSet paths2; + + for (auto & path : paths) { + if (path.path.isDerivation()) + unsupported("buildPaths"); + paths2.insert(path.path.clone()); + } + + if (queryValidPaths(paths2).size() != paths2.size()) + unsupported("buildPaths"); +} + + void copyStorePath(ref srcStore, ref dstStore, - const Path & storePath, bool repair, bool dontCheckSigs) + const StorePath & storePath, RepairFlag repair, CheckSigsFlag checkSigs) { + auto srcUri = srcStore->getUri(); + auto dstUri = dstStore->getUri(); + + Activity act(*logger, lvlInfo, actCopyPath, + srcUri == "local" || srcUri == "daemon" + ? fmt("copying path '%s' to '%s'", srcStore->printStorePath(storePath), dstUri) + : dstUri == "local" || dstUri == "daemon" + ? fmt("copying path '%s' from '%s'", srcStore->printStorePath(storePath), srcUri) + : fmt("copying path '%s' from '%s' to '%s'", srcStore->printStorePath(storePath), srcUri, dstUri), + {srcStore->printStorePath(storePath), srcUri, dstUri}); + PushActivity pact(act.id); + auto info = srcStore->queryPathInfo(storePath); - StringSink sink; - srcStore->narFromPath({storePath}, sink); + uint64_t total = 0; + + if (!info->narHash) { + StringSink sink; + srcStore->narFromPath({storePath}, sink); + auto info2 = make_ref(*info); + info2->narHash = hashString(htSHA256, *sink.s); + if (!info->narSize) info2->narSize = sink.s->size(); + if (info->ultimate) info2->ultimate = false; + info = info2; + + StringSource source(*sink.s); + dstStore->addToStore(*info, source, repair, checkSigs); + return; + } + + if (info->ultimate) { + auto info2 = make_ref(*info); + info2->ultimate = false; + info = info2; + } - dstStore->addToStore(*info, sink.s, repair, dontCheckSigs); + auto source = sinkToSource([&](Sink & sink) { + LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { + sink(data, len); + total += len; + act.progress(total, info->narSize); + }); + srcStore->narFromPath(storePath, wrapperSink); + }, [&]() { + throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore->printStorePath(storePath), srcStore->getUri()); + }); + + dstStore->addToStore(*info, *source, repair, checkSigs); } -void copyClosure(ref srcStore, ref dstStore, - const PathSet & storePaths, bool repair, bool dontCheckSigs) +void copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, + RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { - PathSet closure; + auto valid = dstStore->queryValidPaths(storePaths, substitute); + + PathSet missing; for (auto & path : storePaths) - srcStore->computeFSClosure(path, closure); + if (!valid.count(path)) missing.insert(srcStore->printStorePath(path)); - PathSet valid = dstStore->queryValidPaths(closure); + if (missing.empty()) return; - if (valid.size() == closure.size()) return; + Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); - Paths sorted = srcStore->topoSortPaths(closure); + std::atomic nrDone{0}; + std::atomic nrFailed{0}; + std::atomic bytesExpected{0}; + std::atomic nrRunning{0}; - Paths missing; - for (auto i = sorted.rbegin(); i != sorted.rend(); ++i) - if (!valid.count(*i)) missing.push_back(*i); + auto showProgress = [&]() { + act.progress(nrDone, missing.size(), nrRunning, nrFailed); + }; - printMsg(lvlDebug, format("copying %1% missing paths") % missing.size()); + ThreadPool pool; + + processGraph(pool, + PathSet(missing.begin(), missing.end()), + + [&](const Path & storePath) { + if (dstStore->isValidPath(dstStore->parseStorePath(storePath))) { + nrDone++; + showProgress(); + return PathSet(); + } - for (auto & i : missing) - copyStorePath(srcStore, dstStore, i, repair, dontCheckSigs); + auto info = srcStore->queryPathInfo(srcStore->parseStorePath(storePath)); + + bytesExpected += info->narSize; + act.setExpected(actCopyPath, bytesExpected); + + return srcStore->printStorePathSet(info->references); + }, + + [&](const Path & storePathS) { + checkInterrupt(); + + auto storePath = dstStore->parseStorePath(storePathS); + + if (!dstStore->isValidPath(storePath)) { + MaintainCount mc(nrRunning); + showProgress(); + try { + copyStorePath(srcStore, dstStore, storePath, repair, checkSigs); + } catch (Error &e) { + nrFailed++; + if (!settings.keepGoing) + throw e; + logger->log(lvlError, fmt("could not copy %s: %s", storePathS, e.what())); + showProgress(); + return; + } + } + + nrDone++; + showProgress(); + }); +} + + +void copyClosure(ref srcStore, ref dstStore, + const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, + SubstituteFlag substitute) +{ + StorePathSet closure; + srcStore->computeFSClosure(storePaths, closure); + copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); } -ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) +ValidPathInfo::ValidPathInfo(const ValidPathInfo & other) + : path(other.path.clone()) + , deriver(other.deriver ? other.deriver->clone(): std::optional{}) + , narHash(other.narHash) + , references(cloneStorePathSet(other.references)) + , registrationTime(other.registrationTime) + , narSize(other.narSize) + , id(other.id) + , ultimate(other.ultimate) + , sigs(other.sigs) + , ca(other.ca) { - ValidPathInfo info; - getline(str, info.path); - if (str.eof()) { info.path = ""; return info; } +} + + +std::optional decodeValidPathInfo(const Store & store, std::istream & str, bool hashGiven) +{ + std::string path; + getline(str, path); + if (str.eof()) { return {}; } + ValidPathInfo info(store.parseStorePath(path)); if (hashGiven) { string s; getline(str, s); - info.narHash = parseHash(htSHA256, s); + info.narHash = Hash(s, htSHA256); getline(str, s); if (!string2Int(s, info.narSize)) throw Error("number expected"); } - getline(str, info.deriver); + std::string deriver; + getline(str, deriver); + if (deriver != "") info.deriver = store.parseStorePath(deriver); string s; int n; getline(str, s); if (!string2Int(s, n)) throw Error("number expected"); while (n--) { getline(str, s); - info.references.insert(s); + info.references.insert(store.parseStorePath(s)); } if (!str || str.eof()) throw Error("missing input"); - return info; + return std::optional(std::move(info)); } -string showPaths(const PathSet & paths) +std::string Store::showPaths(const StorePathSet & paths) { - string s; + std::string s; for (auto & i : paths) { if (s.size() != 0) s += ", "; - s += "‘" + i + "’"; + s += "'" + printStorePath(i) + "'"; } return s; } -std::string ValidPathInfo::fingerprint() const +string showPaths(const PathSet & paths) +{ + return concatStringsSep(", ", quoteStrings(paths)); +} + + +std::string ValidPathInfo::fingerprint(const Store & store) const { if (narSize == 0 || !narHash) - throw Error(format("cannot calculate fingerprint of path ‘%s’ because its size/hash is not known") - % path); + throw Error("cannot calculate fingerprint of path '%s' because its size/hash is not known", + store.printStorePath(path)); return - "1;" + path + ";" - + printHashType(narHash.type) + ":" + printHash32(narHash) + ";" + "1;" + store.printStorePath(path) + ";" + + narHash.to_string(Base32, true) + ";" + std::to_string(narSize) + ";" - + concatStringsSep(",", references); + + concatStringsSep(",", store.printStorePathSet(references)); } -void ValidPathInfo::sign(const SecretKey & secretKey) +void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) { - sigs.insert(secretKey.signDetached(fingerprint())); + sigs.insert(secretKey.signDetached(fingerprint(store))); } bool ValidPathInfo::isContentAddressed(const Store & store) const { auto warn = [&]() { - printError(format("warning: path ‘%s’ claims to be content-addressed but isn't") % path); + logWarning( + ErrorInfo{ + .name = "Path not content-addressed", + .hint = hintfmt("path '%s' claims to be content-addressed but isn't", store.printStorePath(path)) + }); }; if (hasPrefix(ca, "text:")) { - auto hash = parseHash(std::string(ca, 5)); - if (store.makeTextPath(storePathToName(path), hash, references) == path) + Hash hash(ca.substr(5)); + if (store.makeTextPath(path.name(), hash, references) == path) return true; else warn(); } else if (hasPrefix(ca, "fixed:")) { - bool recursive = ca.compare(6, 2, "r:") == 0; - auto hash = parseHash(std::string(ca, recursive ? 8 : 6)); - if (store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path) + FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 }; + Hash hash(ca.substr(recursive == FileIngestionMethod::Recursive ? 8 : 6)); + auto refs = cloneStorePathSet(references); + bool hasSelfReference = false; + if (refs.count(path)) { + hasSelfReference = true; + refs.erase(path); + } + if (store.makeFixedOutputPath(recursive, hash, path.name(), refs, hasSelfReference) == path) return true; else warn(); @@ -573,15 +815,15 @@ size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & pu size_t good = 0; for (auto & sig : sigs) - if (checkSignature(publicKeys, sig)) + if (checkSignature(store, publicKeys, sig)) good++; return good; } -bool ValidPathInfo::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const +bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const { - return verifyDetached(fingerprint(), sig, publicKeys); + return verifyDetached(fingerprint(store), sig, publicKeys); } @@ -589,11 +831,19 @@ Strings ValidPathInfo::shortRefs() const { Strings refs; for (auto & r : references) - refs.push_back(baseNameOf(r)); + refs.push_back(std::string(r.to_string())); return refs; } +std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash) +{ + return "fixed:" + + (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "") + + hash.to_string(Base32, true); +} + + } @@ -606,27 +856,35 @@ namespace nix { RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; - -ref openStore(const std::string & uri_) +/* Split URI into protocol+hierarchy part and its parameter set. */ +std::pair splitUriAndParams(const std::string & uri_) { auto uri(uri_); Store::Params params; auto q = uri.find('?'); if (q != std::string::npos) { - for (auto s : tokenizeString(uri.substr(q + 1), "&")) { - auto e = s.find('='); - if (e != std::string::npos) - params[s.substr(0, e)] = s.substr(e + 1); - } + params = decodeQuery(uri.substr(q + 1)); uri = uri_.substr(0, q); } + return {uri, params}; +} + +ref openStore(const std::string & uri_, + const Store::Params & extraParams) +{ + auto [uri, uriParams] = splitUriAndParams(uri_); + auto params = extraParams; + params.insert(uriParams.begin(), uriParams.end()); for (auto fun : *RegisterStoreImplementation::implementations) { auto store = fun(uri, params); - if (store) return ref(store); + if (store) { + store->warnUnknownSettings(); + return ref(store); + } } - throw Error(format("don't know how to open Nix store ‘%s’") % uri); + throw Error("don't know how to open Nix store '%s'", uri); } @@ -634,9 +892,9 @@ StoreType getStoreType(const std::string & uri, const std::string & stateDir) { if (uri == "daemon") { return tDaemon; - } else if (uri == "local") { + } else if (uri == "local" || hasPrefix(uri, "/")) { return tLocal; - } else if (uri == "") { + } else if (uri == "" || uri == "auto") { if (access(stateDir.c_str(), R_OK | W_OK) == 0) return tLocal; else if (pathExists(settings.nixDaemonSocketFile)) @@ -653,11 +911,15 @@ static RegisterStoreImplementation regStore([]( const std::string & uri, const Store::Params & params) -> std::shared_ptr { - switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) { + switch (getStoreType(uri, get(params, "state").value_or(settings.nixStateDir))) { case tDaemon: return std::shared_ptr(std::make_shared(params)); - case tLocal: - return std::shared_ptr(std::make_shared(params)); + case tLocal: { + Store::Params params2 = params; + if (hasPrefix(uri, "/")) + params2["root"] = uri; + return std::shared_ptr(std::make_shared(params2)); + } default: return nullptr; } @@ -666,68 +928,34 @@ static RegisterStoreImplementation regStore([]( std::list> getDefaultSubstituters() { - struct State { - bool done = false; + static auto stores([]() { std::list> stores; - }; - static Sync state_; - - auto state(state_.lock()); - - if (state->done) return state->stores; - StringSet done; - - auto addStore = [&](const std::string & uri) { - if (done.count(uri)) return; - done.insert(uri); - state->stores.push_back(openStore(uri)); - }; + StringSet done; - for (auto uri : settings.get("substituters", Strings())) - addStore(uri); - - for (auto uri : settings.get("binary-caches", Strings())) - addStore(uri); - - for (auto uri : settings.get("extra-binary-caches", Strings())) - addStore(uri); - - state->done = true; - - return state->stores; -} - - -void copyPaths(ref from, ref to, const Paths & storePaths) -{ - std::string copiedLabel = "copied"; - - logger->setExpected(copiedLabel, storePaths.size()); - - ThreadPool pool; - - processGraph(pool, - PathSet(storePaths.begin(), storePaths.end()), - - [&](const Path & storePath) { - return from->queryPathInfo(storePath)->references; - }, - - [&](const Path & storePath) { - checkInterrupt(); + auto addStore = [&](const std::string & uri) { + if (!done.insert(uri).second) return; + try { + stores.push_back(openStore(uri)); + } catch (Error & e) { + logWarning(e.info()); + } + }; - if (!to->isValidPath(storePath)) { - Activity act(*logger, lvlInfo, format("copying ‘%s’...") % storePath); + for (auto uri : settings.substituters.get()) + addStore(uri); - copyStorePath(from, to, storePath); + for (auto uri : settings.extraSubstituters.get()) + addStore(uri); - logger->incProgress(copiedLabel); - } else - logger->incExpected(copiedLabel, -1); + stores.sort([](ref & a, ref & b) { + return a->priority < b->priority; }); - pool.process(); + return stores; + } ()); + + return stores; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index ec3bf5a6fd8..5ef5063266c 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,37 +1,54 @@ #pragma once +#include "path.hh" #include "hash.hh" #include "serialise.hh" #include "crypto.hh" #include "lru-cache.hh" #include "sync.hh" #include "globals.hh" +#include "config.hh" #include #include #include +#include +#include #include #include +#include namespace nix { +MakeError(SubstError, Error); +MakeError(BuildError, Error); // denotes a permanent build failure +MakeError(InvalidPath, Error); +MakeError(Unsupported, Error); +MakeError(SubstituteGone, Error); +MakeError(SubstituterDisabled, Error); +MakeError(NotInStore, Error); + + struct BasicDerivation; struct Derivation; class FSAccessor; class NarInfoDiskCache; class Store; +class JSONPlaceholder; -/* Size of the hash part of store paths, in base-32 characters. */ -const size_t storePathHashLen = 32; // i.e. 160 bits +enum RepairFlag : bool { NoRepair = false, Repair = true }; +enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; +enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; +enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; /* Magic header of exportPath() output (obsolete). */ const uint32_t exportMagic = 0x4558494e; -typedef std::map Roots; +typedef std::unordered_map> Roots; struct GCOptions @@ -65,7 +82,7 @@ struct GCOptions bool ignoreLiveness{false}; /* For `gcDeleteSpecific', the paths to delete. */ - PathSet pathsToDelete; + StorePathSet pathsToDelete; /* Stop after at least `maxFreed' bytes have been freed. */ unsigned long long maxFreed{std::numeric_limits::max()}; @@ -80,39 +97,33 @@ struct GCResults /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the number of bytes that would be or was freed. */ - unsigned long long bytesFreed; - - GCResults() - { - bytesFreed = 0; - } + unsigned long long bytesFreed = 0; }; struct SubstitutablePathInfo { - Path deriver; - PathSet references; + std::optional deriver; + StorePathSet references; unsigned long long downloadSize; /* 0 = unknown or inapplicable */ unsigned long long narSize; /* 0 = unknown */ }; -typedef std::map SubstitutablePathInfos; +typedef std::map SubstitutablePathInfos; struct ValidPathInfo { - Path path; - Path deriver; + StorePath path; + std::optional deriver; Hash narHash; - PathSet references; + StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only - /* Whether the path is ultimately trusted, that is, it was built - locally or is content-addressable (e.g. added via addToStore() - or the result of a fixed-output derivation). */ + /* Whether the path is ultimately trusted, that is, it's a + derivation output that was built locally. */ bool ultimate = false; StringSet sigs; // note: not necessarily verified @@ -127,11 +138,11 @@ struct ValidPathInfo of an output path of a derivation were actually produced by that derivation. In the intensional model, we have to trust that a particular output path was produced by a derivation; the - path name then implies the contents.) + path then implies the contents.) Ideally, the content-addressability assertion would just be a Boolean, and the store path would be computed from - ‘storePathToName(path)’, ‘narHash’ and ‘references’. However, + the name component, ‘narHash’ and ‘references’. However, 1) we've accumulated several types of content-addressed paths over the years; and 2) fixed-output derivations support multiple hash algorithms and serialisation methods (flat file @@ -159,9 +170,9 @@ struct ValidPathInfo the NAR, and the sorted references. The size field is strictly speaking superfluous, but might prevent endless/excessive data attacks. */ - std::string fingerprint() const; + std::string fingerprint(const Store & store) const; - void sign(const SecretKey & secretKey); + void sign(const Store & store, const SecretKey & secretKey); /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; @@ -174,17 +185,20 @@ struct ValidPathInfo size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const; /* Verify a single signature. */ - bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const; + bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const; Strings shortRefs() const; + ValidPathInfo(StorePath && path) : path(std::move(path)) { } + explicit ValidPathInfo(const ValidPathInfo & other); + virtual ~ValidPathInfo() { } }; typedef list ValidPathInfos; -enum BuildMode { bmNormal, bmRepair, bmCheck, bmHash }; +enum BuildMode { bmNormal, bmRepair, bmCheck }; struct BuildResult @@ -228,19 +242,48 @@ struct BuildResult }; -class Store : public std::enable_shared_from_this +class Store : public std::enable_shared_from_this, public Config { public: typedef std::map Params; - const Path storeDir; + const PathSetting storeDir_{this, false, settings.nixStore, + "store", "path to the Nix store"}; + const Path storeDir = storeDir_; + + const Setting pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"}; + + const Setting isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"}; + + Setting priority{this, 0, "priority", "priority of this substituter (lower value means higher priority)"}; + + Setting wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"}; protected: + struct PathInfoCacheValue { + + // Time of cache entry creation or update + std::chrono::time_point time_point = std::chrono::steady_clock::now(); + + // Null if missing + std::shared_ptr value; + + // Whether the value is valid as a cache entry. The path may not exist. + bool isKnownNow(); + + // Past tense, because a path can only be assumed to exists when + // isKnownNow() && didExist() + inline bool didExist() { + return value != nullptr; + } + }; + struct State { - LRUCache> pathInfoCache{64 * 1024}; + // FIXME: fix key + LRUCache pathInfoCache; }; Sync state; @@ -255,47 +298,68 @@ public: virtual std::string getUri() = 0; + StorePath parseStorePath(std::string_view path) const; + + std::optional maybeParseStorePath(std::string_view path) const; + + std::string printStorePath(const StorePath & path) const; + + // FIXME: remove + StorePathSet parseStorePathSet(const PathSet & paths) const; + + PathSet printStorePathSet(const StorePathSet & path) const; + + /* Split a string specifying a derivation and a set of outputs + (/nix/store/hash-foo!out1,out2,...) into the derivation path + and the outputs. */ + StorePathWithOutputs parsePathWithOutputs(const string & s); + + /* Display a set of paths in human-readable form (i.e., between quotes + and separated by commas). */ + std::string showPaths(const StorePathSet & paths); + /* Return true if ‘path’ is in the Nix store (but not the Nix store itself). */ bool isInStore(const Path & path) const; /* Return true if ‘path’ is a store path, i.e. a direct child of the Nix store. */ - bool isStorePath(const Path & path) const; - - /* Throw an exception if ‘path’ is not a store path. */ - void assertStorePath(const Path & path) const; + bool isStorePath(std::string_view path) const; /* Chop off the parts after the top-level store name, e.g., /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */ Path toStorePath(const Path & path) const; /* Follow symlinks until we end up with a path in the Nix store. */ - Path followLinksToStore(const Path & path) const; + Path followLinksToStore(std::string_view path) const; /* Same as followLinksToStore(), but apply toStorePath() to the result. */ - Path followLinksToStorePath(const Path & path) const; + StorePath followLinksToStorePath(std::string_view path) const; + + StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const; /* Constructs a unique store path name. */ - Path makeStorePath(const string & type, - const Hash & hash, const string & name) const; + StorePath makeStorePath(const string & type, + const Hash & hash, std::string_view name) const; - Path makeOutputPath(const string & id, - const Hash & hash, const string & name) const; + StorePath makeOutputPath(const string & id, + const Hash & hash, std::string_view name) const; - Path makeFixedOutputPath(bool recursive, - const Hash & hash, const string & name) const; + StorePath makeFixedOutputPath(FileIngestionMethod method, + const Hash & hash, std::string_view name, + const StorePathSet & references = {}, + bool hasSelfReference = false) const; - Path makeTextPath(const string & name, const Hash & hash, - const PathSet & references) const; + StorePath makeTextPath(std::string_view name, const Hash & hash, + const StorePathSet & references) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store path and the cryptographic hash of the contents of srcPath. */ - std::pair computeStorePathForPath(const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter) const; + std::pair computeStorePathForPath(std::string_view name, + const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, + HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; /* Preparatory part of addTextToStore(). @@ -311,97 +375,101 @@ public: simply yield a different store path, so other users wouldn't be affected), but it has some backwards compatibility issues (the hashing scheme changes), so I'm not doing that for now. */ - Path computeStorePathForText(const string & name, const string & s, - const PathSet & references) const; + StorePath computeStorePathForText(const string & name, const string & s, + const StorePathSet & references) const; /* Check whether a path is valid. */ - bool isValidPath(const Path & path); + bool isValidPath(const StorePath & path); protected: - virtual bool isValidPathUncached(const Path & path) = 0; + virtual bool isValidPathUncached(const StorePath & path); public: - /* Query which of the given paths is valid. */ - virtual PathSet queryValidPaths(const PathSet & paths); + /* Query which of the given paths is valid. Optionally, try to + substitute missing paths. */ + virtual StorePathSet queryValidPaths(const StorePathSet & paths, + SubstituteFlag maybeSubstitute = NoSubstitute); /* Query the set of all valid paths. Note that for some store backends, the name part of store paths may be omitted (i.e. you'll get /nix/store/ rather than /nix/store/-). Use queryPathInfo() to obtain the full store path. */ - virtual PathSet queryAllValidPaths() = 0; + virtual StorePathSet queryAllValidPaths() + { unsupported("queryAllValidPaths"); } /* Query information about a valid path. It is permitted to omit the name part of the store path. */ - ref queryPathInfo(const Path & path); + ref queryPathInfo(const StorePath & path); /* Asynchronous version of queryPathInfo(). */ - void queryPathInfo(const Path & path, - std::function)> success, - std::function failure); + void queryPathInfo(const StorePath & path, + Callback> callback) noexcept; protected: - virtual void queryPathInfoUncached(const Path & path, - std::function)> success, - std::function failure) = 0; + virtual void queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept = 0; public: /* Queries the set of incoming FS references for a store path. The result is not cleared. */ - virtual void queryReferrers(const Path & path, - PathSet & referrers) = 0; + virtual void queryReferrers(const StorePath & path, StorePathSet & referrers) + { unsupported("queryReferrers"); } /* Return all currently valid derivations that have `path' as an output. (Note that the result of `queryDeriver()' is the derivation that was actually used to produce `path', which may not exist anymore.) */ - virtual PathSet queryValidDerivers(const Path & path) = 0; + virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; }; /* Query the outputs of the derivation denoted by `path'. */ - virtual PathSet queryDerivationOutputs(const Path & path) = 0; - - /* Query the output names of the derivation denoted by `path'. */ - virtual StringSet queryDerivationOutputNames(const Path & path) = 0; + virtual StorePathSet queryDerivationOutputs(const StorePath & path) + { unsupported("queryDerivationOutputs"); } /* Query the full store path given the hash part of a valid store - path, or "" if the path doesn't exist. */ - virtual Path queryPathFromHashPart(const string & hashPart) = 0; + path, or empty if the path doesn't exist. */ + virtual std::optional queryPathFromHashPart(const std::string & hashPart) = 0; /* Query which of the given paths have substitutes. */ - virtual PathSet querySubstitutablePaths(const PathSet & paths) = 0; + virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; }; /* Query substitute info (i.e. references, derivers and download sizes) of a set of paths. If a path does not have substitute info, it's omitted from the resulting ‘infos’ map. */ - virtual void querySubstitutablePathInfos(const PathSet & paths, - SubstitutablePathInfos & infos) = 0; - - virtual bool wantMassQuery() { return false; } + virtual void querySubstitutablePathInfos(const StorePathSet & paths, + SubstitutablePathInfos & infos) { return; }; /* Import a path into the store. */ - virtual void addToStore(const ValidPathInfo & info, const ref & nar, - bool repair = false, bool dontCheckSigs = false, + virtual void addToStore(const ValidPathInfo & info, Source & narSource, + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, std::shared_ptr accessor = 0) = 0; /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. The function object `filter' can be used to exclude files (see libutil/archive.hh). */ - virtual Path addToStore(const string & name, const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, bool repair = false) = 0; + virtual StorePath addToStore(const string & name, const Path & srcPath, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; + + // FIXME: remove? + virtual StorePath addToStoreFromDump(const string & dump, const string & name, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) + { + throw Error("addToStoreFromDump() is not supported by this store"); + } /* Like addToStore, but the contents written to the output path is a regular file containing the given string. */ - virtual Path addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair = false) = 0; + virtual StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair = NoRepair) = 0; /* Write a NAR dump of a store path. */ - virtual void narFromPath(const Path & path, Sink & sink) = 0; + virtual void narFromPath(const StorePath & path, Sink & sink) = 0; /* For each path, if it's a derivation, build it. Building a derivation means ensuring that the output paths are valid. If @@ -411,29 +479,33 @@ public: output paths can be created by running the builder, after recursively building any sub-derivations. For inputs that are not derivations, substitute them. */ - virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) = 0; + virtual void buildPaths( + const std::vector & paths, + BuildMode buildMode = bmNormal); /* Build a single non-materialized derivation (i.e. not from an on-disk .drv file). Note that ‘drvPath’ is only used for informational purposes. */ - virtual BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal) = 0; /* Ensure that a path is valid. If it is not currently valid, it may be made valid by running a substitute (if defined for the path). */ - virtual void ensurePath(const Path & path) = 0; + virtual void ensurePath(const StorePath & path) = 0; /* Add a store path as a temporary root of the garbage collector. The root disappears as soon as we exit. */ - virtual void addTempRoot(const Path & path) = 0; + virtual void addTempRoot(const StorePath & path) + { unsupported("addTempRoot"); } /* Add an indirect root, which is merely a symlink to `path' from /nix/var/nix/gcroots/auto/. `path' is supposed to be a symlink to a store path. The garbage collector will automatically remove the indirect root when it finds that `path' has disappeared. */ - virtual void addIndirectRoot(const Path & path) = 0; + virtual void addIndirectRoot(const Path & path) + { unsupported("addIndirectRoot"); } /* Acquire the global GC lock, then immediately release it. This function must be called after registering a new permanent root, @@ -453,42 +525,66 @@ public: permanent root and sees our's. In either case the permanent root is seen by the collector. */ - virtual void syncWithGC() = 0; + virtual void syncWithGC() { }; /* Find the roots of the garbage collector. Each root is a pair (link, storepath) where `link' is the path of the symlink - outside of the Nix store that point to `storePath'. */ - virtual Roots findRoots() = 0; + outside of the Nix store that point to `storePath'. If + 'censor' is true, privacy-sensitive information about roots + found in /proc is censored. */ + virtual Roots findRoots(bool censor) + { unsupported("findRoots"); } /* Perform a garbage collection. */ - virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; + virtual void collectGarbage(const GCOptions & options, GCResults & results) + { unsupported("collectGarbage"); } /* Return a string representing information about the path that can be loaded into the database using `nix-store --load-db' or `nix-store --register-validity'. */ - string makeValidityRegistration(const PathSet & paths, + string makeValidityRegistration(const StorePathSet & paths, bool showDerivers, bool showHash); + /* Write a JSON representation of store path metadata, such as the + hash and the references. If ‘includeImpureInfo’ is true, + variable elements such as the registration time are + included. If ‘showClosureSize’ is true, the closure size of + each path is included. */ + void pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, + bool includeImpureInfo, bool showClosureSize, + Base hashBase = Base32, + AllowInvalidFlag allowInvalid = DisallowInvalid); + + /* Return the size of the closure of the specified path, that is, + the sum of the size of the NAR serialisation of each path in + the closure. */ + std::pair getClosureSize(const StorePath & storePath); + /* Optimise the disk space usage of the Nix store by hard-linking files with the same contents. */ - virtual void optimiseStore() = 0; + virtual void optimiseStore() { }; /* Check the integrity of the Nix store. Returns true if errors remain. */ - virtual bool verifyStore(bool checkContents, bool repair) = 0; + virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) { return false; }; /* Return an object to access files in the Nix store. */ - virtual ref getFSAccessor() = 0; + virtual ref getFSAccessor() + { unsupported("getFSAccessor"); } /* Add signatures to the specified store path. The signatures are not verified. */ - virtual void addSignatures(const Path & storePath, const StringSet & sigs) = 0; + virtual void addSignatures(const StorePath & storePath, const StringSet & sigs) + { unsupported("addSignatures"); } /* Utility functions. */ /* Read a derivation, after ensuring its existence through ensurePath(). */ - Derivation derivationFromPath(const Path & drvPath); + Derivation derivationFromPath(const StorePath & drvPath); + + /* Read a derivation (which must already be valid). */ + Derivation readDerivation(const StorePath & drvPath); /* Place in `out' the set of all store paths in the file system closure of `storePath'; that is, all paths than can be directly @@ -497,37 +593,37 @@ public: `storePath' is returned; that is, the closures under the `referrers' relation instead of the `references' relation is returned. */ - void computeFSClosure(const PathSet & paths, - PathSet & out, bool flipDirection = false, + virtual void computeFSClosure(const StorePathSet & paths, + StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false); - void computeFSClosure(const Path & path, - PathSet & out, bool flipDirection = false, + void computeFSClosure(const StorePath & path, + StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false); /* Given a set of paths that are to be built, return the set of derivations that will be built, and the set of output paths that will be substituted. */ - void queryMissing(const PathSet & targets, - PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, + virtual void queryMissing(const std::vector & targets, + StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, unsigned long long & downloadSize, unsigned long long & narSize); /* Sort a set of paths topologically under the references - relation. If p refers to q, then p preceeds q in this list. */ - Paths topoSortPaths(const PathSet & paths); + relation. If p refers to q, then p precedes q in this list. */ + StorePaths topoSortPaths(const StorePathSet & paths); /* Export multiple paths in the format expected by ‘nix-store --import’. */ - void exportPaths(const Paths & paths, Sink & sink); + void exportPaths(const StorePathSet & paths, Sink & sink); - void exportPath(const Path & path, Sink & sink); + void exportPath(const StorePath & path, Sink & sink); /* Import a sequence of NAR dumps created by exportPaths() into the Nix store. Optionally, the contents of the NARs are preloaded into the specified FS accessor to speed up subsequent access. */ - Paths importPaths(Source & source, std::shared_ptr accessor, - bool dontCheckSigs = false); + StorePaths importPaths(Source & source, std::shared_ptr accessor, + CheckSigsFlag checkSigs = CheckSigs); struct Stats { @@ -548,58 +644,119 @@ public: const Stats & getStats(); + /* Return the build log of the specified store path, if available, + or null otherwise. */ + virtual std::shared_ptr getBuildLog(const StorePath & path) + { return nullptr; } + + /* Hack to allow long-running processes like hydra-queue-runner to + occasionally flush their path info cache. */ + void clearPathInfoCache() + { + state.lock()->pathInfoCache.clear(); + } + + /* Establish a connection to the store, for store types that have + a notion of connection. Otherwise this is a no-op. */ + virtual void connect() { }; + + /* Get the protocol version of this store or it's connection. */ + virtual unsigned int getProtocol() + { + return 0; + }; + + virtual Path toRealPath(const Path & storePath) + { + return storePath; + } + + Path toRealPath(const StorePath & storePath) + { + return toRealPath(printStorePath(storePath)); + } + + virtual void createUser(const std::string & userName, uid_t userId) + { } + protected: Stats stats; + /* Unsupported methods. */ + [[noreturn]] void unsupported(const std::string & op) + { + throw Unsupported("operation '%s' is not supported by store '%s'", op, getUri()); + } + }; class LocalFSStore : public virtual Store { public: - const Path rootDir; - const Path stateDir; - const Path logDir; + + // FIXME: the (Store*) cast works around a bug in gcc that causes + // it to omit the call to the Setting constructor. Clang works fine + // either way. + const PathSetting rootDir{(Store*) this, true, "", + "root", "directory prefixed to all other paths"}; + const PathSetting stateDir{(Store*) this, false, + rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, + "state", "directory where Nix will store state"}; + const PathSetting logDir{(Store*) this, false, + rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, + "log", "directory where Nix will store state"}; + + const static string drvsLogDir; LocalFSStore(const Params & params); - void narFromPath(const Path & path, Sink & sink) override; + void narFromPath(const StorePath & path, Sink & sink) override; ref getFSAccessor() override; /* Register a permanent GC root. */ - Path addPermRoot(const Path & storePath, + Path addPermRoot(const StorePath & storePath, const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false); virtual Path getRealStoreDir() { return storeDir; } - Path toRealPath(const Path & storePath) + Path toRealPath(const Path & storePath) override { - return getRealStoreDir() + "/" + baseNameOf(storePath); + assert(isInStore(storePath)); + return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); } -}; + std::shared_ptr getBuildLog(const StorePath & path) override; +}; -/* Extract the name part of the given store path. */ -string storePathToName(const Path & path); /* Extract the hash part of the given store path. */ string storePathToHash(const Path & path); -/* Check whether ‘name’ is a valid store path name part, i.e. contains - only the characters [a-zA-Z0-9\+\-\.\_\?\=] and doesn't start with - a dot. */ -void checkStoreName(const string & name); - /* Copy a path from one store to another. */ void copyStorePath(ref srcStore, ref dstStore, - const Path & storePath, bool repair = false, bool dontCheckSigs = false); + const StorePath & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); + + +/* Copy store paths from one store to another. The paths may be copied + in parallel. They are copied in a topologically sorted order + (i.e. if A is a reference of B, then A is copied before B), but + the set of store paths is not automatically closed; use + copyClosure() for that. */ +void copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs, + SubstituteFlag substitute = NoSubstitute); /* Copy the closure of the specified paths from one store to another. */ void copyClosure(ref srcStore, ref dstStore, - const PathSet & storePaths, bool repair = false, bool dontCheckSigs = false); + const StorePathSet & storePaths, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs, + SubstituteFlag substitute = NoSubstitute); /* Remove the temporary roots file for this process. Any temporary @@ -611,21 +768,35 @@ void removeTempRoots(); /* Return a Store object to access the Nix store denoted by ‘uri’ (slight misnomer...). Supported values are: - * ‘direct’: The Nix store in /nix/store and database in + * ‘local’: The Nix store in /nix/store and database in /nix/var/nix/db, accessed directly. * ‘daemon’: The Nix store accessed via a Unix domain socket connection to nix-daemon. + * ‘unix://’: The Nix store accessed via a Unix domain socket + connection to nix-daemon, with the socket located at . + + * ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on + whether the user has write access to the local Nix + store/database. + * ‘file://’: A binary cache stored in . - If ‘uri’ is empty, it defaults to ‘direct’ or ‘daemon’ depending on - whether the user has write access to the local Nix store/database. - set to true *unless* you're going to collect garbage. */ -ref openStore(const std::string & uri = getEnv("NIX_REMOTE")); + * ‘https://’: A binary cache accessed via HTTP. + * ‘s3://’: A writable binary cache stored on Amazon's Simple + Storage Service. + + * ‘ssh://[user@]’: A remote Nix store accessed by running + ‘nix-store --serve’ via SSH. + + You can pass parameters to the store implementation by appending + ‘?key=value&key=value&...’ to the URI. +*/ +ref openStore(const std::string & uri = settings.storeUri.get(), + const Store::Params & extraParams = Store::Params()); -void copyPaths(ref from, ref to, const Paths & storePaths); enum StoreType { tDaemon, @@ -634,11 +805,11 @@ enum StoreType { }; -StoreType getStoreType(const std::string & uri = getEnv("NIX_REMOTE"), const std::string & stateDir = settings.nixStateDir); +StoreType getStoreType(const std::string & uri = settings.storeUri.get(), + const std::string & stateDir = settings.nixStateDir); /* Return the default substituter stores, defined by the - ‘substituters’ option and various legacy options like - ‘binary-caches’. */ + ‘substituters’ option and various legacy options. */ std::list> getDefaultSubstituters(); @@ -665,13 +836,18 @@ struct RegisterStoreImplementation string showPaths(const PathSet & paths); -ValidPathInfo decodeValidPathInfo(std::istream & str, +std::optional decodeValidPathInfo( + const Store & store, + std::istream & str, bool hashGiven = false); -MakeError(SubstError, Error) -MakeError(BuildError, Error) /* denotes a permanent build failure */ -MakeError(InvalidPath, Error) +/* Compute the content-addressability assertion (ValidPathInfo::ca) + for paths created by makeFixedOutputPath() / addToStore(). */ +std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); + +/* Split URI into protocol+hierarchy part and its parameter set. */ +std::pair splitUriAndParams(const std::string & uri); } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 6a4ed47cc9f..ac42457fc41 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x112 +#define PROTOCOL_VERSION 0x115 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -36,7 +36,7 @@ typedef enum { wopClearFailedPaths = 25, wopQueryPathInfo = 26, wopImportPaths = 27, // obsolete - wopQueryDerivationOutputNames = 28, + wopQueryDerivationOutputNames = 28, // obsolete wopQueryPathFromHashPart = 29, wopQuerySubstitutablePathInfos = 30, wopQueryValidPaths = 31, @@ -47,7 +47,8 @@ typedef enum { wopBuildDerivation = 36, wopAddSignatures = 37, wopNarFromPath = 38, - wopAddToStoreNar = 39 + wopAddToStoreNar = 39, + wopQueryMissing = 40, } WorkerOp; @@ -56,10 +57,17 @@ typedef enum { #define STDERR_WRITE 0x64617416 // data for sink #define STDERR_LAST 0x616c7473 #define STDERR_ERROR 0x63787470 +#define STDERR_START_ACTIVITY 0x53545254 +#define STDERR_STOP_ACTIVITY 0x53544f50 +#define STDERR_RESULT 0x52534c54 -Path readStorePath(Store & store, Source & from); -template T readStorePaths(Store & store, Source & from); +class Store; +struct Source; + +template T readStorePaths(const Store & store, Source & from); + +void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); } diff --git a/src/libutil/affinity.cc b/src/libutil/affinity.cc index 98f8287ada6..ac2295e4ab7 100644 --- a/src/libutil/affinity.cc +++ b/src/libutil/affinity.cc @@ -12,6 +12,17 @@ namespace nix { #if __linux__ static bool didSaveAffinity = false; static cpu_set_t savedAffinity; + +std::ostream& operator<<(std::ostream &os, const cpu_set_t &cset) +{ + auto count = CPU_COUNT(&cset); + for (int i=0; i < count; ++i) + { + os << (CPU_ISSET(i,&cset) ? "1" : "0"); + } + + return os; +} #endif @@ -25,7 +36,7 @@ void setAffinityTo(int cpu) CPU_ZERO(&newAffinity); CPU_SET(cpu, &newAffinity); if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1) - printError(format("failed to lock thread to CPU %1%") % cpu); + printError("failed to lock thread to CPU %1%", cpu); #endif } @@ -47,7 +58,11 @@ void restoreAffinity() #if __linux__ if (!didSaveAffinity) return; if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) - printError("failed to restore affinity %1%"); + { + std::ostringstream oss; + oss << savedAffinity; + printError("failed to restore CPU affinity %1%", oss.str()); + } #endif } diff --git a/src/libutil/ansicolor.hh b/src/libutil/ansicolor.hh new file mode 100644 index 00000000000..8ae07b0924e --- /dev/null +++ b/src/libutil/ansicolor.hh @@ -0,0 +1,15 @@ +#pragma once + +namespace nix { + +/* Some ANSI escape sequences. */ +#define ANSI_NORMAL "\e[0m" +#define ANSI_BOLD "\e[1m" +#define ANSI_FAINT "\e[2m" +#define ANSI_ITALIC "\e[3m" +#define ANSI_RED "\e[31;1m" +#define ANSI_GREEN "\e[32;1m" +#define ANSI_YELLOW "\e[33;1m" +#define ANSI_BLUE "\e[34;1m" + +} diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index fbba7f853f9..6a848470520 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -1,5 +1,3 @@ -#include "config.h" - #include #include #include @@ -15,23 +13,31 @@ #include "archive.hh" #include "util.hh" - +#include "config.hh" namespace nix { +struct ArchiveSettings : Config +{ + Setting useCaseHack{this, + #if __APPLE__ + true, + #else + false, + #endif + "use-case-hack", + "Whether to enable a Darwin-specific hack for dealing with file name collisions."}; +}; -bool useCaseHack = -#if __APPLE__ - true; -#else - false; -#endif +static ArchiveSettings archiveSettings; + +static GlobalConfig::Register r1(&archiveSettings); const std::string narVersionMagic1 = "nix-archive-1"; static string caseHackSuffix = "~nix~case~hack~"; -PathFilter defaultPathFilter; +PathFilter defaultPathFilter = [](const Path &) { return true; }; static void dumpContents(const Path & path, size_t size, @@ -40,16 +46,16 @@ static void dumpContents(const Path & path, size_t size, sink << "contents" << size; AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) throw SysError(format("opening file ‘%1%’") % path); + if (!fd) throw SysError("opening file '%1%'", path); - unsigned char buf[65536]; + std::vector buf(65536); size_t left = size; while (left > 0) { - size_t n = left > sizeof(buf) ? sizeof(buf) : left; - readFull(fd.get(), buf, n); + auto n = std::min(left, buf.size()); + readFull(fd.get(), buf.data(), n); left -= n; - sink(buf, n); + sink(buf.data(), n); } writePadding(size, sink); @@ -58,9 +64,11 @@ static void dumpContents(const Path & path, size_t size, static void dump(const Path & path, Sink & sink, PathFilter & filter) { + checkInterrupt(); + struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path ‘%1%’") % path); + throw SysError("getting attributes of path '%1%'", path); sink << "("; @@ -74,20 +82,21 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) else if (S_ISDIR(st.st_mode)) { sink << "type" << "directory"; - /* If we're on a case-insensitive system like Mac OS X, undo + /* If we're on a case-insensitive system like macOS, undo the case hack applied by restorePath(). */ std::map unhacked; for (auto & i : readDirectory(path)) - if (useCaseHack) { + if (archiveSettings.useCaseHack) { string name(i.name); size_t pos = i.name.find(caseHackSuffix); if (pos != string::npos) { - debug(format("removing case hack suffix from ‘%1%’") % (path + "/" + i.name)); + debug(format("removing case hack suffix from '%1%'") % (path + "/" + i.name)); name.erase(pos); } if (unhacked.find(name) != unhacked.end()) - throw Error(format("file name collision in between ‘%1%’ and ‘%2%’") - % (path + "/" + unhacked[name]) % (path + "/" + i.name)); + throw Error("file name collision in between '%1%' and '%2%'", + (path + "/" + unhacked[name]), + (path + "/" + i.name)); unhacked[name] = i.name; } else unhacked[i.name] = i.name; @@ -103,7 +112,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) else if (S_ISLNK(st.st_mode)) sink << "type" << "symlink" << "target" << readLink(path); - else throw Error(format("file ‘%1%’ has an unsupported type") % path); + else throw Error("file '%1%' has an unsupported type", path); sink << ")"; } @@ -146,14 +155,14 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path) sink.preallocateContents(size); unsigned long long left = size; - unsigned char buf[65536]; + std::vector buf(65536); while (left) { checkInterrupt(); - unsigned int n = sizeof(buf); - if ((unsigned long long) n > left) n = left; - source(buf, n); - sink.receiveContents(buf, n); + auto n = buf.size(); + if ((unsigned long long)n > left) n = left; + source(buf.data(), n); + sink.receiveContents(buf.data(), n); left -= n; } @@ -239,14 +248,14 @@ static void parse(ParseSink & sink, Source & source, const Path & path) } else if (s == "name") { name = readString(source); if (name.empty() || name == "." || name == ".." || name.find('/') != string::npos || name.find((char) 0) != string::npos) - throw Error(format("NAR contains invalid file name ‘%1%’") % name); + throw Error("NAR contains invalid file name '%1%'", name); if (name <= prevName) throw Error("NAR directory is not sorted"); prevName = name; - if (useCaseHack) { + if (archiveSettings.useCaseHack) { auto i = names.find(name); if (i != names.end()) { - debug(format("case collision between ‘%1%’ and ‘%2%’") % i->first % name); + debug(format("case collision between '%1%' and '%2%'") % i->first % name); name += caseHackSuffix; name += std::to_string(++i->second); } else @@ -275,7 +284,7 @@ void parseDump(ParseSink & sink, Source & source) { string version; try { - version = readString(source); + version = readString(source, narVersionMagic1.size()); } catch (SerialisationError & e) { /* This generally means the integer at the start couldn't be decoded. Ignore and throw the exception below. */ @@ -295,14 +304,14 @@ struct RestoreSink : ParseSink { Path p = dstPath + path; if (mkdir(p.c_str(), 0777) == -1) - throw SysError(format("creating directory ‘%1%’") % p); + throw SysError("creating directory '%1%'", p); }; void createRegularFile(const Path & path) { Path p = dstPath + path; fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); - if (!fd) throw SysError(format("creating file ‘%1%’") % p); + if (!fd) throw SysError("creating file '%1%'", p); } void isExecutable() @@ -323,8 +332,8 @@ struct RestoreSink : ParseSink filesystem doesn't support preallocation (e.g. on OpenSolaris). Since preallocation is just an optimisation, ignore it. */ - if (errno && errno != EINVAL) - throw SysError(format("preallocating file of %1% bytes") % len); + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) + throw SysError("preallocating file of %1% bytes", len); } #endif } @@ -350,4 +359,30 @@ void restorePath(const Path & path, Source & source) } +void copyNAR(Source & source, Sink & sink) +{ + // FIXME: if 'source' is the output of dumpPath() followed by EOF, + // we should just forward all data directly without parsing. + + ParseSink parseSink; /* null sink; just parse the NAR */ + + LambdaSource wrapper([&](unsigned char * data, size_t len) { + auto n = source.read(data, len); + sink(data, n); + return n; + }); + + parseDump(parseSink, wrapper); +} + + +void copyPath(const Path & from, const Path & to) +{ + auto source = sinkToSource([&](Sink & sink) { + dumpPath(from, sink); + }); + restorePath(to, *source); +} + + } diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index d58b91df046..768fe25368d 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -44,13 +44,6 @@ namespace nix { `+' denotes string concatenation. */ -struct PathFilter -{ - virtual ~PathFilter() { } - virtual bool operator () (const Path & path) { return true; } -}; - -extern PathFilter defaultPathFilter; void dumpPath(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); @@ -70,13 +63,21 @@ struct ParseSink virtual void createSymlink(const Path & path, const string & target) { }; }; +struct TeeSink : ParseSink +{ + TeeSource source; + + TeeSink(Source & source) : source(source) { } +}; + void parseDump(ParseSink & sink, Source & source); void restorePath(const Path & path, Source & source); +/* Read a NAR from 'source' and write it to 'sink'. */ +void copyNAR(Source & source, Sink & sink); -// FIXME: global variables are bad m'kay. -extern bool useCaseHack; +void copyPath(const Path & from, const Path & to); extern const std::string narVersionMagic1; diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 115484f9e6c..ce65801191a 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -3,6 +3,16 @@ namespace nix { +void Args::addFlag(Flag && flag_) +{ + auto flag = std::make_shared(std::move(flag_)); + if (flag->handler.arity != ArityAny) + assert(flag->handler.arity == flag->labels.size()); + assert(flag->longName != ""); + longFlags[flag->longName] = flag; + if (flag->shortName) shortFlags[flag->shortName] = flag; +} + void Args::parseCmdline(const Strings & _cmdline) { Strings pendingArgs; @@ -35,7 +45,7 @@ void Args::parseCmdline(const Strings & _cmdline) } else if (!dashDash && std::string(arg, 0, 1) == "-") { if (!processFlag(pos, cmdline.end())) - throw UsageError(format("unrecognised flag ‘%1%’") % arg); + throw UsageError("unrecognised flag '%1%'", arg); } else { pendingArgs.push_back(*pos++); @@ -49,21 +59,22 @@ void Args::parseCmdline(const Strings & _cmdline) void Args::printHelp(const string & programName, std::ostream & out) { - std::cout << "Usage: " << programName << " ..."; + std::cout << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "FLAGS..." ANSI_NORMAL, programName); for (auto & exp : expectedArgs) { std::cout << renderLabels({exp.label}); // FIXME: handle arity > 1 if (exp.arity == 0) std::cout << "..."; + if (exp.optional) std::cout << "?"; } std::cout << "\n"; auto s = description(); if (s != "") - std::cout << "\nSummary: " << s << ".\n"; + std::cout << "\n" ANSI_BOLD "Summary:" ANSI_NORMAL " " << s << ".\n"; if (longFlags.size()) { std::cout << "\n"; - std::cout << "Flags:\n"; + std::cout << ANSI_BOLD "Flags:" ANSI_NORMAL "\n"; printFlags(out); } } @@ -71,11 +82,13 @@ void Args::printHelp(const string & programName, std::ostream & out) void Args::printFlags(std::ostream & out) { Table2 table; - for (auto & flag : longFlags) + for (auto & flag : longFlags) { + if (hiddenCategories.count(flag.second->category)) continue; table.push_back(std::make_pair( - (flag.second.shortName ? std::string("-") + flag.second.shortName + ", " : " ") - + "--" + flag.first + renderLabels(flag.second.labels), - flag.second.description)); + (flag.second->shortName ? std::string("-") + flag.second->shortName + ", " : " ") + + "--" + flag.first + renderLabels(flag.second->labels), + flag.second->description)); + } printTable(out, table); } @@ -85,28 +98,29 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) auto process = [&](const std::string & name, const Flag & flag) -> bool { ++pos; - Strings args; - for (size_t n = 0 ; n < flag.arity; ++n) { - if (pos == end) - throw UsageError(format("flag ‘%1%’ requires %2% argument(s)") - % name % flag.arity); + std::vector args; + for (size_t n = 0 ; n < flag.handler.arity; ++n) { + if (pos == end) { + if (flag.handler.arity == ArityAny) break; + throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity); + } args.push_back(*pos++); } - flag.handler(args); + flag.handler.fun(std::move(args)); return true; }; if (string(*pos, 0, 2) == "--") { auto i = longFlags.find(string(*pos, 2)); if (i == longFlags.end()) return false; - return process("--" + i->first, i->second); + return process("--" + i->first, *i->second); } if (string(*pos, 0, 1) == "-" && pos->size() == 2) { auto c = (*pos)[1]; auto i = shortFlags.find(c); if (i == shortFlags.end()) return false; - return process(std::string("-") + c, i->second); + return process(std::string("-") + c, *i->second); } return false; @@ -116,7 +130,7 @@ bool Args::processArgs(const Strings & args, bool finish) { if (expectedArgs.empty()) { if (!args.empty()) - throw UsageError(format("unexpected argument ‘%1%’") % args.front()); + throw UsageError("unexpected argument '%1%'", args.front()); return true; } @@ -127,24 +141,31 @@ bool Args::processArgs(const Strings & args, bool finish) if ((exp.arity == 0 && finish) || (exp.arity > 0 && args.size() == exp.arity)) { - exp.handler(args); + std::vector ss; + for (auto & s : args) ss.push_back(s); + exp.handler(std::move(ss)); expectedArgs.pop_front(); res = true; } - if (finish && !expectedArgs.empty()) + if (finish && !expectedArgs.empty() && !expectedArgs.front().optional) throw UsageError("more arguments are required"); return res; } -void Args::mkHashTypeFlag(const std::string & name, HashType * ht) +Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) { - mkFlag1(0, name, "TYPE", "hash algorithm (‘md5’, ‘sha1’, ‘sha256’, or ‘sha512’)", [=](std::string s) { - *ht = parseHashType(s); - if (*ht == htUnknown) - throw UsageError(format("unknown hash type ‘%1%’") % s); - }); + return Flag { + .longName = std::move(longName), + .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')", + .labels = {"hash-algo"}, + .handler = {[ht](std::string s) { + *ht = parseHashType(s); + if (*ht == htUnknown) + throw UsageError("unknown hash type '%1%'", s); + }} + }; } Strings argvToStrings(int argc, char * * argv) @@ -160,7 +181,7 @@ std::string renderLabels(const Strings & labels) std::string res; for (auto label : labels) { for (auto & c : label) c = std::toupper(c); - res += " <" + label + ">"; + res += " " ANSI_ITALIC + label + ANSI_NORMAL; } return res; } @@ -169,12 +190,92 @@ void printTable(std::ostream & out, const Table2 & table) { size_t max = 0; for (auto & row : table) - max = std::max(max, row.first.size()); + max = std::max(max, filterANSIEscapes(row.first, true).size()); for (auto & row : table) { out << " " << row.first - << std::string(max - row.first.size() + 2, ' ') + << std::string(max - filterANSIEscapes(row.first, true).size() + 2, ' ') << row.second << "\n"; } } +void Command::printHelp(const string & programName, std::ostream & out) +{ + Args::printHelp(programName, out); + + auto exs = examples(); + if (!exs.empty()) { + out << "\n" ANSI_BOLD "Examples:" ANSI_NORMAL "\n"; + for (auto & ex : exs) + out << "\n" + << " " << ex.description << "\n" // FIXME: wrap + << " $ " << ex.command << "\n"; + } +} + +MultiCommand::MultiCommand(const Commands & commands) + : commands(commands) +{ + expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector ss) { + assert(!command); + auto cmd = ss[0]; + if (auto alias = get(deprecatedAliases, cmd)) { + warn("'%s' is a deprecated alias for '%s'", cmd, *alias); + cmd = *alias; + } + auto i = commands.find(cmd); + if (i == commands.end()) + throw UsageError("'%s' is not a recognised command", cmd); + command = {cmd, i->second()}; + }}); + + categories[Command::catDefault] = "Available commands"; +} + +void MultiCommand::printHelp(const string & programName, std::ostream & out) +{ + if (command) { + command->second->printHelp(programName + " " + command->first, out); + return; + } + + out << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "COMMAND FLAGS... ARGS..." ANSI_NORMAL "\n", programName); + + out << "\n" ANSI_BOLD "Common flags:" ANSI_NORMAL "\n"; + printFlags(out); + + std::map>> commandsByCategory; + + for (auto & [name, commandFun] : commands) { + auto command = commandFun(); + commandsByCategory[command->category()].insert_or_assign(name, command); + } + + for (auto & [category, commands] : commandsByCategory) { + out << fmt("\n" ANSI_BOLD "%s:" ANSI_NORMAL "\n", categories[category]); + + Table2 table; + for (auto & [name, command] : commands) { + auto descr = command->description(); + if (!descr.empty()) + table.push_back(std::make_pair(name, descr)); + } + printTable(out, table); + } +} + +bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end) +{ + if (Args::processFlag(pos, end)) return true; + if (command && command->second->processFlag(pos, end)) return true; + return false; +} + +bool MultiCommand::processArgs(const Strings & args, bool finish) +{ + if (command) + return command->second->processArgs(args, finish); + else + return Args::processArgs(args, finish); +} + } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index ac12f8be633..154d1e6aadb 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -26,61 +26,106 @@ public: protected: + static const size_t ArityAny = std::numeric_limits::max(); + /* Flags. */ struct Flag { - char shortName; + typedef std::shared_ptr ptr; + + struct Handler + { + std::function)> fun; + size_t arity; + + Handler() {} + + Handler(std::function)> && fun) + : fun(std::move(fun)) + , arity(ArityAny) + { } + + Handler(std::function && handler) + : fun([handler{std::move(handler)}](std::vector) { handler(); }) + , arity(0) + { } + + Handler(std::function && handler) + : fun([handler{std::move(handler)}](std::vector ss) { + handler(std::move(ss[0])); + }) + , arity(1) + { } + + Handler(std::function && handler) + : fun([handler{std::move(handler)}](std::vector ss) { + handler(std::move(ss[0]), std::move(ss[1])); + }) + , arity(2) + { } + + template + Handler(T * dest) + : fun([=](std::vector ss) { *dest = ss[0]; }) + , arity(1) + { } + + template + Handler(T * dest, const T & val) + : fun([=](std::vector ss) { *dest = val; }) + , arity(0) + { } + }; + + std::string longName; + char shortName = 0; std::string description; + std::string category; Strings labels; - size_t arity; - std::function handler; + Handler handler; + + static Flag mkHashTypeFlag(std::string && longName, HashType * ht); }; - std::map longFlags; - std::map shortFlags; + std::map longFlags; + std::map shortFlags; virtual bool processFlag(Strings::iterator & pos, Strings::iterator end); - void printFlags(std::ostream & out); + virtual void printFlags(std::ostream & out); /* Positional arguments. */ struct ExpectedArg { std::string label; size_t arity; // 0 = any - std::function handler; + bool optional; + std::function)> handler; }; std::list expectedArgs; virtual bool processArgs(const Strings & args, bool finish); + std::set hiddenCategories; + public: + void addFlag(Flag && flag); + /* Helper functions for constructing flags / positional arguments. */ - void mkFlag(char shortName, const std::string & longName, - const Strings & labels, const std::string & description, - size_t arity, std::function handler) - { - auto flag = Flag{shortName, description, labels, arity, handler}; - if (shortName) shortFlags[shortName] = flag; - longFlags[longName] = flag; - } - - void mkFlag(char shortName, const std::string & longName, - const std::string & description, std::function fun) - { - mkFlag(shortName, longName, {}, description, 0, std::bind(fun)); - } - void mkFlag1(char shortName, const std::string & longName, const std::string & label, const std::string & description, std::function fun) { - mkFlag(shortName, longName, {label}, description, 1, [=](Strings ss) { - fun(ss.front()); + addFlag({ + .longName = longName, + .shortName = shortName, + .description = description, + .labels = {label}, + .handler = {[=](std::string s) { fun(s); }} }); } @@ -90,23 +135,15 @@ public: mkFlag(shortName, name, description, dest, true); } - void mkFlag(char shortName, const std::string & longName, - const std::string & label, const std::string & description, - string * dest) - { - mkFlag1(shortName, longName, label, description, [=](std::string s) { - *dest = s; - }); - } - - void mkHashTypeFlag(const std::string & name, HashType * ht); - template void mkFlag(char shortName, const std::string & longName, const std::string & description, T * dest, const T & value) { - mkFlag(shortName, longName, {}, description, 0, [=](Strings ss) { - *dest = value; + addFlag({ + .longName = longName, + .shortName = shortName, + .description = description, + .handler = {[=]() { *dest = value; }} }); } @@ -123,33 +160,94 @@ public: void mkFlag(char shortName, const std::string & longName, const std::string & description, std::function fun) { - mkFlag(shortName, longName, {"N"}, description, 1, [=](Strings ss) { - I n; - if (!string2Int(ss.front(), n)) - throw UsageError(format("flag ‘--%1%’ requires a integer argument") % longName); - fun(n); + addFlag({ + .longName = longName, + .shortName = shortName, + .description = description, + .labels = {"N"}, + .handler = {[=](std::string s) { + I n; + if (!string2Int(s, n)) + throw UsageError("flag '--%s' requires a integer argument", longName); + fun(n); + }} }); } /* Expect a string argument. */ - void expectArg(const std::string & label, string * dest) + void expectArg(const std::string & label, string * dest, bool optional = false) { - expectedArgs.push_back(ExpectedArg{label, 1, [=](Strings ss) { - *dest = ss.front(); + expectedArgs.push_back(ExpectedArg{label, 1, optional, [=](std::vector ss) { + *dest = ss[0]; }}); } /* Expect 0 or more arguments. */ - void expectArgs(const std::string & label, Strings * dest) + void expectArgs(const std::string & label, std::vector * dest) { - expectedArgs.push_back(ExpectedArg{label, 0, [=](Strings ss) { - *dest = ss; + expectedArgs.push_back(ExpectedArg{label, 0, false, [=](std::vector ss) { + *dest = std::move(ss); }}); } friend class MultiCommand; }; +/* A command is an argument parser that can be executed by calling its + run() method. */ +struct Command : virtual Args +{ + friend class MultiCommand; + + virtual ~Command() { } + + virtual void prepare() { }; + virtual void run() = 0; + + struct Example + { + std::string description; + std::string command; + }; + + typedef std::list Examples; + + virtual Examples examples() { return Examples(); } + + typedef int Category; + + static constexpr Category catDefault = 0; + + virtual Category category() { return catDefault; } + + void printHelp(const string & programName, std::ostream & out) override; +}; + +typedef std::map()>> Commands; + +/* An argument parser that supports multiple subcommands, + i.e. ‘ ’. */ +class MultiCommand : virtual Args +{ +public: + Commands commands; + + std::map categories; + + std::map deprecatedAliases; + + // Selected command, if any. + std::optional>> command; + + MultiCommand(const Commands & commands); + + void printHelp(const string & programName, std::ostream & out) override; + + bool processFlag(Strings::iterator & pos, Strings::iterator end) override; + + bool processArgs(const Strings & args, bool finish) override; +}; + Strings argvToStrings(int argc, char * * argv); /* Helper function for rendering argument labels. */ diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index a3bbb5170d9..a117ddc72e1 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -1,188 +1,362 @@ #include "compression.hh" #include "util.hh" #include "finally.hh" +#include "logging.hh" #include #include #include #include +#include +#include + +#include + #include namespace nix { -static ref decompressXZ(const std::string & in) +// Don't feed brotli too much at once. +struct ChunkedCompressionSink : CompressionSink { - lzma_stream strm(LZMA_STREAM_INIT); + uint8_t outbuf[32 * 1024]; - lzma_ret ret = lzma_stream_decoder( - &strm, UINT64_MAX, LZMA_CONCATENATED); - if (ret != LZMA_OK) - throw Error("unable to initialise lzma decoder"); + void write(const unsigned char * data, size_t len) override + { + const size_t CHUNK_SIZE = sizeof(outbuf) << 2; + while (len) { + size_t n = std::min(CHUNK_SIZE, len); + writeInternal(data, n); + data += n; + len -= n; + } + } - Finally free([&]() { lzma_end(&strm); }); + virtual void writeInternal(const unsigned char * data, size_t len) = 0; +}; + +struct NoneSink : CompressionSink +{ + Sink & nextSink; + NoneSink(Sink & nextSink) : nextSink(nextSink) { } + void finish() override { flush(); } + void write(const unsigned char * data, size_t len) override { nextSink(data, len); } +}; - lzma_action action = LZMA_RUN; +struct GzipDecompressionSink : CompressionSink +{ + Sink & nextSink; + z_stream strm; + bool finished = false; uint8_t outbuf[BUFSIZ]; - ref res = make_ref(); - strm.next_in = (uint8_t *) in.c_str(); - strm.avail_in = in.size(); - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); - while (true) { - checkInterrupt(); + GzipDecompressionSink(Sink & nextSink) : nextSink(nextSink) + { + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = 0; + strm.next_in = Z_NULL; + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + + // Enable gzip and zlib decoding (+32) with 15 windowBits + int ret = inflateInit2(&strm,15+32); + if (ret != Z_OK) + throw CompressionError("unable to initialise gzip encoder"); + } + + ~GzipDecompressionSink() + { + inflateEnd(&strm); + } + + void finish() override + { + CompressionSink::flush(); + write(nullptr, 0); + } - if (strm.avail_in == 0) - action = LZMA_FINISH; + void write(const unsigned char * data, size_t len) override + { + assert(len <= std::numeric_limits::max()); - lzma_ret ret = lzma_code(&strm, action); + strm.next_in = (Bytef *) data; + strm.avail_in = len; + + while (!finished && (!data || strm.avail_in)) { + checkInterrupt(); - if (strm.avail_out == 0 || ret == LZMA_STREAM_END) { - res->append((char *) outbuf, sizeof(outbuf) - strm.avail_out); - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); + int ret = inflate(&strm,Z_SYNC_FLUSH); + if (ret != Z_OK && ret != Z_STREAM_END) + throw CompressionError("error while decompressing gzip file: %d (%d, %d)", + zError(ret), len, strm.avail_in); + + finished = ret == Z_STREAM_END; + + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = (Bytef *) outbuf; + strm.avail_out = sizeof(outbuf); + } } + } +}; - if (ret == LZMA_STREAM_END) - return res; +struct XzDecompressionSink : CompressionSink +{ + Sink & nextSink; + uint8_t outbuf[BUFSIZ]; + lzma_stream strm = LZMA_STREAM_INIT; + bool finished = false; + XzDecompressionSink(Sink & nextSink) : nextSink(nextSink) + { + lzma_ret ret = lzma_stream_decoder( + &strm, UINT64_MAX, LZMA_CONCATENATED); if (ret != LZMA_OK) - throw Error("error while decompressing xz file"); + throw CompressionError("unable to initialise lzma decoder"); + + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); } -} -static ref decompressBzip2(const std::string & in) + ~XzDecompressionSink() + { + lzma_end(&strm); + } + + void finish() override + { + CompressionSink::flush(); + write(nullptr, 0); + } + + void write(const unsigned char * data, size_t len) override + { + strm.next_in = data; + strm.avail_in = len; + + while (!finished && (!data || strm.avail_in)) { + checkInterrupt(); + + lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH); + if (ret != LZMA_OK && ret != LZMA_STREAM_END) + throw CompressionError("error %d while decompressing xz file", ret); + + finished = ret == LZMA_STREAM_END; + + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + } + } +}; + +struct BzipDecompressionSink : ChunkedCompressionSink { + Sink & nextSink; bz_stream strm; - memset(&strm, 0, sizeof(strm)); + bool finished = false; - int ret = BZ2_bzDecompressInit(&strm, 0, 0); - if (ret != BZ_OK) - throw Error("unable to initialise bzip2 decoder"); + BzipDecompressionSink(Sink & nextSink) : nextSink(nextSink) + { + memset(&strm, 0, sizeof(strm)); + int ret = BZ2_bzDecompressInit(&strm, 0, 0); + if (ret != BZ_OK) + throw CompressionError("unable to initialise bzip2 decoder"); - Finally free([&]() { BZ2_bzDecompressEnd(&strm); }); + strm.next_out = (char *) outbuf; + strm.avail_out = sizeof(outbuf); + } - char outbuf[BUFSIZ]; - ref res = make_ref(); - strm.next_in = (char *) in.c_str(); - strm.avail_in = in.size(); - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); + ~BzipDecompressionSink() + { + BZ2_bzDecompressEnd(&strm); + } - while (true) { - checkInterrupt(); + void finish() override + { + flush(); + write(nullptr, 0); + } + + void writeInternal(const unsigned char * data, size_t len) override + { + assert(len <= std::numeric_limits::max()); + + strm.next_in = (char *) data; + strm.avail_in = len; + + while (strm.avail_in) { + checkInterrupt(); + + int ret = BZ2_bzDecompress(&strm); + if (ret != BZ_OK && ret != BZ_STREAM_END) + throw CompressionError("error while decompressing bzip2 file"); - int ret = BZ2_bzDecompress(&strm); + finished = ret == BZ_STREAM_END; - if (strm.avail_out == 0 || ret == BZ_STREAM_END) { - res->append(outbuf, sizeof(outbuf) - strm.avail_out); - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = (char *) outbuf; + strm.avail_out = sizeof(outbuf); + } } + } +}; - if (ret == BZ_STREAM_END) - return res; +struct BrotliDecompressionSink : ChunkedCompressionSink +{ + Sink & nextSink; + BrotliDecoderState * state; + bool finished = false; - if (ret != BZ_OK) - throw Error("error while decompressing bzip2 file"); + BrotliDecompressionSink(Sink & nextSink) : nextSink(nextSink) + { + state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); + if (!state) + throw CompressionError("unable to initialize brotli decoder"); } -} -ref compress(const std::string & method, const std::string & in) + ~BrotliDecompressionSink() + { + BrotliDecoderDestroyInstance(state); + } + + void finish() override + { + flush(); + writeInternal(nullptr, 0); + } + + void writeInternal(const unsigned char * data, size_t len) override + { + const uint8_t * next_in = data; + size_t avail_in = len; + uint8_t * next_out = outbuf; + size_t avail_out = sizeof(outbuf); + + while (!finished && (!data || avail_in)) { + checkInterrupt(); + + if (!BrotliDecoderDecompressStream(state, + &avail_in, &next_in, + &avail_out, &next_out, + nullptr)) + throw CompressionError("error while decompressing brotli file"); + + if (avail_out < sizeof(outbuf) || avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - avail_out); + next_out = outbuf; + avail_out = sizeof(outbuf); + } + + finished = BrotliDecoderIsFinished(state); + } + } +}; + +ref decompress(const std::string & method, const std::string & in) { StringSink ssink; - auto sink = makeCompressionSink(method, ssink); + auto sink = makeDecompressionSink(method, ssink); (*sink)(in); sink->finish(); return ssink.s; } -ref decompress(const std::string & method, const std::string & in) +ref makeDecompressionSink(const std::string & method, Sink & nextSink) { - if (method == "none") - return make_ref(in); + if (method == "none" || method == "") + return make_ref(nextSink); else if (method == "xz") - return decompressXZ(in); + return make_ref(nextSink); else if (method == "bzip2") - return decompressBzip2(in); + return make_ref(nextSink); + else if (method == "gzip") + return make_ref(nextSink); + else if (method == "br") + return make_ref(nextSink); else - throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method); + throw UnknownCompressionMethod("unknown compression method '%s'", method); } -struct NoneSink : CompressionSink -{ - Sink & nextSink; - NoneSink(Sink & nextSink) : nextSink(nextSink) { } - void finish() override { flush(); } - void write(const unsigned char * data, size_t len) override { nextSink(data, len); } -}; - -struct XzSink : CompressionSink +struct XzCompressionSink : CompressionSink { Sink & nextSink; uint8_t outbuf[BUFSIZ]; lzma_stream strm = LZMA_STREAM_INIT; bool finished = false; - XzSink(Sink & nextSink) : nextSink(nextSink) + XzCompressionSink(Sink & nextSink, bool parallel) : nextSink(nextSink) { - lzma_ret ret = lzma_easy_encoder( - &strm, 6, LZMA_CHECK_CRC64); + lzma_ret ret; + bool done = false; + + if (parallel) { +#ifdef HAVE_LZMA_MT + lzma_mt mt_options = {}; + mt_options.flags = 0; + mt_options.timeout = 300; // Using the same setting as the xz cmd line + mt_options.preset = LZMA_PRESET_DEFAULT; + mt_options.filters = NULL; + mt_options.check = LZMA_CHECK_CRC64; + mt_options.threads = lzma_cputhreads(); + mt_options.block_size = 0; + if (mt_options.threads == 0) + mt_options.threads = 1; + // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the + // number of threads. + ret = lzma_stream_encoder_mt(&strm, &mt_options); + done = true; +#else + printMsg(lvlError, "warning: parallel XZ compression requested but not supported, falling back to single-threaded compression"); +#endif + } + + if (!done) + ret = lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64); + if (ret != LZMA_OK) - throw Error("unable to initialise lzma encoder"); + throw CompressionError("unable to initialise lzma encoder"); + // FIXME: apply the x86 BCJ filter? strm.next_out = outbuf; strm.avail_out = sizeof(outbuf); } - ~XzSink() + ~XzCompressionSink() { - assert(finished); lzma_end(&strm); } void finish() override { CompressionSink::flush(); - - assert(!finished); - finished = true; - - while (true) { - checkInterrupt(); - - lzma_ret ret = lzma_code(&strm, LZMA_FINISH); - if (ret != LZMA_OK && ret != LZMA_STREAM_END) - throw Error("error while flushing xz file"); - - if (strm.avail_out == 0 || ret == LZMA_STREAM_END) { - nextSink(outbuf, sizeof(outbuf) - strm.avail_out); - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); - } - - if (ret == LZMA_STREAM_END) break; - } + write(nullptr, 0); } void write(const unsigned char * data, size_t len) override { - assert(!finished); - strm.next_in = data; strm.avail_in = len; - while (strm.avail_in) { + while (!finished && (!data || strm.avail_in)) { checkInterrupt(); - lzma_ret ret = lzma_code(&strm, LZMA_RUN); - if (ret != LZMA_OK) - throw Error("error while compressing xz file"); + lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH); + if (ret != LZMA_OK && ret != LZMA_STREAM_END) + throw CompressionError("error %d while compressing xz file", ret); + + finished = ret == LZMA_STREAM_END; - if (strm.avail_out == 0) { - nextSink(outbuf, sizeof(outbuf)); + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); strm.next_out = outbuf; strm.avail_out = sizeof(outbuf); } @@ -190,87 +364,133 @@ struct XzSink : CompressionSink } }; -struct BzipSink : CompressionSink +struct BzipCompressionSink : ChunkedCompressionSink { Sink & nextSink; - char outbuf[BUFSIZ]; bz_stream strm; bool finished = false; - BzipSink(Sink & nextSink) : nextSink(nextSink) + BzipCompressionSink(Sink & nextSink) : nextSink(nextSink) { memset(&strm, 0, sizeof(strm)); int ret = BZ2_bzCompressInit(&strm, 9, 0, 30); if (ret != BZ_OK) - throw Error("unable to initialise bzip2 encoder"); + throw CompressionError("unable to initialise bzip2 encoder"); - strm.next_out = outbuf; + strm.next_out = (char *) outbuf; strm.avail_out = sizeof(outbuf); } - ~BzipSink() + ~BzipCompressionSink() { - assert(finished); BZ2_bzCompressEnd(&strm); } void finish() override { flush(); + writeInternal(nullptr, 0); + } + + void writeInternal(const unsigned char * data, size_t len) override + { + assert(len <= std::numeric_limits::max()); - assert(!finished); - finished = true; + strm.next_in = (char *) data; + strm.avail_in = len; - while (true) { + while (!finished && (!data || strm.avail_in)) { checkInterrupt(); - int ret = BZ2_bzCompress(&strm, BZ_FINISH); - if (ret != BZ_FINISH_OK && ret != BZ_STREAM_END) - throw Error("error while flushing bzip2 file"); + int ret = BZ2_bzCompress(&strm, data ? BZ_RUN : BZ_FINISH); + if (ret != BZ_RUN_OK && ret != BZ_FINISH_OK && ret != BZ_STREAM_END) + throw CompressionError("error %d while compressing bzip2 file", ret); - if (strm.avail_out == 0 || ret == BZ_STREAM_END) { - nextSink((unsigned char *) outbuf, sizeof(outbuf) - strm.avail_out); - strm.next_out = outbuf; + finished = ret == BZ_STREAM_END; + + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = (char *) outbuf; strm.avail_out = sizeof(outbuf); } - - if (ret == BZ_STREAM_END) break; } } +}; - void write(const unsigned char * data, size_t len) override +struct BrotliCompressionSink : ChunkedCompressionSink +{ + Sink & nextSink; + uint8_t outbuf[BUFSIZ]; + BrotliEncoderState *state; + bool finished = false; + + BrotliCompressionSink(Sink & nextSink) : nextSink(nextSink) { - assert(!finished); + state = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr); + if (!state) + throw CompressionError("unable to initialise brotli encoder"); + } - strm.next_in = (char *) data; - strm.avail_in = len; + ~BrotliCompressionSink() + { + BrotliEncoderDestroyInstance(state); + } - while (strm.avail_in) { - checkInterrupt(); + void finish() override + { + flush(); + writeInternal(nullptr, 0); + } - int ret = BZ2_bzCompress(&strm, BZ_RUN); - if (ret != BZ_OK) - Error("error while compressing bzip2 file"); + void writeInternal(const unsigned char * data, size_t len) override + { + const uint8_t * next_in = data; + size_t avail_in = len; + uint8_t * next_out = outbuf; + size_t avail_out = sizeof(outbuf); - if (strm.avail_out == 0) { - nextSink((unsigned char *) outbuf, sizeof(outbuf)); - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); + while (!finished && (!data || avail_in)) { + checkInterrupt(); + + if (!BrotliEncoderCompressStream(state, + data ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH, + &avail_in, &next_in, + &avail_out, &next_out, + nullptr)) + throw CompressionError("error while compressing brotli compression"); + + if (avail_out < sizeof(outbuf) || avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - avail_out); + next_out = outbuf; + avail_out = sizeof(outbuf); } + + finished = BrotliEncoderIsFinished(state); } } }; -ref makeCompressionSink(const std::string & method, Sink & nextSink) +ref makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel) { if (method == "none") return make_ref(nextSink); else if (method == "xz") - return make_ref(nextSink); + return make_ref(nextSink, parallel); else if (method == "bzip2") - return make_ref(nextSink); + return make_ref(nextSink); + else if (method == "br") + return make_ref(nextSink); else - throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method); + throw UnknownCompressionMethod("unknown compression method '%s'", method); +} + +ref compress(const std::string & method, const std::string & in, const bool parallel) +{ + StringSink ssink; + auto sink = makeCompressionSink(method, ssink, parallel); + (*sink)(in); + sink->finish(); + return ssink.s; } } diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index eacf559d65e..dd666a4e19f 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -8,17 +8,21 @@ namespace nix { -ref compress(const std::string & method, const std::string & in); - -ref decompress(const std::string & method, const std::string & in); - struct CompressionSink : BufferedSink { virtual void finish() = 0; }; -ref makeCompressionSink(const std::string & method, Sink & nextSink); +ref decompress(const std::string & method, const std::string & in); + +ref makeDecompressionSink(const std::string & method, Sink & nextSink); + +ref compress(const std::string & method, const std::string & in, const bool parallel = false); + +ref makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false); MakeError(UnknownCompressionMethod, Error); +MakeError(CompressionError, Error); + } diff --git a/src/libutil/config.cc b/src/libutil/config.cc new file mode 100644 index 00000000000..8fc700a2bfa --- /dev/null +++ b/src/libutil/config.cc @@ -0,0 +1,349 @@ +#include "config.hh" +#include "args.hh" +#include "json.hh" + +namespace nix { + +bool Config::set(const std::string & name, const std::string & value) +{ + auto i = _settings.find(name); + if (i == _settings.end()) return false; + i->second.setting->set(value); + i->second.setting->overriden = true; + return true; +} + +void Config::addSetting(AbstractSetting * setting) +{ + _settings.emplace(setting->name, Config::SettingData(false, setting)); + for (auto & alias : setting->aliases) + _settings.emplace(alias, Config::SettingData(true, setting)); + + bool set = false; + + auto i = unknownSettings.find(setting->name); + if (i != unknownSettings.end()) { + setting->set(i->second); + setting->overriden = true; + unknownSettings.erase(i); + set = true; + } + + for (auto & alias : setting->aliases) { + auto i = unknownSettings.find(alias); + if (i != unknownSettings.end()) { + if (set) + warn("setting '%s' is set, but it's an alias of '%s' which is also set", + alias, setting->name); + else { + setting->set(i->second); + setting->overriden = true; + unknownSettings.erase(i); + set = true; + } + } + } +} + +void AbstractConfig::warnUnknownSettings() +{ + for (auto & s : unknownSettings) + warn("unknown setting '%s'", s.first); +} + +void AbstractConfig::reapplyUnknownSettings() +{ + auto unknownSettings2 = std::move(unknownSettings); + for (auto & s : unknownSettings2) + set(s.first, s.second); +} + +void Config::getSettings(std::map & res, bool overridenOnly) +{ + for (auto & opt : _settings) + if (!opt.second.isAlias && (!overridenOnly || opt.second.setting->overriden)) + res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); +} + +void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) { + unsigned int pos = 0; + + while (pos < contents.size()) { + string line; + while (pos < contents.size() && contents[pos] != '\n') + line += contents[pos++]; + pos++; + + string::size_type hash = line.find('#'); + if (hash != string::npos) + line = string(line, 0, hash); + + vector tokens = tokenizeString >(line); + if (tokens.empty()) continue; + + if (tokens.size() < 2) + throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + + auto include = false; + auto ignoreMissing = false; + if (tokens[0] == "include") + include = true; + else if (tokens[0] == "!include") { + include = true; + ignoreMissing = true; + } + + if (include) { + if (tokens.size() != 2) + throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + auto p = absPath(tokens[1], dirOf(path)); + if (pathExists(p)) { + applyConfigFile(p); + } else if (!ignoreMissing) { + throw Error("file '%1%' included from '%2%' not found", p, path); + } + continue; + } + + if (tokens[1] != "=") + throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + + string name = tokens[0]; + + vector::iterator i = tokens.begin(); + advance(i, 2); + + set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow + }; +} + +void AbstractConfig::applyConfigFile(const Path & path) +{ + try { + string contents = readFile(path); + applyConfig(contents, path); + } catch (SysError &) { } +} + +void Config::resetOverriden() +{ + for (auto & s : _settings) + s.second.setting->overriden = false; +} + +void Config::toJSON(JSONObject & out) +{ + for (auto & s : _settings) + if (!s.second.isAlias) { + JSONObject out2(out.object(s.first)); + out2.attr("description", s.second.setting->description); + JSONPlaceholder out3(out2.placeholder("value")); + s.second.setting->toJSON(out3); + } +} + +void Config::convertToArgs(Args & args, const std::string & category) +{ + for (auto & s : _settings) + if (!s.second.isAlias) + s.second.setting->convertToArg(args, category); +} + +AbstractSetting::AbstractSetting( + const std::string & name, + const std::string & description, + const std::set & aliases) + : name(name), description(description), aliases(aliases) +{ +} + +void AbstractSetting::setDefault(const std::string & str) +{ + if (!overriden) set(str); +} + +void AbstractSetting::toJSON(JSONPlaceholder & out) +{ + out.write(to_string()); +} + +void AbstractSetting::convertToArg(Args & args, const std::string & category) +{ +} + +template +void BaseSetting::toJSON(JSONPlaceholder & out) +{ + out.write(value); +} + +template +void BaseSetting::convertToArg(Args & args, const std::string & category) +{ + args.addFlag({ + .longName = name, + .description = description, + .category = category, + .labels = {"value"}, + .handler = {[=](std::string s) { overriden = true; set(s); }}, + }); +} + +template<> void BaseSetting::set(const std::string & str) +{ + value = str; +} + +template<> std::string BaseSetting::to_string() const +{ + return value; +} + +template +void BaseSetting::set(const std::string & str) +{ + static_assert(std::is_integral::value, "Integer required."); + if (!string2Int(str, value)) + throw UsageError("setting '%s' has invalid value '%s'", name, str); +} + +template +std::string BaseSetting::to_string() const +{ + static_assert(std::is_integral::value, "Integer required."); + return std::to_string(value); +} + +template<> void BaseSetting::set(const std::string & str) +{ + if (str == "true" || str == "yes" || str == "1") + value = true; + else if (str == "false" || str == "no" || str == "0") + value = false; + else + throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str); +} + +template<> std::string BaseSetting::to_string() const +{ + return value ? "true" : "false"; +} + +template<> void BaseSetting::convertToArg(Args & args, const std::string & category) +{ + args.addFlag({ + .longName = name, + .description = description, + .category = category, + .handler = {[=]() { override(true); }} + }); + args.addFlag({ + .longName = "no-" + name, + .description = description, + .category = category, + .handler = {[=]() { override(false); }} + }); +} + +template<> void BaseSetting::set(const std::string & str) +{ + value = tokenizeString(str); +} + +template<> std::string BaseSetting::to_string() const +{ + return concatStringsSep(" ", value); +} + +template<> void BaseSetting::toJSON(JSONPlaceholder & out) +{ + JSONList list(out.list()); + for (auto & s : value) + list.elem(s); +} + +template<> void BaseSetting::set(const std::string & str) +{ + value = tokenizeString(str); +} + +template<> std::string BaseSetting::to_string() const +{ + return concatStringsSep(" ", value); +} + +template<> void BaseSetting::toJSON(JSONPlaceholder & out) +{ + JSONList list(out.list()); + for (auto & s : value) + list.elem(s); +} + +template class BaseSetting; +template class BaseSetting; +template class BaseSetting; +template class BaseSetting; +template class BaseSetting; +template class BaseSetting; +template class BaseSetting; +template class BaseSetting; +template class BaseSetting; +template class BaseSetting; + +void PathSetting::set(const std::string & str) +{ + if (str == "") { + if (allowEmpty) + value = ""; + else + throw UsageError("setting '%s' cannot be empty", name); + } else + value = canonPath(str); +} + +bool GlobalConfig::set(const std::string & name, const std::string & value) +{ + for (auto & config : *configRegistrations) + if (config->set(name, value)) return true; + + unknownSettings.emplace(name, value); + + return false; +} + +void GlobalConfig::getSettings(std::map & res, bool overridenOnly) +{ + for (auto & config : *configRegistrations) + config->getSettings(res, overridenOnly); +} + +void GlobalConfig::resetOverriden() +{ + for (auto & config : *configRegistrations) + config->resetOverriden(); +} + +void GlobalConfig::toJSON(JSONObject & out) +{ + for (auto & config : *configRegistrations) + config->toJSON(out); +} + +void GlobalConfig::convertToArgs(Args & args, const std::string & category) +{ + for (auto & config : *configRegistrations) + config->convertToArgs(args, category); +} + +GlobalConfig globalConfig; + +GlobalConfig::ConfigRegistrations * GlobalConfig::configRegistrations; + +GlobalConfig::Register::Register(Config * config) +{ + if (!configRegistrations) + configRegistrations = new ConfigRegistrations; + configRegistrations->emplace_back(config); +} + +} diff --git a/src/libutil/config.hh b/src/libutil/config.hh new file mode 100644 index 00000000000..5c7a70a2edf --- /dev/null +++ b/src/libutil/config.hh @@ -0,0 +1,333 @@ +#include +#include + +#include "types.hh" + +#pragma once + +namespace nix { + +/** + * The Config class provides Nix runtime configurations. + * + * What is a Configuration? + * A collection of uniquely named Settings. + * + * What is a Setting? + * Each property that you can set in a configuration corresponds to a + * `Setting`. A setting records value and description of a property + * with a default and optional aliases. + * + * A valid configuration consists of settings that are registered to a + * `Config` object instance: + * + * Config config; + * Setting systemSetting{&config, "x86_64-linux", "system", "the current system"}; + * + * The above creates a `Config` object and registers a setting called "system" + * via the variable `systemSetting` with it. The setting defaults to the string + * "x86_64-linux", it's description is "the current system". All of the + * registered settings can then be accessed as shown below: + * + * std::map settings; + * config.getSettings(settings); + * config["system"].description == "the current system" + * config["system"].value == "x86_64-linux" + * + * + * The above retrieves all currently known settings from the `Config` object + * and adds them to the `settings` map. + */ + +class Args; +class AbstractSetting; +class JSONPlaceholder; +class JSONObject; + +class AbstractConfig +{ +protected: + StringMap unknownSettings; + + AbstractConfig(const StringMap & initials = {}) + : unknownSettings(initials) + { } + +public: + + /** + * Sets the value referenced by `name` to `value`. Returns true if the + * setting is known, false otherwise. + */ + virtual bool set(const std::string & name, const std::string & value) = 0; + + struct SettingInfo + { + std::string value; + std::string description; + }; + + /** + * Adds the currently known settings to the given result map `res`. + * - res: map to store settings in + * - overridenOnly: when set to true only overridden settings will be added to `res` + */ + virtual void getSettings(std::map & res, bool overridenOnly = false) = 0; + + /** + * Parses the configuration in `contents` and applies it + * - contents: configuration contents to be parsed and applied + * - path: location of the configuration file + */ + void applyConfig(const std::string & contents, const std::string & path = ""); + + /** + * Applies a nix configuration file + * - path: the location of the config file to apply + */ + void applyConfigFile(const Path & path); + + /** + * Resets the `overridden` flag of all Settings + */ + virtual void resetOverriden() = 0; + + /** + * Outputs all settings to JSON + * - out: JSONObject to write the configuration to + */ + virtual void toJSON(JSONObject & out) = 0; + + /** + * Converts settings to `Args` to be used on the command line interface + * - args: args to write to + * - category: category of the settings + */ + virtual void convertToArgs(Args & args, const std::string & category) = 0; + + /** + * Logs a warning for each unregistered setting + */ + void warnUnknownSettings(); + + /** + * Re-applies all previously attempted changes to unknown settings + */ + void reapplyUnknownSettings(); +}; + +/* A class to simplify providing configuration settings. The typical + use is to inherit Config and add Setting members: + + class MyClass : private Config + { + Setting foo{this, 123, "foo", "the number of foos to use"}; + Setting bar{this, "blabla", "bar", "the name of the bar"}; + + MyClass() : Config(readConfigFile("/etc/my-app.conf")) + { + std::cout << foo << "\n"; // will print 123 unless overriden + } + }; +*/ + +class Config : public AbstractConfig +{ + friend class AbstractSetting; + +public: + + struct SettingData + { + bool isAlias; + AbstractSetting * setting; + SettingData(bool isAlias, AbstractSetting * setting) + : isAlias(isAlias), setting(setting) + { } + }; + + typedef std::map Settings; + +private: + + Settings _settings; + +public: + + Config(const StringMap & initials = {}) + : AbstractConfig(initials) + { } + + bool set(const std::string & name, const std::string & value) override; + + void addSetting(AbstractSetting * setting); + + void getSettings(std::map & res, bool overridenOnly = false) override; + + void resetOverriden() override; + + void toJSON(JSONObject & out) override; + + void convertToArgs(Args & args, const std::string & category) override; +}; + +class AbstractSetting +{ + friend class Config; + +public: + + const std::string name; + const std::string description; + const std::set aliases; + + int created = 123; + + bool overriden = false; + + void setDefault(const std::string & str); + +protected: + + AbstractSetting( + const std::string & name, + const std::string & description, + const std::set & aliases); + + virtual ~AbstractSetting() + { + // Check against a gcc miscompilation causing our constructor + // not to run (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431). + assert(created == 123); + } + + virtual void set(const std::string & value) = 0; + + virtual std::string to_string() const = 0; + + virtual void toJSON(JSONPlaceholder & out); + + virtual void convertToArg(Args & args, const std::string & category); + + bool isOverriden() const { return overriden; } +}; + +/* A setting of type T. */ +template +class BaseSetting : public AbstractSetting +{ +protected: + + T value; + +public: + + BaseSetting(const T & def, + const std::string & name, + const std::string & description, + const std::set & aliases = {}) + : AbstractSetting(name, description, aliases) + , value(def) + { } + + operator const T &() const { return value; } + operator T &() { return value; } + const T & get() const { return value; } + bool operator ==(const T & v2) const { return value == v2; } + bool operator !=(const T & v2) const { return value != v2; } + void operator =(const T & v) { assign(v); } + virtual void assign(const T & v) { value = v; } + + void set(const std::string & str) override; + + virtual void override(const T & v) + { + overriden = true; + value = v; + } + + std::string to_string() const override; + + void convertToArg(Args & args, const std::string & category) override; + + void toJSON(JSONPlaceholder & out) override; +}; + +template +std::ostream & operator <<(std::ostream & str, const BaseSetting & opt) +{ + str << (const T &) opt; + return str; +} + +template +bool operator ==(const T & v1, const BaseSetting & v2) { return v1 == (const T &) v2; } + +template +class Setting : public BaseSetting +{ +public: + Setting(Config * options, + const T & def, + const std::string & name, + const std::string & description, + const std::set & aliases = {}) + : BaseSetting(def, name, description, aliases) + { + options->addSetting(this); + } + + void operator =(const T & v) { this->assign(v); } +}; + +/* A special setting for Paths. These are automatically canonicalised + (e.g. "/foo//bar/" becomes "/foo/bar"). */ +class PathSetting : public BaseSetting +{ + bool allowEmpty; + +public: + + PathSetting(Config * options, + bool allowEmpty, + const Path & def, + const std::string & name, + const std::string & description, + const std::set & aliases = {}) + : BaseSetting(def, name, description, aliases) + , allowEmpty(allowEmpty) + { + options->addSetting(this); + } + + void set(const std::string & str) override; + + Path operator +(const char * p) const { return value + p; } + + void operator =(const Path & v) { this->assign(v); } +}; + +struct GlobalConfig : public AbstractConfig +{ + typedef std::vector ConfigRegistrations; + static ConfigRegistrations * configRegistrations; + + bool set(const std::string & name, const std::string & value) override; + + void getSettings(std::map & res, bool overridenOnly = false) override; + + void resetOverriden() override; + + void toJSON(JSONObject & out) override; + + void convertToArgs(Args & args, const std::string & category) override; + + struct Register + { + Register(Config * config); + }; +}; + +extern GlobalConfig globalConfig; + +} diff --git a/src/libutil/error.cc b/src/libutil/error.cc new file mode 100644 index 00000000000..0fad9ae423e --- /dev/null +++ b/src/libutil/error.cc @@ -0,0 +1,221 @@ +#include "error.hh" + +#include +#include +#include "serialise.hh" +#include + +namespace nix { + + +const std::string nativeSystem = SYSTEM; + +// addPrefix is used for show-trace. Strings added with addPrefix +// will print ahead of the error itself. +BaseError & BaseError::addPrefix(const FormatOrString & fs) +{ + prefix_ = fs.s + prefix_; + return *this; +} + +// c++ std::exception descendants must have a 'const char* what()' function. +// This stringifies the error and caches it for use by what(), or similarly by msg(). +const string& BaseError::calcWhat() const +{ + if (what_.has_value()) + return *what_; + else { + err.name = sname(); + + std::ostringstream oss; + oss << err; + what_ = oss.str(); + + return *what_; + } +} + +std::optional ErrorInfo::programName = std::nullopt; + +std::ostream& operator<<(std::ostream &os, const hintformat &hf) +{ + return os << hf.str(); +} + +string showErrPos(const ErrPos &errPos) +{ + if (errPos.line > 0) { + if (errPos.column > 0) { + return fmt("(%1%:%2%)", errPos.line, errPos.column); + } else { + return fmt("(%1%)", errPos.line); + } + } + else { + return ""; + } +} + +// if nixCode contains lines of code, print them to the ostream, indicating the error column. +void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixCode) +{ + // previous line of code. + if (nixCode.prevLineOfCode.has_value()) { + out << std::endl + << fmt("%1% %|2$5d|| %3%", + prefix, + (nixCode.errPos.line - 1), + *nixCode.prevLineOfCode); + } + + if (nixCode.errLineOfCode.has_value()) { + // line of code containing the error. + out << std::endl + << fmt("%1% %|2$5d|| %3%", + prefix, + (nixCode.errPos.line), + *nixCode.errLineOfCode); + // error arrows for the column range. + if (nixCode.errPos.column > 0) { + int start = nixCode.errPos.column; + std::string spaces; + for (int i = 0; i < start; ++i) { + spaces.append(" "); + } + + std::string arrows("^"); + + out << std::endl + << fmt("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL, + prefix, + spaces, + arrows); + } + } + + // next line of code. + if (nixCode.nextLineOfCode.has_value()) { + out << std::endl + << fmt("%1% %|2$5d|| %3%", + prefix, + (nixCode.errPos.line + 1), + *nixCode.nextLineOfCode); + } +} + +std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo) +{ + auto errwidth = std::max(getWindowSize().second, 20); + string prefix = ""; + + string levelString; + switch (einfo.level) { + case Verbosity::lvlError: { + levelString = ANSI_RED; + levelString += "error:"; + levelString += ANSI_NORMAL; + break; + } + case Verbosity::lvlWarn: { + levelString = ANSI_YELLOW; + levelString += "warning:"; + levelString += ANSI_NORMAL; + break; + } + case Verbosity::lvlInfo: { + levelString = ANSI_GREEN; + levelString += "info:"; + levelString += ANSI_NORMAL; + break; + } + case Verbosity::lvlTalkative: { + levelString = ANSI_GREEN; + levelString += "talk:"; + levelString += ANSI_NORMAL; + break; + } + case Verbosity::lvlChatty: { + levelString = ANSI_GREEN; + levelString += "chat:"; + levelString += ANSI_NORMAL; + break; + } + case Verbosity::lvlVomit: { + levelString = ANSI_GREEN; + levelString += "vomit:"; + levelString += ANSI_NORMAL; + break; + } + case Verbosity::lvlDebug: { + levelString = ANSI_YELLOW; + levelString += "debug:"; + levelString += ANSI_NORMAL; + break; + } + default: { + levelString = fmt("invalid error level: %1%", einfo.level); + break; + } + } + + auto ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length(); + auto dashwidth = ndl > (errwidth - 3) ? 3 : errwidth - ndl; + + std::string dashes(dashwidth, '-'); + + // divider. + if (einfo.name != "") + out << fmt("%1%%2%" ANSI_BLUE " --- %3% %4% %5%" ANSI_NORMAL, + prefix, + levelString, + einfo.name, + dashes, + einfo.programName.value_or("")); + else + out << fmt("%1%%2%" ANSI_BLUE " -----%3% %4%" ANSI_NORMAL, + prefix, + levelString, + dashes, + einfo.programName.value_or("")); + + bool nl = false; // intersperse newline between sections. + if (einfo.nixCode.has_value()) { + if (einfo.nixCode->errPos.file != "") { + // filename, line, column. + out << std::endl << fmt("%1%in file: " ANSI_BLUE "%2% %3%" ANSI_NORMAL, + prefix, + einfo.nixCode->errPos.file, + showErrPos(einfo.nixCode->errPos)); + } else { + out << std::endl << fmt("%1%from command line argument", prefix); + } + nl = true; + } + + // description + if (einfo.description != "") { + if (nl) + out << std::endl << prefix; + out << std::endl << prefix << einfo.description; + nl = true; + } + + // lines of code. + if (einfo.nixCode.has_value() && einfo.nixCode->errLineOfCode.has_value()) { + if (nl) + out << std::endl << prefix; + printCodeLines(out, prefix, *einfo.nixCode); + nl = true; + } + + // hint + if (einfo.hint.has_value()) { + if (nl) + out << std::endl << prefix; + out << std::endl << prefix << *einfo.hint; + nl = true; + } + + return out; +} +} diff --git a/src/libutil/error.hh b/src/libutil/error.hh new file mode 100644 index 00000000000..2ba3a3b338b --- /dev/null +++ b/src/libutil/error.hh @@ -0,0 +1,185 @@ +#pragma once + + +#include "ref.hh" +#include "types.hh" + +#include +#include +#include +#include + +#include "fmt.hh" + +/* Before 4.7, gcc's std::exception uses empty throw() specifiers for + * its (virtual) destructor and what() in c++11 mode, in violation of spec + */ +#ifdef __GNUC__ +#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7) +#define EXCEPTION_NEEDS_THROW_SPEC +#endif +#endif + +namespace nix { + +/* + +This file defines two main structs/classes used in nix error handling. + +ErrorInfo provides a standard payload of error information, with conversion to string +happening in the logger rather than at the call site. + +BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains +an ErrorInfo. + +ErrorInfo structs are sent to the logger as part of an exception, or directly with the +logError or logWarning macros. + +See the error-demo.cc program for usage examples. + +*/ + +typedef enum { + lvlError = 0, + lvlWarn, + lvlInfo, + lvlTalkative, + lvlChatty, + lvlDebug, + lvlVomit +} Verbosity; + +// ErrPos indicates the location of an error in a nix file. +struct ErrPos { + int line = 0; + int column = 0; + string file; + + operator bool() const + { + return line != 0; + } + + // convert from the Pos struct, found in libexpr. + template + ErrPos& operator=(const P &pos) + { + line = pos.line; + column = pos.column; + file = pos.file; + return *this; + } + + template + ErrPos(const P &p) + { + *this = p; + } +}; + +struct NixCode { + ErrPos errPos; + std::optional prevLineOfCode; + std::optional errLineOfCode; + std::optional nextLineOfCode; +}; + +struct ErrorInfo { + Verbosity level; + string name; + string description; + std::optional hint; + std::optional nixCode; + + static std::optional programName; +}; + +std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo); + +/* BaseError should generally not be caught, as it has Interrupted as + a subclass. Catch Error instead. */ +class BaseError : public std::exception +{ +protected: + string prefix_; // used for location traces etc. + mutable ErrorInfo err; + + mutable std::optional what_; + const string& calcWhat() const; + +public: + unsigned int status = 1; // exit status + + template + BaseError(unsigned int status, const Args & ... args) + : err { .level = lvlError, + .hint = hintfmt(args...) + } + , status(status) + { } + + template + BaseError(const std::string & fs, const Args & ... args) + : err { .level = lvlError, + .hint = hintfmt(fs, args...) + } + { } + + BaseError(hintformat hint) + : err { .level = lvlError, + .hint = hint + } + { } + + BaseError(ErrorInfo && e) + : err(std::move(e)) + { } + + BaseError(const ErrorInfo & e) + : err(e) + { } + + virtual const char* sname() const { return "BaseError"; } + +#ifdef EXCEPTION_NEEDS_THROW_SPEC + ~BaseError() throw () { }; + const char * what() const throw () { return calcWhat().c_str(); } +#else + const char * what() const noexcept override { return calcWhat().c_str(); } +#endif + + const string & msg() const { return calcWhat(); } + const string & prefix() const { return prefix_; } + BaseError & addPrefix(const FormatOrString & fs); + + const ErrorInfo & info() { calcWhat(); return err; } +}; + +#define MakeError(newClass, superClass) \ + class newClass : public superClass \ + { \ + public: \ + using superClass::superClass; \ + virtual const char* sname() const override { return #newClass; } \ + } + +MakeError(Error, BaseError); + +class SysError : public Error +{ +public: + int errNo; + + template + SysError(const Args & ... args) + :Error("") + { + errNo = errno; + auto hf = hintfmt(args...); + err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); + } + + virtual const char* sname() const override { return "SysError"; } +}; + +} diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh new file mode 100644 index 00000000000..12ab9c40718 --- /dev/null +++ b/src/libutil/fmt.hh @@ -0,0 +1,139 @@ +#pragma once + +#include +#include +#include "ansicolor.hh" + + +namespace nix { + + +/* Inherit some names from other namespaces for convenience. */ +using std::string; +using boost::format; + + +/* A variadic template that does nothing. Useful to call a function + for all variadic arguments but ignoring the result. */ +struct nop { template nop(T...) {} }; + + +struct FormatOrString +{ + string s; + FormatOrString(const string & s) : s(s) { }; + template + FormatOrString(const F & f) : s(f.str()) { }; + FormatOrString(const char * s) : s(s) { }; +}; + + +/* A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is + equivalent to ‘boost::format(format) % a_0 % ... % + ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion + takes place). */ + +template +inline void formatHelper(F & f) +{ +} + +template +inline void formatHelper(F & f, const T & x, const Args & ... args) +{ + formatHelper(f % x, args...); +} + +inline std::string fmt(const std::string & s) +{ + return s; +} + +inline std::string fmt(const char * s) +{ + return s; +} + +inline std::string fmt(const FormatOrString & fs) +{ + return fs.s; +} + +template +inline std::string fmt(const std::string & fs, const Args & ... args) +{ + boost::format f(fs); + f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + formatHelper(f, args...); + return f.str(); +} + +// ----------------------------------------------------------------------------- +// format function for hints in errors. same as fmt, except templated values +// are always in yellow. + +template +struct yellowtxt +{ + yellowtxt(const T &s) : value(s) {} + const T &value; +}; + +template +std::ostream& operator<<(std::ostream &out, const yellowtxt &y) +{ + return out << ANSI_YELLOW << y.value << ANSI_NORMAL; +} + +template +struct normaltxt +{ + normaltxt(const T &s) : value(s) {} + const T &value; +}; + +template +std::ostream& operator<<(std::ostream &out, const normaltxt &y) +{ + return out << ANSI_NORMAL << y.value; +} + +class hintformat +{ +public: + hintformat(const string &format) :fmt(format) + { + fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + } + + hintformat(const hintformat &hf) + : fmt(hf.fmt) + {} + + template + hintformat& operator%(const T &value) + { + fmt % yellowtxt(value); + return *this; + } + + std::string str() const + { + return fmt.str(); + } + +private: + format fmt; +}; + +std::ostream& operator<<(std::ostream &os, const hintformat &hf); + +template +inline hintformat hintfmt(const std::string & fs, const Args & ... args) +{ + hintformat f(fs); + formatHelper(f, args...); + return f; +} + +} diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index aa50fceb9e3..460d479a32f 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -1,5 +1,3 @@ -#include "config.h" - #include #include @@ -9,26 +7,17 @@ #include "hash.hh" #include "archive.hh" #include "util.hh" +#include "istringstream_nocopy.hh" #include #include #include - namespace nix { -Hash::Hash() -{ - type = htUnknown; - hashSize = 0; - memset(hash, 0, maxHashSize); -} - - -Hash::Hash(HashType type) +void Hash::init() { - this->type = type; if (type == htMD5) hashSize = md5HashSize; else if (type == htSHA1) hashSize = sha1HashSize; else if (type == htSHA256) hashSize = sha256HashSize; @@ -56,6 +45,8 @@ bool Hash::operator != (const Hash & h2) const bool Hash::operator < (const Hash & h) const { + if (hashSize < h.hashSize) return true; + if (hashSize > h.hashSize) return false; for (unsigned int i = 0; i < hashSize; i++) { if (hash[i] < h.hash[i]) return true; if (hash[i] > h.hash[i]) return false; @@ -64,16 +55,10 @@ bool Hash::operator < (const Hash & h) const } -std::string Hash::to_string(bool base32) const -{ - return printHashType(type) + ":" + (base32 ? printHash32(*this) : printHash(*this)); -} - - const string base16Chars = "0123456789abcdef"; -string printHash(const Hash & hash) +static string printHash16(const Hash & hash) { char buf[hash.hashSize * 2]; for (unsigned int i = 0; i < hash.hashSize; i++) { @@ -84,42 +69,11 @@ string printHash(const Hash & hash) } -Hash parseHash(const string & s) -{ - string::size_type colon = s.find(':'); - if (colon == string::npos) - throw BadHash(format("invalid hash ‘%s’") % s); - string hts = string(s, 0, colon); - HashType ht = parseHashType(hts); - if (ht == htUnknown) - throw BadHash(format("unknown hash type ‘%s’") % hts); - return parseHash16or32(ht, string(s, colon + 1)); -} - - -Hash parseHash(HashType ht, const string & s) -{ - Hash hash(ht); - if (s.length() != hash.hashSize * 2) - throw BadHash(format("invalid hash ‘%1%’") % s); - for (unsigned int i = 0; i < hash.hashSize; i++) { - string s2(s, i * 2, 2); - if (!isxdigit(s2[0]) || !isxdigit(s2[1])) - throw BadHash(format("invalid hash ‘%1%’") % s); - std::istringstream str(s2); - int n; - str >> std::hex >> n; - hash.hash[i] = n; - } - return hash; -} - - // omitted: E O U T const string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz"; -string printHash32(const Hash & hash) +static string printHash32(const Hash & hash) { assert(hash.hashSize); size_t len = hash.base32Len(); @@ -128,7 +82,7 @@ string printHash32(const Hash & hash) string s; s.reserve(len); - for (int n = len - 1; n >= 0; n--) { + for (int n = (int) len - 1; n >= 0; n--) { unsigned int b = n * 5; unsigned int i = b / 8; unsigned int j = b % 8; @@ -144,66 +98,121 @@ string printHash32(const Hash & hash) string printHash16or32(const Hash & hash) { - return hash.type == htMD5 ? printHash(hash) : printHash32(hash); + return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false); } -Hash parseHash32(HashType ht, const string & s) +std::string Hash::to_string(Base base, bool includeType) const { - Hash hash(ht); - size_t len = hash.base32Len(); - assert(s.size() == len); - - for (unsigned int n = 0; n < len; ++n) { - char c = s[len - n - 1]; - unsigned char digit; - for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ - if (base32Chars[digit] == c) break; - if (digit >= 32) - throw BadHash(format("invalid base-32 hash ‘%1%’") % s); - unsigned int b = n * 5; - unsigned int i = b / 8; - unsigned int j = b % 8; - hash.hash[i] |= digit << j; + std::string s; + if (base == SRI || includeType) { + s += printHashType(type); + s += base == SRI ? '-' : ':'; + } + switch (base) { + case Base16: + s += printHash16(*this); + break; + case Base32: + s += printHash32(*this); + break; + case Base64: + case SRI: + s += base64Encode(std::string((const char *) hash, hashSize)); + break; + } + return s; +} - if (i < hash.hashSize - 1) { - hash.hash[i + 1] |= digit >> (8 - j); - } else { - if (digit >> (8 - j)) - throw BadHash(format("invalid base-32 hash ‘%1%’") % s); + +Hash::Hash(std::string_view s, HashType type) + : type(type) +{ + size_t pos = 0; + bool isSRI = false; + + auto sep = s.find(':'); + if (sep == string::npos) { + sep = s.find('-'); + if (sep != string::npos) { + isSRI = true; + } else if (type == htUnknown) + throw BadHash("hash '%s' does not include a type", s); + } + + if (sep != string::npos) { + string hts = string(s, 0, sep); + this->type = parseHashType(hts); + if (this->type == htUnknown) + throw BadHash("unknown hash type '%s'", hts); + if (type != htUnknown && type != this->type) + throw BadHash("hash '%s' should have type '%s'", s, printHashType(type)); + pos = sep + 1; + } + + init(); + + size_t size = s.size() - pos; + + if (!isSRI && size == base16Len()) { + + auto parseHexDigit = [&](char c) { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + throw BadHash("invalid base-16 hash '%s'", s); + }; + + for (unsigned int i = 0; i < hashSize; i++) { + hash[i] = + parseHexDigit(s[pos + i * 2]) << 4 + | parseHexDigit(s[pos + i * 2 + 1]); } } - return hash; -} + else if (!isSRI && size == base32Len()) { + + for (unsigned int n = 0; n < size; ++n) { + char c = s[pos + size - n - 1]; + unsigned char digit; + for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ + if (base32Chars[digit] == c) break; + if (digit >= 32) + throw BadHash("invalid base-32 hash '%s'", s); + unsigned int b = n * 5; + unsigned int i = b / 8; + unsigned int j = b % 8; + hash[i] |= digit << j; + + if (i < hashSize - 1) { + hash[i + 1] |= digit >> (8 - j); + } else { + if (digit >> (8 - j)) + throw BadHash("invalid base-32 hash '%s'", s); + } + } + } + else if (isSRI || size == base64Len()) { + auto d = base64Decode(s.substr(pos)); + if (d.size() != hashSize) + throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", s); + assert(hashSize); + memcpy(hash, d.data(), hashSize); + } -Hash parseHash16or32(HashType ht, const string & s) -{ - Hash hash(ht); - if (s.size() == hash.hashSize * 2) - /* hexadecimal representation */ - hash = parseHash(ht, s); - else if (s.size() == hash.base32Len()) - /* base-32 representation */ - hash = parseHash32(ht, s); else - throw BadHash(format("hash ‘%1%’ has wrong length for hash type ‘%2%’") - % s % printHashType(ht)); - return hash; + throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(type)); } - -bool isHash(const string & s) +Hash newHashAllowEmpty(std::string hashStr, HashType ht) { - if (s.length() != 32) return false; - for (int i = 0; i < 32; i++) { - char c = s[i]; - if (!((c >= '0' && c <= '9') || - (c >= 'a' && c <= 'f'))) - return false; - } - return true; + if (hashStr.empty()) { + Hash h(ht); + warn("found empty hash, assuming '%s'", h.to_string(SRI, true)); + return h; + } else + return Hash(hashStr, ht); } @@ -226,7 +235,7 @@ static void start(HashType ht, Ctx & ctx) static void update(HashType ht, Ctx & ctx, - const unsigned char * bytes, unsigned int len) + const unsigned char * bytes, size_t len) { if (ht == htMD5) MD5_Update(&ctx.md5, bytes, len); else if (ht == htSHA1) SHA1_Update(&ctx.sha1, bytes, len); @@ -257,23 +266,9 @@ Hash hashString(HashType ht, const string & s) Hash hashFile(HashType ht, const Path & path) { - Ctx ctx; - Hash hash(ht); - start(ht, ctx); - - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) throw SysError(format("opening file ‘%1%’") % path); - - unsigned char buf[8192]; - ssize_t n; - while ((n = read(fd.get(), buf, sizeof(buf)))) { - checkInterrupt(); - if (n == -1) throw SysError(format("reading file ‘%1%’") % path); - update(ht, ctx, buf, n); - } - - finish(ht, ctx, hash.hash); - return hash; + HashSink sink(ht); + readFile(path, sink); + return sink.finish().first; } diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 02e213fc7b3..180fb76334d 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -20,20 +20,31 @@ const int sha512HashSize = 64; extern const string base32Chars; +enum Base : int { Base64, Base32, Base16, SRI }; + struct Hash { static const unsigned int maxHashSize = 64; - unsigned int hashSize; - unsigned char hash[maxHashSize]; + unsigned int hashSize = 0; + unsigned char hash[maxHashSize] = {}; - HashType type; + HashType type = htUnknown; /* Create an unset hash object. */ - Hash(); + Hash() { }; /* Create a zero-filled hash object. */ - Hash(HashType type); + Hash(HashType type) : type(type) { init(); }; + + /* Initialize the hash from a string representation, in the format + "[:]" or "-" (a + Subresource Integrity hash expression). If the 'type' argument + is htUnknown, then the hash type must be specified in the + string. */ + Hash(std::string_view s, HashType type = htUnknown); + + void init(); /* Check whether a hash is set. */ operator bool () const { return type != htUnknown; } @@ -59,33 +70,36 @@ struct Hash return (hashSize * 8 - 1) / 5 + 1; } - std::string to_string(bool base32 = true) const; -}; - + /* Returns the length of a base-64 representation of this hash. */ + size_t base64Len() const + { + return ((4 * hashSize / 3) + 3) & ~3; + } -/* Convert a hash to a hexadecimal representation. */ -string printHash(const Hash & hash); + /* Return a string representation of the hash, in base-16, base-32 + or base-64. By default, this is prefixed by the hash type + (e.g. "sha256:"). */ + std::string to_string(Base base, bool includeType) const; -Hash parseHash(const string & s); + std::string gitRev() const + { + assert(type == htSHA1); + return to_string(Base16, false); + } -/* Parse a hexadecimal representation of a hash code. */ -Hash parseHash(HashType ht, const string & s); + std::string gitShortRev() const + { + assert(type == htSHA1); + return std::string(to_string(Base16, false), 0, 7); + } +}; -/* Convert a hash to a base-32 representation. */ -string printHash32(const Hash & hash); +/* Helper that defaults empty hashes to the 0 hash. */ +Hash newHashAllowEmpty(std::string hashStr, HashType ht); /* Print a hash in base-16 if it's MD5, or base-32 otherwise. */ string printHash16or32(const Hash & hash); -/* Parse a base-32 representation of a hash code. */ -Hash parseHash32(HashType ht, const string & s); - -/* Parse a base-16 or base-32 representation of a hash code. */ -Hash parseHash16or32(HashType ht, const string & s); - -/* Verify that the given string is a valid hash code. */ -bool isHash(const string & s); - /* Compute the hash of the given string. */ Hash hashString(HashType ht, const string & s); @@ -94,8 +108,6 @@ Hash hashFile(HashType ht, const Path & path); /* Compute the hash of the given path. The hash is defined as (essentially) hashString(ht, dumpPath(path)). */ -struct PathFilter; -extern PathFilter defaultPathFilter; typedef std::pair HashResult; HashResult hashPath(HashType ht, const Path & path, PathFilter & filter = defaultPathFilter); @@ -113,7 +125,12 @@ string printHashType(HashType ht); union Ctx; -class HashSink : public BufferedSink +struct AbstractHashSink : virtual Sink +{ + virtual HashResult finish() = 0; +}; + +class HashSink : public BufferedSink, public AbstractHashSink { private: HashType ht; @@ -124,8 +141,8 @@ public: HashSink(HashType ht); HashSink(const HashSink & h); ~HashSink(); - void write(const unsigned char * data, size_t len); - HashResult finish(); + void write(const unsigned char * data, size_t len) override; + HashResult finish() override; HashResult currentHash(); }; diff --git a/src/libutil/istringstream_nocopy.hh b/src/libutil/istringstream_nocopy.hh new file mode 100644 index 00000000000..f7beac578e3 --- /dev/null +++ b/src/libutil/istringstream_nocopy.hh @@ -0,0 +1,92 @@ +/* This file provides a variant of std::istringstream that doesn't + copy its string argument. This is useful for large strings. The + caller must ensure that the string object is not destroyed while + it's referenced by this object. */ + +#pragma once + +#include +#include + +template , class Allocator = std::allocator> +class basic_istringbuf_nocopy : public std::basic_streambuf +{ +public: + typedef std::basic_string string_type; + + typedef typename std::basic_streambuf::off_type off_type; + + typedef typename std::basic_streambuf::pos_type pos_type; + + typedef typename std::basic_streambuf::int_type int_type; + + typedef typename std::basic_streambuf::traits_type traits_type; + +private: + const string_type & s; + + off_type off; + +public: + basic_istringbuf_nocopy(const string_type & s) : s{s}, off{0} + { + } + +private: + pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode which) + { + if (which & std::ios_base::in) { + this->off = dir == std::ios_base::beg + ? off + : (dir == std::ios_base::end + ? s.size() + off + : this->off + off); + } + return pos_type(this->off); + } + + pos_type seekpos(pos_type pos, std::ios_base::openmode which) + { + return seekoff(pos, std::ios_base::beg, which); + } + + std::streamsize showmanyc() + { + return s.size() - off; + } + + int_type underflow() + { + if (typename string_type::size_type(off) == s.size()) + return traits_type::eof(); + return traits_type::to_int_type(s[off]); + } + + int_type uflow() + { + if (typename string_type::size_type(off) == s.size()) + return traits_type::eof(); + return traits_type::to_int_type(s[off++]); + } + + int_type pbackfail(int_type ch) + { + if (off == 0 || (ch != traits_type::eof() && ch != s[off - 1])) + return traits_type::eof(); + + return traits_type::to_int_type(s[--off]); + } + +}; + +template , class Allocator = std::allocator> +class basic_istringstream_nocopy : public std::basic_iostream +{ + typedef basic_istringbuf_nocopy buf_type; + buf_type buf; +public: + basic_istringstream_nocopy(const typename buf_type::string_type & s) : + std::basic_iostream(&buf), buf(s) {}; +}; + +typedef basic_istringstream_nocopy istringstream_nocopy; diff --git a/src/libutil/json.cc b/src/libutil/json.cc index 6023d1d4fb8..74e37b4c442 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -19,68 +19,54 @@ void toJSON(std::ostream & str, const char * start, const char * end) str << '"'; } -void toJSON(std::ostream & str, const std::string & s) -{ - toJSON(str, s.c_str(), s.c_str() + s.size()); -} - void toJSON(std::ostream & str, const char * s) { if (!s) str << "null"; else toJSON(str, s, s + strlen(s)); } -void toJSON(std::ostream & str, unsigned long long n) -{ - str << n; -} - -void toJSON(std::ostream & str, unsigned long n) -{ - str << n; -} - -void toJSON(std::ostream & str, long n) -{ - str << n; -} - -void toJSON(std::ostream & str, unsigned int n) -{ - str << n; -} +template<> void toJSON(std::ostream & str, const int & n) { str << n; } +template<> void toJSON(std::ostream & str, const unsigned int & n) { str << n; } +template<> void toJSON(std::ostream & str, const long & n) { str << n; } +template<> void toJSON(std::ostream & str, const unsigned long & n) { str << n; } +template<> void toJSON(std::ostream & str, const long long & n) { str << n; } +template<> void toJSON(std::ostream & str, const unsigned long long & n) { str << n; } +template<> void toJSON(std::ostream & str, const float & n) { str << n; } +template<> void toJSON(std::ostream & str, const double & n) { str << n; } -void toJSON(std::ostream & str, int n) +template<> void toJSON(std::ostream & str, const std::string & s) { - str << n; + toJSON(str, s.c_str(), s.c_str() + s.size()); } -void toJSON(std::ostream & str, double f) +template<> void toJSON(std::ostream & str, const bool & b) { - str << f; + str << (b ? "true" : "false"); } -void toJSON(std::ostream & str, bool b) +template<> void toJSON(std::ostream & str, const std::nullptr_t & b) { - str << (b ? "true" : "false"); + str << "null"; } JSONWriter::JSONWriter(std::ostream & str, bool indent) : state(new JSONState(str, indent)) { - state->stack.push_back(this); + state->stack++; } JSONWriter::JSONWriter(JSONState * state) : state(state) { - state->stack.push_back(this); + state->stack++; } JSONWriter::~JSONWriter() { - assertActive(); - state->stack.pop_back(); - if (state->stack.empty()) delete state; + if (state) { + assertActive(); + state->stack--; + if (state->stack == 0) delete state; + } } void JSONWriter::comma() @@ -138,9 +124,11 @@ void JSONObject::open() JSONObject::~JSONObject() { - state->depth--; - if (state->indent && !first) indent(); - state->str << "}"; + if (state) { + state->depth--; + if (state->indent && !first) indent(); + state->str << "}"; + } } void JSONObject::attr(const std::string & s) @@ -183,4 +171,9 @@ JSONObject JSONPlaceholder::object() return JSONObject(state); } +JSONPlaceholder::~JSONPlaceholder() +{ + assert(!first || std::uncaught_exception()); +} + } diff --git a/src/libutil/json.hh b/src/libutil/json.hh index 03eecb73258..83213ca6664 100644 --- a/src/libutil/json.hh +++ b/src/libutil/json.hh @@ -7,15 +7,10 @@ namespace nix { void toJSON(std::ostream & str, const char * start, const char * end); -void toJSON(std::ostream & str, const std::string & s); void toJSON(std::ostream & str, const char * s); -void toJSON(std::ostream & str, unsigned long long n); -void toJSON(std::ostream & str, unsigned long n); -void toJSON(std::ostream & str, long n); -void toJSON(std::ostream & str, unsigned int n); -void toJSON(std::ostream & str, int n); -void toJSON(std::ostream & str, double f); -void toJSON(std::ostream & str, bool b); + +template +void toJSON(std::ostream & str, const T & n); class JSONWriter { @@ -26,11 +21,11 @@ protected: std::ostream & str; bool indent; size_t depth = 0; - std::vector stack; + size_t stack = 0; JSONState(std::ostream & str, bool indent) : str(str), indent(indent) { } ~JSONState() { - assert(stack.empty()); + assert(stack == 0); } }; @@ -46,7 +41,7 @@ protected: void assertActive() { - assert(!state->stack.empty() && state->stack.back() == this); + assert(state->stack != 0); } void comma(); @@ -122,6 +117,14 @@ public: open(); } + JSONObject(const JSONObject & obj) = delete; + + JSONObject(JSONObject && obj) + : JSONWriter(obj.state) + { + obj.state = 0; + } + ~JSONObject(); template @@ -165,10 +168,7 @@ public: { } - ~JSONPlaceholder() - { - assert(!first || std::uncaught_exception()); - } + ~JSONPlaceholder(); template void write(const T & v) diff --git a/src/libutil/lazy.hh b/src/libutil/lazy.hh new file mode 100644 index 00000000000..d073e486c2e --- /dev/null +++ b/src/libutil/lazy.hh @@ -0,0 +1,48 @@ +#include +#include +#include + +namespace nix { + +/* A helper class for lazily-initialized variables. + + Lazy var([]() { return value; }); + + declares a variable of type T that is initialized to 'value' (in a + thread-safe way) on first use, that is, when var() is first + called. If the initialiser code throws an exception, then all + subsequent calls to var() will rethrow that exception. */ +template +class Lazy +{ + + typedef std::function Init; + + Init init; + + std::once_flag done; + + T value; + + std::exception_ptr ex; + +public: + + Lazy(Init init) : init(init) + { } + + const T & operator () () + { + std::call_once(done, [&]() { + try { + value = init(); + } catch (...) { + ex = std::current_exception(); + } + }); + if (ex) std::rethrow_exception(ex); + return value; + } +}; + +} diff --git a/src/libutil/local.mk b/src/libutil/local.mk index cac5c8795db..16c1fa03ff8 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,6 +6,6 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) -libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) +libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context -libutil_LIBS = libformat +libutil_LIBS = libnixrust diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index d9e8d22d768..105fadb1584 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -1,22 +1,53 @@ #include "logging.hh" #include "util.hh" +#include +#include +#include + namespace nix { -Logger * logger = 0; +static thread_local ActivityId curActivity = 0; + +ActivityId getCurActivity() +{ + return curActivity; +} +void setCurActivity(const ActivityId activityId) +{ + curActivity = activityId; +} + +Logger * logger = makeSimpleLogger(true); + +void Logger::warn(const std::string & msg) +{ + log(lvlWarn, ANSI_YELLOW "warning:" ANSI_NORMAL " " + msg); +} + +void Logger::writeToStdout(std::string_view s) +{ + std::cout << s << "\n"; +} class SimpleLogger : public Logger { public: bool systemd, tty; + bool printBuildLogs; - SimpleLogger() + SimpleLogger(bool printBuildLogs) + : printBuildLogs(printBuildLogs) { systemd = getEnv("IN_SYSTEMD") == "1"; tty = isatty(STDERR_FILENO); } + bool isVerbose() override { + return printBuildLogs; + } + void log(Verbosity lvl, const FormatOrString & fs) override { if (lvl > verbosity) return; @@ -27,6 +58,7 @@ class SimpleLogger : public Logger char c; switch (lvl) { case lvlError: c = '3'; break; + case lvlWarn: c = '4'; break; case lvlInfo: c = '5'; break; case lvlTalkative: case lvlChatty: c = '6'; break; default: c = '7'; @@ -34,16 +66,35 @@ class SimpleLogger : public Logger prefix = std::string("<") + c + ">"; } - writeToStderr(prefix + (tty ? fs.s : filterANSIEscapes(fs.s)) + "\n"); + writeToStderr(prefix + filterANSIEscapes(fs.s, !tty) + "\n"); + } + + void logEI(const ErrorInfo & ei) override + { + std::stringstream oss; + oss << ei; + + log(ei.level, oss.str()); } - void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override + void startActivity(ActivityId act, Verbosity lvl, ActivityType type, + const std::string & s, const Fields & fields, ActivityId parent) + override { - log(lvl, fs); + if (lvl <= verbosity && !s.empty()) + log(lvl, s + "..."); } - void stopActivity(Activity & activity) override + void result(ActivityId act, ResultType type, const Fields & fields) override { + if (type == resBuildLogLine && printBuildLogs) { + auto lastLine = fields[0].s; + printError(lastLine); + } + else if (type == resPostBuildLogLine && printBuildLogs) { + auto lastLine = fields[0].s; + printError("post-build-hook: " + lastLine); + } } }; @@ -52,7 +103,7 @@ Verbosity verbosity = lvlInfo; void warnOnce(bool & haveWarned, const FormatOrString & fs) { if (!haveWarned) { - printError(format("warning: %1%") % fs.s); + warn(fs.s); haveWarned = true; } } @@ -69,9 +120,175 @@ void writeToStderr(const string & s) } } -Logger * makeDefaultLogger() +Logger * makeSimpleLogger(bool printBuildLogs) +{ + return new SimpleLogger(printBuildLogs); +} + +std::atomic nextId{(uint64_t) getpid() << 32}; + +Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type, + const std::string & s, const Logger::Fields & fields, ActivityId parent) + : logger(logger), id(nextId++) +{ + logger.startActivity(id, lvl, type, s, fields, parent); +} + +struct JSONLogger : Logger { + Logger & prevLogger; + + JSONLogger(Logger & prevLogger) : prevLogger(prevLogger) { } + + bool isVerbose() override { + return true; + } + + void addFields(nlohmann::json & json, const Fields & fields) + { + if (fields.empty()) return; + auto & arr = json["fields"] = nlohmann::json::array(); + for (auto & f : fields) + if (f.type == Logger::Field::tInt) + arr.push_back(f.i); + else if (f.type == Logger::Field::tString) + arr.push_back(f.s); + else + abort(); + } + + void write(const nlohmann::json & json) + { + prevLogger.log(lvlError, "@nix " + json.dump()); + } + + void log(Verbosity lvl, const FormatOrString & fs) override + { + nlohmann::json json; + json["action"] = "msg"; + json["level"] = lvl; + json["msg"] = fs.s; + write(json); + } + + void logEI(const ErrorInfo & ei) override + { + std::ostringstream oss; + oss << ei; + + nlohmann::json json; + json["action"] = "msg"; + json["level"] = ei.level; + json["msg"] = oss.str(); + + write(json); + } + + void startActivity(ActivityId act, Verbosity lvl, ActivityType type, + const std::string & s, const Fields & fields, ActivityId parent) override + { + nlohmann::json json; + json["action"] = "start"; + json["id"] = act; + json["level"] = lvl; + json["type"] = type; + json["text"] = s; + addFields(json, fields); + // FIXME: handle parent + write(json); + } + + void stopActivity(ActivityId act) override + { + nlohmann::json json; + json["action"] = "stop"; + json["id"] = act; + write(json); + } + + void result(ActivityId act, ResultType type, const Fields & fields) override + { + nlohmann::json json; + json["action"] = "result"; + json["id"] = act; + json["type"] = type; + addFields(json, fields); + write(json); + } +}; + +Logger * makeJSONLogger(Logger & prevLogger) +{ + return new JSONLogger(prevLogger); +} + +static Logger::Fields getFields(nlohmann::json & json) +{ + Logger::Fields fields; + for (auto & f : json) { + if (f.type() == nlohmann::json::value_t::number_unsigned) + fields.emplace_back(Logger::Field(f.get())); + else if (f.type() == nlohmann::json::value_t::string) + fields.emplace_back(Logger::Field(f.get())); + else throw Error("unsupported JSON type %d", (int) f.type()); + } + return fields; +} + +bool handleJSONLogMessage(const std::string & msg, + const Activity & act, std::map & activities, bool trusted) { - return new SimpleLogger(); + if (!hasPrefix(msg, "@nix ")) return false; + + try { + auto json = nlohmann::json::parse(std::string(msg, 5)); + + std::string action = json["action"]; + + if (action == "start") { + auto type = (ActivityType) json["type"]; + if (trusted || type == actFileTransfer) + activities.emplace(std::piecewise_construct, + std::forward_as_tuple(json["id"]), + std::forward_as_tuple(*logger, (Verbosity) json["level"], type, + json["text"], getFields(json["fields"]), act.id)); + } + + else if (action == "stop") + activities.erase((ActivityId) json["id"]); + + else if (action == "result") { + auto i = activities.find((ActivityId) json["id"]); + if (i != activities.end()) + i->second.result((ResultType) json["type"], getFields(json["fields"])); + } + + else if (action == "setPhase") { + std::string phase = json["phase"]; + act.result(resSetPhase, phase); + } + + else if (action == "msg") { + std::string msg = json["msg"]; + logger->log((Verbosity) json["level"], msg); + } + + } catch (std::exception & e) { + logError({ + .name = "Json log message", + .hint = hintfmt("bad log message from builder: %s", e.what()) + }); + } + + return true; +} + +Activity::~Activity() +{ + try { + logger.stopActivity(id); + } catch (...) { + ignoreException(); + } } } diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 3e6c4b54853..b1583ecede6 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -1,28 +1,65 @@ #pragma once #include "types.hh" +#include "error.hh" namespace nix { typedef enum { - lvlError = 0, - lvlInfo, - lvlTalkative, - lvlChatty, - lvlDebug, - lvlVomit -} Verbosity; + actUnknown = 0, + actCopyPath = 100, + actFileTransfer = 101, + actRealise = 102, + actCopyPaths = 103, + actBuilds = 104, + actBuild = 105, + actOptimiseStore = 106, + actVerifyPaths = 107, + actSubstitute = 108, + actQueryPathInfo = 109, + actPostBuildHook = 110, + actBuildWaiting = 111, +} ActivityType; -class Activity; +typedef enum { + resFileLinked = 100, + resBuildLogLine = 101, + resUntrustedPath = 102, + resCorruptedPath = 103, + resSetPhase = 104, + resProgress = 105, + resSetExpected = 106, + resPostBuildLogLine = 107, +} ResultType; + +typedef uint64_t ActivityId; class Logger { - friend class Activity; + friend struct Activity; public: + struct Field + { + // FIXME: use std::variant. + enum { tInt = 0, tString = 1 } type; + uint64_t i = 0; + std::string s; + Field(const std::string & s) : type(tString), s(s) { } + Field(const char * s) : type(tString), s(s) { } + Field(const uint64_t & i) : type(tInt), i(i) { } + }; + + typedef std::vector Fields; + virtual ~Logger() { } + virtual void stop() { }; + + // Whether the logger prints the whole build log + virtual bool isVerbose() { return false; } + virtual void log(Verbosity lvl, const FormatOrString & fs) = 0; void log(const FormatOrString & fs) @@ -30,45 +67,112 @@ public: log(lvlInfo, fs); } - virtual void setExpected(const std::string & label, uint64_t value = 1) { } - virtual void setProgress(const std::string & label, uint64_t value = 1) { } - virtual void incExpected(const std::string & label, uint64_t value = 1) { } - virtual void incProgress(const std::string & label, uint64_t value = 1) { } + virtual void logEI(const ErrorInfo &ei) = 0; + + void logEI(Verbosity lvl, ErrorInfo ei) + { + ei.level = lvl; + logEI(ei); + } + + virtual void warn(const std::string & msg); -private: + virtual void startActivity(ActivityId act, Verbosity lvl, ActivityType type, + const std::string & s, const Fields & fields, ActivityId parent) { }; - virtual void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) = 0; + virtual void stopActivity(ActivityId act) { }; - virtual void stopActivity(Activity & activity) = 0; + virtual void result(ActivityId act, ResultType type, const Fields & fields) { }; + virtual void writeToStdout(std::string_view s); + + template + inline void stdout(const std::string & fs, const Args & ... args) + { + boost::format f(fs); + formatHelper(f, args...); + writeToStdout(f.str()); + } }; -class Activity +ActivityId getCurActivity(); +void setCurActivity(const ActivityId activityId); + +struct Activity { -public: Logger & logger; - Activity(Logger & logger, Verbosity lvl, const FormatOrString & fs) - : logger(logger) + const ActivityId id; + + Activity(Logger & logger, Verbosity lvl, ActivityType type, const std::string & s = "", + const Logger::Fields & fields = {}, ActivityId parent = getCurActivity()); + + Activity(Logger & logger, ActivityType type, + const Logger::Fields & fields = {}, ActivityId parent = getCurActivity()) + : Activity(logger, lvlError, type, "", fields, parent) { }; + + Activity(const Activity & act) = delete; + + ~Activity(); + + void progress(uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) const + { result(resProgress, done, expected, running, failed); } + + void setExpected(ActivityType type2, uint64_t expected) const + { result(resSetExpected, type2, expected); } + + template + void result(ResultType type, const Args & ... args) const { - logger.startActivity(*this, lvl, fs); + Logger::Fields fields; + nop{(fields.emplace_back(Logger::Field(args)), 1)...}; + result(type, fields); } - ~Activity() + void result(ResultType type, const Logger::Fields & fields) const { - logger.stopActivity(*this); + logger.result(id, type, fields); } + + friend class Logger; +}; + +struct PushActivity +{ + const ActivityId prevAct; + PushActivity(ActivityId act) : prevAct(getCurActivity()) { setCurActivity(act); } + ~PushActivity() { setCurActivity(prevAct); } }; extern Logger * logger; -Logger * makeDefaultLogger(); +Logger * makeSimpleLogger(bool printBuildLogs = true); + +Logger * makeJSONLogger(Logger & prevLogger); + +bool handleJSONLogMessage(const std::string & msg, + const Activity & act, std::map & activities, + bool trusted); extern Verbosity verbosity; /* suppress msgs > this */ -/* Print a message if the current log level is at least the specified - level. Note that this has to be implemented as a macro to ensure - that the arguments are evaluated lazily. */ +/* Print a message with the standard ErrorInfo format. + In general, use these 'log' macros for reporting problems that may require user + intervention or that need more explanation. Use the 'print' macros for more + lightweight status messages. */ +#define logErrorInfo(level, errorInfo...) \ + do { \ + if (level <= nix::verbosity) { \ + logger->logEI(level, errorInfo); \ + } \ + } while (0) + +#define logError(errorInfo...) logErrorInfo(lvlError, errorInfo) +#define logWarning(errorInfo...) logErrorInfo(lvlWarn, errorInfo) + +/* Print a string message if the current log level is at least the specified + level. Note that this has to be implemented as a macro to ensure that the + arguments are evaluated lazily. */ #define printMsg(level, args...) \ do { \ if (level <= nix::verbosity) { \ @@ -78,9 +182,19 @@ extern Verbosity verbosity; /* suppress msgs > this */ #define printError(args...) printMsg(lvlError, args) #define printInfo(args...) printMsg(lvlInfo, args) +#define printTalkative(args...) printMsg(lvlTalkative, args) #define debug(args...) printMsg(lvlDebug, args) #define vomit(args...) printMsg(lvlVomit, args) +/* if verbosity >= lvlWarn, print a message with a yellow 'warning:' prefix. */ +template +inline void warn(const std::string & fs, const Args & ... args) +{ + boost::format f(fs); + formatHelper(f, args...); + logger->warn(f.str()); +} + void warnOnce(bool & haveWarned, const FormatOrString & fs); void writeToStderr(const string & s); diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh index 35983aa2c91..6ef4a3e067d 100644 --- a/src/libutil/lru-cache.hh +++ b/src/libutil/lru-cache.hh @@ -1,7 +1,9 @@ #pragma once +#include #include #include +#include namespace nix { @@ -11,7 +13,7 @@ class LRUCache { private: - size_t maxSize; + size_t capacity; // Stupid wrapper to get around circular dependency between Data // and LRU. @@ -27,14 +29,16 @@ private: public: - LRUCache(size_t maxSize) : maxSize(maxSize) { } + LRUCache(size_t capacity) : capacity(capacity) { } /* Insert or upsert an item in the cache. */ void upsert(const Key & key, const Value & value) { + if (capacity == 0) return; + erase(key); - if (data.size() >= maxSize) { + if (data.size() >= capacity) { /* Retire the oldest item. */ auto oldest = lru.begin(); data.erase(*oldest); @@ -61,18 +65,17 @@ public: /* Look up an item in the cache. If it exists, it becomes the most recently used item. */ - // FIXME: use boost::optional? - Value * get(const Key & key) + std::optional get(const Key & key) { auto i = data.find(key); - if (i == data.end()) return 0; + if (i == data.end()) return {}; /* Move this item to the back of the LRU list. */ lru.erase(i->second.first.it); auto j = lru.insert(lru.end(), i); i->second.first.it = j; - return &i->second.second; + return i->second.second; } size_t size() diff --git a/src/libutil/monitor-fd.hh b/src/libutil/monitor-fd.hh index 6f01ccd91a4..5ee0b88ef50 100644 --- a/src/libutil/monitor-fd.hh +++ b/src/libutil/monitor-fd.hh @@ -21,14 +21,29 @@ public: MonitorFdHup(int fd) { thread = std::thread([fd]() { - /* Wait indefinitely until a POLLHUP occurs. */ - struct pollfd fds[1]; - fds[0].fd = fd; - fds[0].events = 0; - if (poll(fds, 1, -1) == -1) abort(); // can't happen - assert(fds[0].revents & POLLHUP); - /* We got POLLHUP, so send an INT signal to the main thread. */ - kill(getpid(), SIGINT); + while (true) { + /* Wait indefinitely until a POLLHUP occurs. */ + struct pollfd fds[1]; + fds[0].fd = fd; + /* This shouldn't be necessary, but macOS doesn't seem to + like a zeroed out events field. + See rdar://37537852. + */ + fds[0].events = POLLHUP; + auto count = poll(fds, 1, -1); + if (count == -1) abort(); // can't happen + /* This shouldn't happen, but can on macOS due to a bug. + See rdar://37550628. + + This may eventually need a delay or further + coordination with the main thread if spinning proves + too harmful. + */ + if (count == 0) continue; + assert(fds[0].revents & POLLHUP); + triggerInterrupt(); + break; + } }); }; diff --git a/src/libutil/pool.hh b/src/libutil/pool.hh index f291cd57838..d49067bb95d 100644 --- a/src/libutil/pool.hh +++ b/src/libutil/pool.hh @@ -68,6 +68,22 @@ public: state_->max = max; } + void incCapacity() + { + auto state_(state.lock()); + state_->max++; + /* we could wakeup here, but this is only used when we're + * about to nest Pool usages, and we want to save the slot for + * the nested use if we can + */ + } + + void decCapacity() + { + auto state_(state.lock()); + state_->max--; + } + ~Pool() { auto state_(state.lock()); @@ -81,6 +97,7 @@ public: private: Pool & pool; std::shared_ptr r; + bool bad = false; friend Pool; @@ -96,7 +113,8 @@ public: if (!r) return; { auto state_(pool.state.lock()); - state_->idle.push_back(ref(r)); + if (!bad) + state_->idle.push_back(ref(r)); assert(state_->inUse); state_->inUse--; } @@ -105,6 +123,8 @@ public: R * operator -> () { return &*r; } R & operator * () { return *r; } + + void markBad() { bad = true; } }; Handle get() @@ -137,15 +157,31 @@ public: } catch (...) { auto state_(state.lock()); state_->inUse--; + wakeup.notify_one(); throw; } } - unsigned int count() + size_t count() { auto state_(state.lock()); return state_->idle.size() + state_->inUse; } + + size_t capacity() + { + return state.lock()->max; + } + + void flushBad() + { + auto state_(state.lock()); + std::vector> left; + for (auto & p : state_->idle) + if (validator(p)) + left.push_back(p); + std::swap(state_->idle, left); + } }; } diff --git a/src/libutil/rust-ffi.cc b/src/libutil/rust-ffi.cc new file mode 100644 index 00000000000..6f36b31922c --- /dev/null +++ b/src/libutil/rust-ffi.cc @@ -0,0 +1,22 @@ +#include "logging.hh" +#include "rust-ffi.hh" + +extern "C" std::exception_ptr * make_error(rust::StringSlice s) +{ + return new std::exception_ptr(std::make_exception_ptr(nix::Error(std::string(s.ptr, s.size)))); +} + +extern "C" void destroy_error(std::exception_ptr * ex) +{ + free(ex); +} + +namespace rust { + +std::ostream & operator << (std::ostream & str, const String & s) +{ + str << (std::string_view) s; + return str; +} + +} diff --git a/src/libutil/rust-ffi.hh b/src/libutil/rust-ffi.hh new file mode 100644 index 00000000000..228e2eead68 --- /dev/null +++ b/src/libutil/rust-ffi.hh @@ -0,0 +1,187 @@ +#pragma once + +#include "serialise.hh" + +#include +#include +#include + +namespace rust { + +typedef void (*DropFun)(void *); + +/* A Rust value of N bytes. It can be moved but not copied. When it + goes out of scope, the C++ destructor will run the drop + function. */ +template +struct Value +{ +protected: + + std::array raw; + + ~Value() + { + if (!isEvacuated()) { + drop(this); + evacuate(); + } + } + + // Must not be called directly. + Value() + { } + + Value(Value && other) + : raw(other.raw) + { + other.evacuate(); + } + + void operator =(Value && other) + { + if (!isEvacuated()) + drop(this); + raw = other.raw; + other.evacuate(); + } + +private: + + /* FIXME: optimize these (ideally in such a way that the compiler + can elide most calls to evacuate() / isEvacuated(). */ + inline void evacuate() + { + for (auto & i : raw) i = 0; + } + + inline bool isEvacuated() + { + for (auto & i : raw) + if (i != 0) return false; + return true; + } +}; + +/* A Rust vector. */ +template +struct Vec : Value<3 * sizeof(void *), drop> +{ + inline size_t size() const + { + return ((const size_t *) &this->raw)[2]; + } + + const T * data() const + { + return ((const T * *) &this->raw)[0]; + } +}; + +/* A Rust slice. */ +template +struct Slice +{ + const T * ptr; + size_t size; + + Slice(const T * ptr, size_t size) : ptr(ptr), size(size) + { + assert(ptr); + } +}; + +struct StringSlice : Slice +{ + StringSlice(const std::string & s): Slice(s.data(), s.size()) {} + explicit StringSlice(std::string_view s): Slice(s.data(), s.size()) {} + StringSlice(const char * s): Slice(s, strlen(s)) {} + + operator std::string_view() const + { + return std::string_view(ptr, size); + } +}; + +/* A Rust string. */ +struct String; + +extern "C" { + void ffi_String_new(StringSlice s, String * out); + void ffi_String_drop(void * s); +} + +struct String : Vec +{ + String() = delete; + + String(std::string_view s) + { + ffi_String_new(StringSlice(s), this); + } + + String(const char * s) + : String({s, std::strlen(s)}) + { + } + + operator std::string_view() const + { + return std::string_view(data(), size()); + } +}; + +std::ostream & operator << (std::ostream & str, const String & s); + +/* C++ representation of Rust's Result. */ +template +struct Result +{ + enum { Ok = 0, Err = 1, Uninit = 2 } tag; + + union { + T data; + std::exception_ptr * exc; + }; + + Result() : tag(Uninit) { }; // FIXME: remove + + Result(const Result &) = delete; + + Result(Result && other) + : tag(other.tag) + { + other.tag = Uninit; + if (tag == Ok) + data = std::move(other.data); + else if (tag == Err) + exc = other.exc; + } + + ~Result() + { + if (tag == Ok) + data.~T(); + else if (tag == Err) + free(exc); + else if (tag == Uninit) + ; + else + abort(); + } + + /* Rethrow the wrapped exception or return the wrapped value. */ + T unwrap() + { + if (tag == Ok) { + tag = Uninit; + return std::move(data); + } + else if (tag == Err) + std::rethrow_exception(*exc); + else + abort(); + } +}; + +} diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index a68f7a0fa8e..c8b71188fe0 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -5,6 +5,8 @@ #include #include +#include + namespace nix { @@ -50,7 +52,10 @@ size_t threshold = 256 * 1024 * 1024; static void warnLargeDump() { - printError("warning: dumping very large path (> 256 MiB); this may run out of memory"); + logWarning({ + .name = "Large path", + .description = "dumping very large path (> 256 MiB); this may run out of memory" + }); } @@ -67,7 +72,8 @@ void FdSink::write(const unsigned char * data, size_t len) try { writeFull(fd, data, len); } catch (SysError & e) { - _good = true; + _good = false; + throw; } } @@ -87,6 +93,23 @@ void Source::operator () (unsigned char * data, size_t len) } +std::string Source::drain() +{ + std::string s; + std::vector buf(8192); + while (true) { + size_t n; + try { + n = read(buf.data(), buf.size()); + s.append((char *) buf.data(), n); + } catch (EndOfFile &) { + break; + } + } + return s; +} + + size_t BufferedSource::read(unsigned char * data, size_t len) { if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]); @@ -113,7 +136,7 @@ size_t FdSource::readUnbuffered(unsigned char * data, size_t len) ssize_t n; do { checkInterrupt(); - n = ::read(fd, (char *) data, bufSize); + n = ::read(fd, (char *) data, len); } while (n == -1 && errno == EINTR); if (n == -1) { _good = false; throw SysError("reading from file"); } if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); } @@ -137,6 +160,61 @@ size_t StringSource::read(unsigned char * data, size_t len) } +#if BOOST_VERSION >= 106300 && BOOST_VERSION < 106600 +#error Coroutines are broken in this version of Boost! +#endif + +std::unique_ptr sinkToSource( + std::function fun, + std::function eof) +{ + struct SinkToSource : Source + { + typedef boost::coroutines2::coroutine coro_t; + + std::function fun; + std::function eof; + std::optional coro; + bool started = false; + + SinkToSource(std::function fun, std::function eof) + : fun(fun), eof(eof) + { + } + + std::string cur; + size_t pos = 0; + + size_t read(unsigned char * data, size_t len) override + { + if (!coro) + coro = coro_t::pull_type([&](coro_t::push_type & yield) { + LambdaSink sink([&](const unsigned char * data, size_t len) { + if (len) yield(std::string((const char *) data, len)); + }); + fun(sink); + }); + + if (!*coro) { eof(); abort(); } + + if (pos == cur.size()) { + if (!cur.empty()) (*coro)(); + cur = coro->get(); + pos = 0; + } + + auto n = std::min(cur.size() - pos, len); + memcpy(data, (unsigned char *) cur.data() + pos, n); + pos += n; + + return n; + } + }; + + return std::make_unique(fun, eof); +} + + void writePadding(size_t len, Sink & sink) { if (len % 8) { @@ -194,53 +272,24 @@ void readPadding(size_t len, Source & source) } -unsigned int readInt(Source & source) -{ - unsigned char buf[8]; - source(buf, sizeof(buf)); - if (buf[4] || buf[5] || buf[6] || buf[7]) - throw SerialisationError("implementation cannot deal with > 32-bit integers"); - return - buf[0] | - (buf[1] << 8) | - (buf[2] << 16) | - (buf[3] << 24); -} - - -unsigned long long readLongLong(Source & source) -{ - unsigned char buf[8]; - source(buf, sizeof(buf)); - return - ((unsigned long long) buf[0]) | - ((unsigned long long) buf[1] << 8) | - ((unsigned long long) buf[2] << 16) | - ((unsigned long long) buf[3] << 24) | - ((unsigned long long) buf[4] << 32) | - ((unsigned long long) buf[5] << 40) | - ((unsigned long long) buf[6] << 48) | - ((unsigned long long) buf[7] << 56); -} - - size_t readString(unsigned char * buf, size_t max, Source & source) { - size_t len = readInt(source); - if (len > max) throw Error("string is too long"); + auto len = readNum(source); + if (len > max) throw SerialisationError("string is too long"); source(buf, len); readPadding(len, source); return len; } -string readString(Source & source) +string readString(Source & source, size_t max) { - size_t len = readInt(source); - auto buf = std::make_unique(len); - source(buf.get(), len); + auto len = readNum(source); + if (len > max) throw SerialisationError("string is too long"); + std::string res(len, 0); + source((unsigned char*) res.data(), len); readPadding(len, source); - return string((char *) buf.get(), len); + return res; } Source & operator >> (Source & in, string & s) @@ -250,16 +299,9 @@ Source & operator >> (Source & in, string & s) } -Source & operator >> (Source & in, unsigned int & n) -{ - n = readInt(in); - return in; -} - - template T readStrings(Source & source) { - unsigned int count = readInt(source); + auto count = readNum(source); T ss; while (count--) ss.insert(ss.end(), readString(source)); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index f12f02543bc..a04118512fc 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -24,7 +24,7 @@ struct Sink /* A buffered abstract sink. */ -struct BufferedSink : Sink +struct BufferedSink : virtual Sink { size_t bufSize, bufPos; std::unique_ptr buffer; @@ -56,11 +56,13 @@ struct Source void operator () (unsigned char * data, size_t len); /* Store up to ‘len’ in the buffer pointed to by ‘data’, and - return the number of bytes stored. If blocks until at least + return the number of bytes stored. It blocks until at least one byte is available. */ virtual size_t read(unsigned char * data, size_t len) = 0; virtual bool good() { return true; } + + std::string drain(); }; @@ -75,10 +77,11 @@ struct BufferedSource : Source size_t read(unsigned char * data, size_t len) override; + bool hasData(); + +protected: /* Underlying read call, to be overridden. */ virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0; - - bool hasData(); }; @@ -92,7 +95,17 @@ struct FdSink : BufferedSink FdSink() : fd(-1) { } FdSink(int fd) : fd(fd) { } FdSink(FdSink&&) = default; - FdSink& operator=(FdSink&&) = default; + + FdSink& operator=(FdSink && s) + { + flush(); + fd = s.fd; + s.fd = -1; + warn = s.warn; + written = s.written; + return *this; + } + ~FdSink(); void write(const unsigned char * data, size_t len) override; @@ -112,8 +125,19 @@ struct FdSource : BufferedSource FdSource() : fd(-1) { } FdSource(int fd) : fd(fd) { } - size_t readUnbuffered(unsigned char * data, size_t len) override; + FdSource(FdSource&&) = default; + + FdSource& operator=(FdSource && s) + { + fd = s.fd; + s.fd = -1; + read = s.read; + return *this; + } + bool good() override; +protected: + size_t readUnbuffered(unsigned char * data, size_t len) override; private: bool _good = true; }; @@ -124,6 +148,9 @@ struct StringSink : Sink { ref s; StringSink() : s(make_ref()) { }; + explicit StringSink(const size_t reservedSize) : s(make_ref()) { + s->reserve(reservedSize); + }; StringSink(ref s) : s(s) { }; void operator () (const unsigned char * data, size_t len) override; }; @@ -139,6 +166,93 @@ struct StringSource : Source }; +/* Adapter class of a Source that saves all data read to `s'. */ +struct TeeSource : Source +{ + Source & orig; + ref data; + TeeSource(Source & orig) + : orig(orig), data(make_ref()) { } + size_t read(unsigned char * data, size_t len) + { + size_t n = orig.read(data, len); + this->data->append((const char *) data, n); + return n; + } +}; + +/* A reader that consumes the original Source until 'size'. */ +struct SizedSource : Source +{ + Source & orig; + size_t remain; + SizedSource(Source & orig, size_t size) + : orig(orig), remain(size) { } + size_t read(unsigned char * data, size_t len) + { + if (this->remain <= 0) { + throw EndOfFile("sized: unexpected end-of-file"); + } + len = std::min(len, this->remain); + size_t n = this->orig.read(data, len); + this->remain -= n; + return n; + } + + /* Consume the original source until no remain data is left to consume. */ + size_t drainAll() + { + std::vector buf(8192); + size_t sum = 0; + while (this->remain > 0) { + size_t n = read(buf.data(), buf.size()); + sum += n; + } + return sum; + } +}; + +/* Convert a function into a sink. */ +struct LambdaSink : Sink +{ + typedef std::function lambda_t; + + lambda_t lambda; + + LambdaSink(const lambda_t & lambda) : lambda(lambda) { } + + virtual void operator () (const unsigned char * data, size_t len) + { + lambda(data, len); + } +}; + + +/* Convert a function into a source. */ +struct LambdaSource : Source +{ + typedef std::function lambda_t; + + lambda_t lambda; + + LambdaSource(const lambda_t & lambda) : lambda(lambda) { } + + size_t read(unsigned char * data, size_t len) override + { + return lambda(data, len); + } +}; + + +/* Convert a function that feeds data into a Sink into a Source. The + Source executes the function as a coroutine. */ +std::unique_ptr sinkToSource( + std::function fun, + std::function eof = []() { + throw EndOfFile("coroutine has finished"); + }); + + void writePadding(size_t len, Sink & sink); void writeString(const unsigned char * buf, size_t len, Sink & sink); @@ -152,7 +266,7 @@ inline Sink & operator << (Sink & sink, uint64_t n) buf[4] = (n >> 32) & 0xff; buf[5] = (n >> 40) & 0xff; buf[6] = (n >> 48) & 0xff; - buf[7] = (n >> 56) & 0xff; + buf[7] = (unsigned char) (n >> 56) & 0xff; sink(buf, sizeof(buf)); return sink; } @@ -162,18 +276,64 @@ Sink & operator << (Sink & sink, const Strings & s); Sink & operator << (Sink & sink, const StringSet & s); +MakeError(SerialisationError, Error); + + +template +T readNum(Source & source) +{ + unsigned char buf[8]; + source(buf, sizeof(buf)); + + uint64_t n = + ((unsigned long long) buf[0]) | + ((unsigned long long) buf[1] << 8) | + ((unsigned long long) buf[2] << 16) | + ((unsigned long long) buf[3] << 24) | + ((unsigned long long) buf[4] << 32) | + ((unsigned long long) buf[5] << 40) | + ((unsigned long long) buf[6] << 48) | + ((unsigned long long) buf[7] << 56); + + if (n > std::numeric_limits::max()) + throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name()); + + return (T) n; +} + + +inline unsigned int readInt(Source & source) +{ + return readNum(source); +} + + +inline uint64_t readLongLong(Source & source) +{ + return readNum(source); +} + + void readPadding(size_t len, Source & source); -unsigned int readInt(Source & source); -unsigned long long readLongLong(Source & source); size_t readString(unsigned char * buf, size_t max, Source & source); -string readString(Source & source); +string readString(Source & source, size_t max = std::numeric_limits::max()); template T readStrings(Source & source); Source & operator >> (Source & in, string & s); -Source & operator >> (Source & in, unsigned int & n); +template +Source & operator >> (Source & in, T & n) +{ + n = readNum(in); + return in; +} -MakeError(SerialisationError, Error) +template +Source & operator >> (Source & in, bool & b) +{ + b = readNum(in); + return in; +} } diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh index 2aa074299b2..e1d591d77a8 100644 --- a/src/libutil/sync.hh +++ b/src/libutil/sync.hh @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -33,6 +34,7 @@ public: Sync() { } Sync(const T & data) : data(data) { } + Sync(T && data) noexcept : data(std::move(data)) { } class Lock { @@ -55,11 +57,11 @@ public: } template - void wait_for(std::condition_variable & cv, + std::cv_status wait_for(std::condition_variable & cv, const std::chrono::duration & duration) { assert(s); - cv.wait_for(lk, duration); + return cv.wait_for(lk, duration); } template diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc new file mode 100644 index 00000000000..c4d8a4f919f --- /dev/null +++ b/src/libutil/tarfile.cc @@ -0,0 +1,125 @@ +#include +#include + +#include "serialise.hh" + +namespace nix { + +struct TarArchive { + struct archive * archive; + Source * source; + std::vector buffer; + + void check(int err, const char * reason = "failed to extract archive: %s") + { + if (err == ARCHIVE_EOF) + throw EndOfFile("reached end of archive"); + else if (err != ARCHIVE_OK) + throw Error(reason, archive_error_string(this->archive)); + } + + TarArchive(Source & source) : buffer(4096) + { + this->archive = archive_read_new(); + this->source = &source; + + archive_read_support_filter_all(archive); + archive_read_support_format_all(archive); + check(archive_read_open(archive, + (void *)this, + TarArchive::callback_open, + TarArchive::callback_read, + TarArchive::callback_close), + "failed to open archive: %s"); + } + + TarArchive(const Path & path) + { + this->archive = archive_read_new(); + + archive_read_support_filter_all(archive); + archive_read_support_format_all(archive); + check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s"); + } + + TarArchive(const TarArchive &) = delete; + + void close() + { + check(archive_read_close(archive), "failed to close archive: %s"); + } + + ~TarArchive() + { + if (this->archive) archive_read_free(this->archive); + } + +private: + + static int callback_open(struct archive *, void * self) { + return ARCHIVE_OK; + } + + static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer) + { + auto self = (TarArchive *)_self; + *buffer = self->buffer.data(); + + try { + return self->source->read(self->buffer.data(), 4096); + } catch (EndOfFile &) { + return 0; + } catch (std::exception & err) { + archive_set_error(archive, EIO, "source threw exception: %s", err.what()); + return -1; + } + } + + static int callback_close(struct archive *, void * self) { + return ARCHIVE_OK; + } +}; + +static void extract_archive(TarArchive & archive, const Path & destDir) +{ + int flags = ARCHIVE_EXTRACT_FFLAGS + | ARCHIVE_EXTRACT_PERM + | ARCHIVE_EXTRACT_TIME + | ARCHIVE_EXTRACT_SECURE_SYMLINKS + | ARCHIVE_EXTRACT_SECURE_NODOTDOT; + + for (;;) { + struct archive_entry * entry; + int r = archive_read_next_header(archive.archive, &entry); + if (r == ARCHIVE_EOF) break; + else if (r == ARCHIVE_WARN) + warn(archive_error_string(archive.archive)); + else + archive.check(r); + + archive_entry_set_pathname(entry, + (destDir + "/" + archive_entry_pathname(entry)).c_str()); + + archive.check(archive_read_extract(archive.archive, entry, flags)); + } + + archive.close(); +} + +void unpackTarfile(Source & source, const Path & destDir) +{ + auto archive = TarArchive(source); + + createDirs(destDir); + extract_archive(archive, destDir); +} + +void unpackTarfile(const Path & tarFile, const Path & destDir) +{ + auto archive = TarArchive(tarFile); + + createDirs(destDir); + extract_archive(archive, destDir); +} + +} diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh new file mode 100644 index 00000000000..89a024f1d10 --- /dev/null +++ b/src/libutil/tarfile.hh @@ -0,0 +1,9 @@ +#include "serialise.hh" + +namespace nix { + +void unpackTarfile(Source & source, const Path & destDir); + +void unpackTarfile(const Path & tarFile, const Path & destDir); + +} diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc new file mode 100644 index 00000000000..74c59fd315b --- /dev/null +++ b/src/libutil/tests/config.cc @@ -0,0 +1,264 @@ +#include "json.hh" +#include "config.hh" +#include "args.hh" + +#include +#include + +namespace nix { + + /* ---------------------------------------------------------------------------- + * Config + * --------------------------------------------------------------------------*/ + + TEST(Config, setUndefinedSetting) { + Config config; + ASSERT_EQ(config.set("undefined-key", "value"), false); + } + + TEST(Config, setDefinedSetting) { + Config config; + std::string value; + Setting foo{&config, value, "name-of-the-setting", "description"}; + ASSERT_EQ(config.set("name-of-the-setting", "value"), true); + } + + TEST(Config, getDefinedSetting) { + Config config; + std::string value; + std::map settings; + Setting foo{&config, value, "name-of-the-setting", "description"}; + + config.getSettings(settings, /* overridenOnly = */ false); + const auto iter = settings.find("name-of-the-setting"); + ASSERT_NE(iter, settings.end()); + ASSERT_EQ(iter->second.value, ""); + ASSERT_EQ(iter->second.description, "description"); + } + + TEST(Config, getDefinedOverridenSettingNotSet) { + Config config; + std::string value; + std::map settings; + Setting foo{&config, value, "name-of-the-setting", "description"}; + + config.getSettings(settings, /* overridenOnly = */ true); + const auto e = settings.find("name-of-the-setting"); + ASSERT_EQ(e, settings.end()); + } + + TEST(Config, getDefinedSettingSet1) { + Config config; + std::string value; + std::map settings; + Setting setting{&config, value, "name-of-the-setting", "description"}; + + setting.assign("value"); + + config.getSettings(settings, /* overridenOnly = */ false); + const auto iter = settings.find("name-of-the-setting"); + ASSERT_NE(iter, settings.end()); + ASSERT_EQ(iter->second.value, "value"); + ASSERT_EQ(iter->second.description, "description"); + } + + TEST(Config, getDefinedSettingSet2) { + Config config; + std::map settings; + Setting setting{&config, "", "name-of-the-setting", "description"}; + + ASSERT_TRUE(config.set("name-of-the-setting", "value")); + + config.getSettings(settings, /* overridenOnly = */ false); + const auto e = settings.find("name-of-the-setting"); + ASSERT_NE(e, settings.end()); + ASSERT_EQ(e->second.value, "value"); + ASSERT_EQ(e->second.description, "description"); + } + + TEST(Config, addSetting) { + class TestSetting : public AbstractSetting { + public: + TestSetting() : AbstractSetting("test", "test", {}) {} + void set(const std::string & value) {} + std::string to_string() const { return {}; } + }; + + Config config; + TestSetting setting; + + ASSERT_FALSE(config.set("test", "value")); + config.addSetting(&setting); + ASSERT_TRUE(config.set("test", "value")); + } + + TEST(Config, withInitialValue) { + const StringMap initials = { + { "key", "value" }, + }; + Config config(initials); + + { + std::map settings; + config.getSettings(settings, /* overridenOnly = */ false); + ASSERT_EQ(settings.find("key"), settings.end()); + } + + Setting setting{&config, "default-value", "key", "description"}; + + { + std::map settings; + config.getSettings(settings, /* overridenOnly = */ false); + ASSERT_EQ(settings["key"].value, "value"); + } + } + + TEST(Config, resetOverriden) { + Config config; + config.resetOverriden(); + } + + TEST(Config, resetOverridenWithSetting) { + Config config; + Setting setting{&config, "", "name-of-the-setting", "description"}; + + { + std::map settings; + + setting.set("foo"); + ASSERT_EQ(setting.get(), "foo"); + config.getSettings(settings, /* overridenOnly = */ true); + ASSERT_TRUE(settings.empty()); + } + + { + std::map settings; + + setting.override("bar"); + ASSERT_TRUE(setting.overriden); + ASSERT_EQ(setting.get(), "bar"); + config.getSettings(settings, /* overridenOnly = */ true); + ASSERT_FALSE(settings.empty()); + } + + { + std::map settings; + + config.resetOverriden(); + ASSERT_FALSE(setting.overriden); + config.getSettings(settings, /* overridenOnly = */ true); + ASSERT_TRUE(settings.empty()); + } + } + + TEST(Config, toJSONOnEmptyConfig) { + std::stringstream out; + { // Scoped to force the destructor of JSONObject to write the final `}` + JSONObject obj(out); + Config config; + config.toJSON(obj); + } + + ASSERT_EQ(out.str(), "{}"); + } + + TEST(Config, toJSONOnNonEmptyConfig) { + std::stringstream out; + { // Scoped to force the destructor of JSONObject to write the final `}` + JSONObject obj(out); + + Config config; + std::map settings; + Setting setting{&config, "", "name-of-the-setting", "description"}; + setting.assign("value"); + + config.toJSON(obj); + } + ASSERT_EQ(out.str(), R"#({"name-of-the-setting":{"description":"description","value":"value"}})#"); + } + + TEST(Config, setSettingAlias) { + Config config; + Setting setting{&config, "", "some-int", "best number", { "another-int" }}; + ASSERT_TRUE(config.set("some-int", "1")); + ASSERT_EQ(setting.get(), "1"); + ASSERT_TRUE(config.set("another-int", "2")); + ASSERT_EQ(setting.get(), "2"); + ASSERT_TRUE(config.set("some-int", "3")); + ASSERT_EQ(setting.get(), "3"); + } + + /* FIXME: The reapplyUnknownSettings method doesn't seem to do anything + * useful (these days). Whenever we add a new setting to Config the + * unknown settings are always considered. In which case is this function + * actually useful? Is there some way to register a Setting without calling + * addSetting? */ + TEST(Config, DISABLED_reapplyUnknownSettings) { + Config config; + ASSERT_FALSE(config.set("name-of-the-setting", "unknownvalue")); + Setting setting{&config, "default", "name-of-the-setting", "description"}; + ASSERT_EQ(setting.get(), "default"); + config.reapplyUnknownSettings(); + ASSERT_EQ(setting.get(), "unknownvalue"); + } + + TEST(Config, applyConfigEmpty) { + Config config; + std::map settings; + config.applyConfig(""); + config.getSettings(settings); + ASSERT_TRUE(settings.empty()); + } + + TEST(Config, applyConfigEmptyWithComment) { + Config config; + std::map settings; + config.applyConfig("# just a comment"); + config.getSettings(settings); + ASSERT_TRUE(settings.empty()); + } + + TEST(Config, applyConfigAssignment) { + Config config; + std::map settings; + Setting setting{&config, "", "name-of-the-setting", "description"}; + config.applyConfig( + "name-of-the-setting = value-from-file #useful comment\n" + "# name-of-the-setting = foo\n" + ); + config.getSettings(settings); + ASSERT_FALSE(settings.empty()); + ASSERT_EQ(settings["name-of-the-setting"].value, "value-from-file"); + } + + TEST(Config, applyConfigWithReassignedSetting) { + Config config; + std::map settings; + Setting setting{&config, "", "name-of-the-setting", "description"}; + config.applyConfig( + "name-of-the-setting = first-value\n" + "name-of-the-setting = second-value\n" + ); + config.getSettings(settings); + ASSERT_FALSE(settings.empty()); + ASSERT_EQ(settings["name-of-the-setting"].value, "second-value"); + } + + TEST(Config, applyConfigFailsOnMissingIncludes) { + Config config; + std::map settings; + Setting setting{&config, "", "name-of-the-setting", "description"}; + + ASSERT_THROW(config.applyConfig( + "name-of-the-setting = value-from-file\n" + "# name-of-the-setting = foo\n" + "include /nix/store/does/not/exist.nix" + ), Error); + } + + TEST(Config, applyConfigInvalidThrows) { + Config config; + ASSERT_THROW(config.applyConfig("value == key"), UsageError); + ASSERT_THROW(config.applyConfig("value "), UsageError); + } +} diff --git a/src/libutil/tests/hash.cc b/src/libutil/tests/hash.cc new file mode 100644 index 00000000000..5334b046e5d --- /dev/null +++ b/src/libutil/tests/hash.cc @@ -0,0 +1,80 @@ +#include "hash.hh" +#include + +namespace nix { + + /* ---------------------------------------------------------------------------- + * hashString + * --------------------------------------------------------------------------*/ + + TEST(hashString, testKnownMD5Hashes1) { + // values taken from: https://tools.ietf.org/html/rfc1321 + auto s1 = ""; + auto hash = hashString(HashType::htMD5, s1); + ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e"); + } + + TEST(hashString, testKnownMD5Hashes2) { + // values taken from: https://tools.ietf.org/html/rfc1321 + auto s2 = "abc"; + auto hash = hashString(HashType::htMD5, s2); + ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72"); + } + + TEST(hashString, testKnownSHA1Hashes1) { + // values taken from: https://tools.ietf.org/html/rfc3174 + auto s = "abc"; + auto hash = hashString(HashType::htSHA1, s); + ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); + } + + TEST(hashString, testKnownSHA1Hashes2) { + // values taken from: https://tools.ietf.org/html/rfc3174 + auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; + auto hash = hashString(HashType::htSHA1, s); + ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); + } + + TEST(hashString, testKnownSHA256Hashes1) { + // values taken from: https://tools.ietf.org/html/rfc4634 + auto s = "abc"; + + auto hash = hashString(HashType::htSHA256, s); + ASSERT_EQ(hash.to_string(Base::Base16, true), + "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + } + + TEST(hashString, testKnownSHA256Hashes2) { + // values taken from: https://tools.ietf.org/html/rfc4634 + auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; + auto hash = hashString(HashType::htSHA256, s); + ASSERT_EQ(hash.to_string(Base::Base16, true), + "sha256:248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); + } + + TEST(hashString, testKnownSHA512Hashes1) { + // values taken from: https://tools.ietf.org/html/rfc4634 + auto s = "abc"; + auto hash = hashString(HashType::htSHA512, s); + ASSERT_EQ(hash.to_string(Base::Base16, true), + "sha512:ddaf35a193617abacc417349ae20413112e6fa4e89a9" + "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd" + "454d4423643ce80e2a9ac94fa54ca49f"); + } + + TEST(hashString, testKnownSHA512Hashes2) { + // values taken from: https://tools.ietf.org/html/rfc4634 + auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; + + auto hash = hashString(HashType::htSHA512, s); + ASSERT_EQ(hash.to_string(Base::Base16, true), + "sha512:8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa1" + "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a" + "c7d329eeb6dd26545e96e55b874be909"); + } + + TEST(hashString, hashingWithUnknownAlgoExits) { + auto s = "unknown"; + ASSERT_DEATH(hashString(HashType::htUnknown, s), ""); + } +} diff --git a/src/libutil/tests/json.cc b/src/libutil/tests/json.cc new file mode 100644 index 00000000000..dea73f53a91 --- /dev/null +++ b/src/libutil/tests/json.cc @@ -0,0 +1,193 @@ +#include "json.hh" +#include +#include + +namespace nix { + + /* ---------------------------------------------------------------------------- + * toJSON + * --------------------------------------------------------------------------*/ + + TEST(toJSON, quotesCharPtr) { + const char* input = "test"; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "\"test\""); + } + + TEST(toJSON, quotesStdString) { + std::string input = "test"; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "\"test\""); + } + + TEST(toJSON, convertsNullptrtoNull) { + auto input = nullptr; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "null"); + } + + TEST(toJSON, convertsNullToNull) { + const char* input = 0; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "null"); + } + + + TEST(toJSON, convertsFloat) { + auto input = 1.024f; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "1.024"); + } + + TEST(toJSON, convertsDouble) { + const double input = 1.024; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "1.024"); + } + + TEST(toJSON, convertsBool) { + auto input = false; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "false"); + } + + TEST(toJSON, quotesTab) { + std::stringstream out; + toJSON(out, "\t"); + + ASSERT_EQ(out.str(), "\"\\t\""); + } + + TEST(toJSON, quotesNewline) { + std::stringstream out; + toJSON(out, "\n"); + + ASSERT_EQ(out.str(), "\"\\n\""); + } + + TEST(toJSON, quotesCreturn) { + std::stringstream out; + toJSON(out, "\r"); + + ASSERT_EQ(out.str(), "\"\\r\""); + } + + TEST(toJSON, quotesCreturnNewLine) { + std::stringstream out; + toJSON(out, "\r\n"); + + ASSERT_EQ(out.str(), "\"\\r\\n\""); + } + + TEST(toJSON, quotesDoublequotes) { + std::stringstream out; + toJSON(out, "\""); + + ASSERT_EQ(out.str(), "\"\\\"\""); + } + + TEST(toJSON, substringEscape) { + std::stringstream out; + const char *s = "foo\t"; + toJSON(out, s+3, s + strlen(s)); + + ASSERT_EQ(out.str(), "\"\\t\""); + } + + /* ---------------------------------------------------------------------------- + * JSONObject + * --------------------------------------------------------------------------*/ + + TEST(JSONObject, emptyObject) { + std::stringstream out; + { + JSONObject t(out); + } + ASSERT_EQ(out.str(), "{}"); + } + + TEST(JSONObject, objectWithList) { + std::stringstream out; + { + JSONObject t(out); + auto l = t.list("list"); + l.elem("element"); + } + ASSERT_EQ(out.str(), R"#({"list":["element"]})#"); + } + + TEST(JSONObject, objectWithListIndent) { + std::stringstream out; + { + JSONObject t(out, true); + auto l = t.list("list"); + l.elem("element"); + } + ASSERT_EQ(out.str(), +R"#({ + "list": [ + "element" + ] +})#"); + } + + TEST(JSONObject, objectWithPlaceholderAndList) { + std::stringstream out; + { + JSONObject t(out); + auto l = t.placeholder("list"); + l.list().elem("element"); + } + + ASSERT_EQ(out.str(), R"#({"list":["element"]})#"); + } + + TEST(JSONObject, objectWithPlaceholderAndObject) { + std::stringstream out; + { + JSONObject t(out); + auto l = t.placeholder("object"); + l.object().attr("key", "value"); + } + + ASSERT_EQ(out.str(), R"#({"object":{"key":"value"}})#"); + } + + /* ---------------------------------------------------------------------------- + * JSONList + * --------------------------------------------------------------------------*/ + + TEST(JSONList, empty) { + std::stringstream out; + { + JSONList l(out); + } + ASSERT_EQ(out.str(), R"#([])#"); + } + + TEST(JSONList, withElements) { + std::stringstream out; + { + JSONList l(out); + l.elem("one"); + l.object(); + l.placeholder().write("three"); + } + ASSERT_EQ(out.str(), R"#(["one",{},"three"])#"); + } +} + diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk new file mode 100644 index 00000000000..815e185600e --- /dev/null +++ b/src/libutil/tests/local.mk @@ -0,0 +1,15 @@ +check: libutil-tests_RUN + +programs += libutil-tests + +libutil-tests_DIR := $(d) + +libutil-tests_INSTALL_DIR := + +libutil-tests_SOURCES := $(wildcard $(d)/*.cc) + +libutil-tests_CXXFLAGS += -I src/libutil -I src/libexpr + +libutil-tests_LIBS = libutil + +libutil-tests_LDFLAGS := $(GTEST_LIBS) diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc new file mode 100644 index 00000000000..fbdc91253b5 --- /dev/null +++ b/src/libutil/tests/logging.cc @@ -0,0 +1,255 @@ +#include "logging.hh" +#include "nixexpr.hh" +#include "util.hh" + +#include + +namespace nix { + + /* ---------------------------------------------------------------------------- + * logEI + * --------------------------------------------------------------------------*/ + + TEST(logEI, catpuresBasicProperties) { + + MakeError(TestError, Error); + ErrorInfo::programName = std::optional("error-unit-test"); + + try { + throw TestError("an error for testing purposes"); + } catch (Error &e) { + testing::internal::CaptureStderr(); + logger->logEI(e.info()); + auto str = testing::internal::GetCapturedStderr(); + + ASSERT_STREQ(str.c_str(),"\x1B[31;1merror:\x1B[0m\x1B[34;1m --- TestError ------------------------------------ error-unit-test\x1B[0m\nan error for testing purposes\n"); + } + } + + TEST(logEI, appendingHintsToPreviousError) { + + MakeError(TestError, Error); + ErrorInfo::programName = std::optional("error-unit-test"); + + try { + auto e = Error("initial error"); + throw TestError(e.info()); + } catch (Error &e) { + ErrorInfo ei = e.info(); + ei.hint = hintfmt("%s; subsequent error message.", normaltxt(e.info().hint ? e.info().hint->str() : "")); + + testing::internal::CaptureStderr(); + logger->logEI(ei); + auto str = testing::internal::GetCapturedStderr(); + + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- TestError ------------------------------------ error-unit-test\x1B[0m\n\x1B[33;1m\x1B[0minitial error\x1B[0m; subsequent error message.\n"); + } + + } + + TEST(logEI, picksUpSysErrorExitCode) { + + MakeError(TestError, Error); + ErrorInfo::programName = std::optional("error-unit-test"); + + try { + auto x = readFile(-1); + } + catch (SysError &e) { + testing::internal::CaptureStderr(); + logError(e.info()); + auto str = testing::internal::GetCapturedStderr(); + + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError ------------------------------------- error-unit-test\x1B[0m\n\x1B[33;1m\x1B[0mstatting file\x1B[0m: \x1B[33;1mBad file descriptor\x1B[0m\n"); + + } + } + + TEST(logEI, loggingErrorOnInfoLevel) { + testing::internal::CaptureStderr(); + + logger->logEI({ .level = lvlInfo, + .name = "Info name", + .description = "Info description", + }); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[32;1minfo:\x1B[0m\x1B[34;1m --- Info name ------------------------------------- error-unit-test\x1B[0m\nInfo description\n"); + } + + TEST(logEI, loggingErrorOnTalkativeLevel) { + verbosity = lvlTalkative; + + testing::internal::CaptureStderr(); + + logger->logEI({ .level = lvlTalkative, + .name = "Talkative name", + .description = "Talkative description", + }); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[32;1mtalk:\x1B[0m\x1B[34;1m --- Talkative name -------------------------------- error-unit-test\x1B[0m\nTalkative description\n"); + } + + TEST(logEI, loggingErrorOnChattyLevel) { + verbosity = lvlChatty; + + testing::internal::CaptureStderr(); + + logger->logEI({ .level = lvlChatty, + .name = "Chatty name", + .description = "Talkative description", + }); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[32;1mchat:\x1B[0m\x1B[34;1m --- Chatty name ----------------------------------- error-unit-test\x1B[0m\nTalkative description\n"); + } + + TEST(logEI, loggingErrorOnDebugLevel) { + verbosity = lvlDebug; + + testing::internal::CaptureStderr(); + + logger->logEI({ .level = lvlDebug, + .name = "Debug name", + .description = "Debug description", + }); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[33;1mdebug:\x1B[0m\x1B[34;1m --- Debug name ----------------------------------- error-unit-test\x1B[0m\nDebug description\n"); + } + + TEST(logEI, loggingErrorOnVomitLevel) { + verbosity = lvlVomit; + + testing::internal::CaptureStderr(); + + logger->logEI({ .level = lvlVomit, + .name = "Vomit name", + .description = "Vomit description", + }); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[32;1mvomit:\x1B[0m\x1B[34;1m --- Vomit name ----------------------------------- error-unit-test\x1B[0m\nVomit description\n"); + } + + /* ---------------------------------------------------------------------------- + * logError + * --------------------------------------------------------------------------*/ + + + TEST(logError, logErrorWithoutHintOrCode) { + testing::internal::CaptureStderr(); + + logError({ + .name = "name", + .description = "error description", + }); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- name ----------------------------------------- error-unit-test\x1B[0m\nerror description\n"); + } + + TEST(logError, logErrorWithPreviousAndNextLinesOfCode) { + SymbolTable testTable; + auto problem_file = testTable.create("myfile.nix"); + + testing::internal::CaptureStderr(); + + logError({ + .name = "error name", + .description = "error with code lines", + .hint = hintfmt("this hint has %1% templated %2%!!", + "yellow", + "values"), + .nixCode = NixCode { + .errPos = Pos(problem_file, 40, 13), + .prevLineOfCode = "previous line of code", + .errLineOfCode = "this is the problem line of code", + .nextLineOfCode = "next line of code", + }}); + + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name ----------------------------------- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nerror with code lines\n\n 39| previous line of code\n 40| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 41| next line of code\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); + } + + TEST(logError, logErrorWithoutLinesOfCode) { + SymbolTable testTable; + auto problem_file = testTable.create("myfile.nix"); + testing::internal::CaptureStderr(); + + logError({ + .name = "error name", + .description = "error without any code lines.", + .hint = hintfmt("this hint has %1% templated %2%!!", + "yellow", + "values"), + .nixCode = NixCode { + .errPos = Pos(problem_file, 40, 13) + }}); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name ----------------------------------- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); + } + + TEST(logError, logErrorWithOnlyHintAndName) { + SymbolTable testTable; + auto problem_file = testTable.create("myfile.nix"); + testing::internal::CaptureStderr(); + + logError({ + .name = "error name", + .hint = hintfmt("hint %1%", "only"), + .nixCode = NixCode { + .errPos = Pos(problem_file, 40, 13) + }}); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name ----------------------------------- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nhint \x1B[33;1monly\x1B[0m\n"); + + } + + /* ---------------------------------------------------------------------------- + * logWarning + * --------------------------------------------------------------------------*/ + + TEST(logWarning, logWarningWithNameDescriptionAndHint) { + testing::internal::CaptureStderr(); + + logWarning({ + .name = "name", + .description = "error description", + .hint = hintfmt("there was a %1%", "warning"), + }); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- name --------------------------------------- error-unit-test\x1B[0m\nerror description\n\nthere was a \x1B[33;1mwarning\x1B[0m\n"); + } + + TEST(logWarning, logWarningWithFileLineNumAndCode) { + + SymbolTable testTable; + auto problem_file = testTable.create("myfile.nix"); + + testing::internal::CaptureStderr(); + + logWarning({ + .name = "warning name", + .description = "warning description", + .hint = hintfmt("this hint has %1% templated %2%!!", + "yellow", + "values"), + .nixCode = NixCode { + .errPos = Pos(problem_file, 40, 13), + .prevLineOfCode = std::nullopt, + .errLineOfCode = "this is the problem line of code", + .nextLineOfCode = std::nullopt + }}); + + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- warning name ------------------------------- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nwarning description\n\n 40| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); + } + +} diff --git a/src/libutil/tests/lru-cache.cc b/src/libutil/tests/lru-cache.cc new file mode 100644 index 00000000000..091d3d5ede1 --- /dev/null +++ b/src/libutil/tests/lru-cache.cc @@ -0,0 +1,130 @@ +#include "lru-cache.hh" +#include + +namespace nix { + + /* ---------------------------------------------------------------------------- + * size + * --------------------------------------------------------------------------*/ + + TEST(LRUCache, sizeOfEmptyCacheIsZero) { + LRUCache c(10); + ASSERT_EQ(c.size(), 0); + } + + TEST(LRUCache, sizeOfSingleElementCacheIsOne) { + LRUCache c(10); + c.upsert("foo", "bar"); + ASSERT_EQ(c.size(), 1); + } + + /* ---------------------------------------------------------------------------- + * upsert / get + * --------------------------------------------------------------------------*/ + + TEST(LRUCache, getFromEmptyCache) { + LRUCache c(10); + auto val = c.get("x"); + ASSERT_EQ(val.has_value(), false); + } + + TEST(LRUCache, getExistingValue) { + LRUCache c(10); + c.upsert("foo", "bar"); + auto val = c.get("foo"); + ASSERT_EQ(val, "bar"); + } + + TEST(LRUCache, getNonExistingValueFromNonEmptyCache) { + LRUCache c(10); + c.upsert("foo", "bar"); + auto val = c.get("another"); + ASSERT_EQ(val.has_value(), false); + } + + TEST(LRUCache, upsertOnZeroCapacityCache) { + LRUCache c(0); + c.upsert("foo", "bar"); + auto val = c.get("foo"); + ASSERT_EQ(val.has_value(), false); + } + + TEST(LRUCache, updateExistingValue) { + LRUCache c(1); + c.upsert("foo", "bar"); + + auto val = c.get("foo"); + ASSERT_EQ(val.value_or("error"), "bar"); + ASSERT_EQ(c.size(), 1); + + c.upsert("foo", "changed"); + val = c.get("foo"); + ASSERT_EQ(val.value_or("error"), "changed"); + ASSERT_EQ(c.size(), 1); + } + + TEST(LRUCache, overwriteOldestWhenCapacityIsReached) { + LRUCache c(3); + c.upsert("one", "eins"); + c.upsert("two", "zwei"); + c.upsert("three", "drei"); + + ASSERT_EQ(c.size(), 3); + ASSERT_EQ(c.get("one").value_or("error"), "eins"); + + // exceed capacity + c.upsert("another", "whatever"); + + ASSERT_EQ(c.size(), 3); + // Retrieving "one" makes it the most recent element thus + // two will be the oldest one and thus replaced. + ASSERT_EQ(c.get("two").has_value(), false); + ASSERT_EQ(c.get("another").value(), "whatever"); + } + + /* ---------------------------------------------------------------------------- + * clear + * --------------------------------------------------------------------------*/ + + TEST(LRUCache, clearEmptyCache) { + LRUCache c(10); + c.clear(); + ASSERT_EQ(c.size(), 0); + } + + TEST(LRUCache, clearNonEmptyCache) { + LRUCache c(10); + c.upsert("one", "eins"); + c.upsert("two", "zwei"); + c.upsert("three", "drei"); + ASSERT_EQ(c.size(), 3); + c.clear(); + ASSERT_EQ(c.size(), 0); + } + + /* ---------------------------------------------------------------------------- + * erase + * --------------------------------------------------------------------------*/ + + TEST(LRUCache, eraseFromEmptyCache) { + LRUCache c(10); + ASSERT_EQ(c.erase("foo"), false); + ASSERT_EQ(c.size(), 0); + } + + TEST(LRUCache, eraseMissingFromNonEmptyCache) { + LRUCache c(10); + c.upsert("one", "eins"); + ASSERT_EQ(c.erase("foo"), false); + ASSERT_EQ(c.size(), 1); + ASSERT_EQ(c.get("one").value_or("error"), "eins"); + } + + TEST(LRUCache, eraseFromNonEmptyCache) { + LRUCache c(10); + c.upsert("one", "eins"); + ASSERT_EQ(c.erase("one"), true); + ASSERT_EQ(c.size(), 0); + ASSERT_EQ(c.get("one").value_or("empty"), "empty"); + } +} diff --git a/src/libutil/tests/pool.cc b/src/libutil/tests/pool.cc new file mode 100644 index 00000000000..127e42dda2b --- /dev/null +++ b/src/libutil/tests/pool.cc @@ -0,0 +1,127 @@ +#include "pool.hh" +#include + +namespace nix { + + struct TestResource + { + + TestResource() { + static int counter = 0; + num = counter++; + } + + int dummyValue = 1; + bool good = true; + int num; + }; + + /* ---------------------------------------------------------------------------- + * Pool + * --------------------------------------------------------------------------*/ + + TEST(Pool, freshPoolHasZeroCountAndSpecifiedCapacity) { + auto isGood = [](const ref & r) { return r->good; }; + auto createResource = []() { return make_ref(); }; + + Pool pool = Pool((size_t)1, createResource, isGood); + + ASSERT_EQ(pool.count(), 0); + ASSERT_EQ(pool.capacity(), 1); + } + + TEST(Pool, freshPoolCanGetAResource) { + auto isGood = [](const ref & r) { return r->good; }; + auto createResource = []() { return make_ref(); }; + + Pool pool = Pool((size_t)1, createResource, isGood); + ASSERT_EQ(pool.count(), 0); + + TestResource r = *(pool.get()); + + ASSERT_EQ(pool.count(), 1); + ASSERT_EQ(pool.capacity(), 1); + ASSERT_EQ(r.dummyValue, 1); + ASSERT_EQ(r.good, true); + } + + TEST(Pool, capacityCanBeIncremented) { + auto isGood = [](const ref & r) { return r->good; }; + auto createResource = []() { return make_ref(); }; + + Pool pool = Pool((size_t)1, createResource, isGood); + ASSERT_EQ(pool.capacity(), 1); + pool.incCapacity(); + ASSERT_EQ(pool.capacity(), 2); + } + + TEST(Pool, capacityCanBeDecremented) { + auto isGood = [](const ref & r) { return r->good; }; + auto createResource = []() { return make_ref(); }; + + Pool pool = Pool((size_t)1, createResource, isGood); + ASSERT_EQ(pool.capacity(), 1); + pool.decCapacity(); + ASSERT_EQ(pool.capacity(), 0); + } + + TEST(Pool, flushBadDropsOutOfScopeResources) { + auto isGood = [](const ref & r) { return false; }; + auto createResource = []() { return make_ref(); }; + + Pool pool = Pool((size_t)1, createResource, isGood); + + { + auto _r = pool.get(); + ASSERT_EQ(pool.count(), 1); + } + + pool.flushBad(); + ASSERT_EQ(pool.count(), 0); + } + + // Test that the resources we allocate are being reused when they are still good. + TEST(Pool, reuseResource) { + auto isGood = [](const ref & r) { return true; }; + auto createResource = []() { return make_ref(); }; + + Pool pool = Pool((size_t)1, createResource, isGood); + + // Compare the instance counter between the two handles. We expect them to be equal + // as the pool should hand out the same (still) good one again. + int counter = -1; + { + Pool::Handle h = pool.get(); + counter = h->num; + } // the first handle goes out of scope + + { // the second handle should contain the same resource (with the same counter value) + Pool::Handle h = pool.get(); + ASSERT_EQ(h->num, counter); + } + } + + // Test that the resources we allocate are being thrown away when they are no longer good. + TEST(Pool, badResourceIsNotReused) { + auto isGood = [](const ref & r) { return false; }; + auto createResource = []() { return make_ref(); }; + + Pool pool = Pool((size_t)1, createResource, isGood); + + // Compare the instance counter between the two handles. We expect them + // to *not* be equal as the pool should hand out a new instance after + // the first one was returned. + int counter = -1; + { + Pool::Handle h = pool.get(); + counter = h->num; + } // the first handle goes out of scope + + { + // the second handle should contain a different resource (with a + //different counter value) + Pool::Handle h = pool.get(); + ASSERT_NE(h->num, counter); + } + } +} diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc new file mode 100644 index 00000000000..8e77ccbe17a --- /dev/null +++ b/src/libutil/tests/tests.cc @@ -0,0 +1,589 @@ +#include "util.hh" +#include "types.hh" + +#include + +namespace nix { + +/* ----------- tests for util.hh ------------------------------------------------*/ + + /* ---------------------------------------------------------------------------- + * absPath + * --------------------------------------------------------------------------*/ + + TEST(absPath, doesntChangeRoot) { + auto p = absPath("/"); + + ASSERT_EQ(p, "/"); + } + + + + + TEST(absPath, turnsEmptyPathIntoCWD) { + char cwd[PATH_MAX+1]; + auto p = absPath(""); + + ASSERT_EQ(p, getcwd((char*)&cwd, PATH_MAX)); + } + + TEST(absPath, usesOptionalBasePathWhenGiven) { + char _cwd[PATH_MAX+1]; + char* cwd = getcwd((char*)&_cwd, PATH_MAX); + + auto p = absPath("", cwd); + + ASSERT_EQ(p, cwd); + } + + TEST(absPath, isIdempotent) { + char _cwd[PATH_MAX+1]; + char* cwd = getcwd((char*)&_cwd, PATH_MAX); + auto p1 = absPath(cwd); + auto p2 = absPath(p1); + + ASSERT_EQ(p1, p2); + } + + + TEST(absPath, pathIsCanonicalised) { + auto path = "/some/path/with/trailing/dot/."; + auto p1 = absPath(path); + auto p2 = absPath(p1); + + ASSERT_EQ(p1, "/some/path/with/trailing/dot"); + ASSERT_EQ(p1, p2); + } + + /* ---------------------------------------------------------------------------- + * canonPath + * --------------------------------------------------------------------------*/ + + TEST(canonPath, removesTrailingSlashes) { + auto path = "/this/is/a/path//"; + auto p = canonPath(path); + + ASSERT_EQ(p, "/this/is/a/path"); + } + + TEST(canonPath, removesDots) { + auto path = "/this/./is/a/path/./"; + auto p = canonPath(path); + + ASSERT_EQ(p, "/this/is/a/path"); + } + + TEST(canonPath, removesDots2) { + auto path = "/this/a/../is/a////path/foo/.."; + auto p = canonPath(path); + + ASSERT_EQ(p, "/this/is/a/path"); + } + + TEST(canonPath, requiresAbsolutePath) { + ASSERT_ANY_THROW(canonPath(".")); + ASSERT_ANY_THROW(canonPath("..")); + ASSERT_ANY_THROW(canonPath("../")); + ASSERT_DEATH({ canonPath(""); }, "path != \"\""); + } + + /* ---------------------------------------------------------------------------- + * dirOf + * --------------------------------------------------------------------------*/ + + TEST(dirOf, returnsEmptyStringForRoot) { + auto p = dirOf("/"); + + ASSERT_EQ(p, "/"); + } + + TEST(dirOf, returnsFirstPathComponent) { + auto p1 = dirOf("/dir/"); + ASSERT_EQ(p1, "/dir"); + auto p2 = dirOf("/dir"); + ASSERT_EQ(p2, "/"); + auto p3 = dirOf("/dir/.."); + ASSERT_EQ(p3, "/dir"); + auto p4 = dirOf("/dir/../"); + ASSERT_EQ(p4, "/dir/.."); + } + + /* ---------------------------------------------------------------------------- + * baseNameOf + * --------------------------------------------------------------------------*/ + + TEST(baseNameOf, emptyPath) { + auto p1 = baseNameOf(""); + ASSERT_EQ(p1, ""); + } + + TEST(baseNameOf, pathOnRoot) { + auto p1 = baseNameOf("/dir"); + ASSERT_EQ(p1, "dir"); + } + + TEST(baseNameOf, relativePath) { + auto p1 = baseNameOf("dir/foo"); + ASSERT_EQ(p1, "foo"); + } + + TEST(baseNameOf, pathWithTrailingSlashRoot) { + auto p1 = baseNameOf("/"); + ASSERT_EQ(p1, ""); + } + + TEST(baseNameOf, trailingSlash) { + auto p1 = baseNameOf("/dir/"); + ASSERT_EQ(p1, "dir"); + } + + /* ---------------------------------------------------------------------------- + * isInDir + * --------------------------------------------------------------------------*/ + + TEST(isInDir, trivialCase) { + auto p1 = isInDir("/foo/bar", "/foo"); + ASSERT_EQ(p1, true); + } + + TEST(isInDir, notInDir) { + auto p1 = isInDir("/zes/foo/bar", "/foo"); + ASSERT_EQ(p1, false); + } + + // XXX: hm, bug or feature? :) Looking at the implementation + // this might be problematic. + TEST(isInDir, emptyDir) { + auto p1 = isInDir("/zes/foo/bar", ""); + ASSERT_EQ(p1, true); + } + + /* ---------------------------------------------------------------------------- + * isDirOrInDir + * --------------------------------------------------------------------------*/ + + TEST(isDirOrInDir, trueForSameDirectory) { + ASSERT_EQ(isDirOrInDir("/nix", "/nix"), true); + ASSERT_EQ(isDirOrInDir("/", "/"), true); + } + + TEST(isDirOrInDir, trueForEmptyPaths) { + ASSERT_EQ(isDirOrInDir("", ""), true); + } + + TEST(isDirOrInDir, falseForDisjunctPaths) { + ASSERT_EQ(isDirOrInDir("/foo", "/bar"), false); + } + + TEST(isDirOrInDir, relativePaths) { + ASSERT_EQ(isDirOrInDir("/foo/..", "/foo"), true); + } + + // XXX: while it is possible to use "." or ".." in the + // first argument this doesn't seem to work in the second. + TEST(isDirOrInDir, DISABLED_shouldWork) { + ASSERT_EQ(isDirOrInDir("/foo/..", "/foo/."), true); + + } + + /* ---------------------------------------------------------------------------- + * pathExists + * --------------------------------------------------------------------------*/ + + TEST(pathExists, rootExists) { + ASSERT_TRUE(pathExists("/")); + } + + TEST(pathExists, cwdExists) { + ASSERT_TRUE(pathExists(".")); + } + + TEST(pathExists, bogusPathDoesNotExist) { + ASSERT_FALSE(pathExists("/home/schnitzel/darmstadt/pommes")); + } + + /* ---------------------------------------------------------------------------- + * concatStringsSep + * --------------------------------------------------------------------------*/ + + TEST(concatStringsSep, buildCommaSeparatedString) { + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(concatStringsSep(",", strings), "this,is,great"); + } + + TEST(concatStringsSep, buildStringWithEmptySeparator) { + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(concatStringsSep("", strings), "thisisgreat"); + } + + TEST(concatStringsSep, buildSingleString) { + Strings strings; + strings.push_back("this"); + + ASSERT_EQ(concatStringsSep(",", strings), "this"); + } + + /* ---------------------------------------------------------------------------- + * hasPrefix + * --------------------------------------------------------------------------*/ + + TEST(hasPrefix, emptyStringHasNoPrefix) { + ASSERT_FALSE(hasPrefix("", "foo")); + } + + TEST(hasPrefix, emptyStringIsAlwaysPrefix) { + ASSERT_TRUE(hasPrefix("foo", "")); + ASSERT_TRUE(hasPrefix("jshjkfhsadf", "")); + } + + TEST(hasPrefix, trivialCase) { + ASSERT_TRUE(hasPrefix("foobar", "foo")); + } + + /* ---------------------------------------------------------------------------- + * hasSuffix + * --------------------------------------------------------------------------*/ + + TEST(hasSuffix, emptyStringHasNoSuffix) { + ASSERT_FALSE(hasSuffix("", "foo")); + } + + TEST(hasSuffix, trivialCase) { + ASSERT_TRUE(hasSuffix("foo", "foo")); + ASSERT_TRUE(hasSuffix("foobar", "bar")); + } + + /* ---------------------------------------------------------------------------- + * base64Encode + * --------------------------------------------------------------------------*/ + + TEST(base64Encode, emptyString) { + ASSERT_EQ(base64Encode(""), ""); + } + + TEST(base64Encode, encodesAString) { + ASSERT_EQ(base64Encode("quod erat demonstrandum"), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="); + } + + TEST(base64Encode, encodeAndDecode) { + auto s = "quod erat demonstrandum"; + auto encoded = base64Encode(s); + auto decoded = base64Decode(encoded); + + ASSERT_EQ(decoded, s); + } + + /* ---------------------------------------------------------------------------- + * base64Decode + * --------------------------------------------------------------------------*/ + + TEST(base64Decode, emptyString) { + ASSERT_EQ(base64Decode(""), ""); + } + + TEST(base64Decode, decodeAString) { + ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum"); + } + + /* ---------------------------------------------------------------------------- + * toLower + * --------------------------------------------------------------------------*/ + + TEST(toLower, emptyString) { + ASSERT_EQ(toLower(""), ""); + } + + TEST(toLower, nonLetters) { + auto s = "!@(*$#)(@#=\\234_"; + ASSERT_EQ(toLower(s), s); + } + + // std::tolower() doesn't handle unicode characters. In the context of + // store paths this isn't relevant but doesn't hurt to record this behavior + // here. + TEST(toLower, umlauts) { + auto s = "ÄÖÜ"; + ASSERT_EQ(toLower(s), "ÄÖÜ"); + } + + /* ---------------------------------------------------------------------------- + * string2Float + * --------------------------------------------------------------------------*/ + + TEST(string2Float, emptyString) { + double n; + ASSERT_EQ(string2Float("", n), false); + } + + TEST(string2Float, trivialConversions) { + double n; + ASSERT_EQ(string2Float("1.0", n), true); + ASSERT_EQ(n, 1.0); + + ASSERT_EQ(string2Float("0.0", n), true); + ASSERT_EQ(n, 0.0); + + ASSERT_EQ(string2Float("-100.25", n), true); + ASSERT_EQ(n, (-100.25)); + } + + /* ---------------------------------------------------------------------------- + * string2Int + * --------------------------------------------------------------------------*/ + + TEST(string2Int, emptyString) { + double n; + ASSERT_EQ(string2Int("", n), false); + } + + TEST(string2Int, trivialConversions) { + double n; + ASSERT_EQ(string2Int("1", n), true); + ASSERT_EQ(n, 1); + + ASSERT_EQ(string2Int("0", n), true); + ASSERT_EQ(n, 0); + + ASSERT_EQ(string2Int("-100", n), true); + ASSERT_EQ(n, (-100)); + } + + /* ---------------------------------------------------------------------------- + * statusOk + * --------------------------------------------------------------------------*/ + + TEST(statusOk, zeroIsOk) { + ASSERT_EQ(statusOk(0), true); + ASSERT_EQ(statusOk(1), false); + } + + + /* ---------------------------------------------------------------------------- + * rewriteStrings + * --------------------------------------------------------------------------*/ + + TEST(rewriteStrings, emptyString) { + StringMap rewrites; + rewrites["this"] = "that"; + + ASSERT_EQ(rewriteStrings("", rewrites), ""); + } + + TEST(rewriteStrings, emptyRewrites) { + StringMap rewrites; + + ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); + } + + TEST(rewriteStrings, successfulRewrite) { + StringMap rewrites; + rewrites["this"] = "that"; + + ASSERT_EQ(rewriteStrings("this and that", rewrites), "that and that"); + } + + TEST(rewriteStrings, doesntOccur) { + StringMap rewrites; + rewrites["foo"] = "bar"; + + ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); + } + + /* ---------------------------------------------------------------------------- + * replaceStrings + * --------------------------------------------------------------------------*/ + + TEST(replaceStrings, emptyString) { + ASSERT_EQ(replaceStrings("", "this", "that"), ""); + ASSERT_EQ(replaceStrings("this and that", "", ""), "this and that"); + } + + TEST(replaceStrings, successfulReplace) { + ASSERT_EQ(replaceStrings("this and that", "this", "that"), "that and that"); + } + + TEST(replaceStrings, doesntOccur) { + ASSERT_EQ(replaceStrings("this and that", "foo", "bar"), "this and that"); + } + + /* ---------------------------------------------------------------------------- + * trim + * --------------------------------------------------------------------------*/ + + TEST(trim, emptyString) { + ASSERT_EQ(trim(""), ""); + } + + TEST(trim, removesWhitespace) { + ASSERT_EQ(trim("foo"), "foo"); + ASSERT_EQ(trim(" foo "), "foo"); + ASSERT_EQ(trim(" foo bar baz"), "foo bar baz"); + ASSERT_EQ(trim(" \t foo bar baz\n"), "foo bar baz"); + } + + /* ---------------------------------------------------------------------------- + * chomp + * --------------------------------------------------------------------------*/ + + TEST(chomp, emptyString) { + ASSERT_EQ(chomp(""), ""); + } + + TEST(chomp, removesWhitespace) { + ASSERT_EQ(chomp("foo"), "foo"); + ASSERT_EQ(chomp("foo "), "foo"); + ASSERT_EQ(chomp(" foo "), " foo"); + ASSERT_EQ(chomp(" foo bar baz "), " foo bar baz"); + ASSERT_EQ(chomp("\t foo bar baz\n"), "\t foo bar baz"); + } + + /* ---------------------------------------------------------------------------- + * quoteStrings + * --------------------------------------------------------------------------*/ + + TEST(quoteStrings, empty) { + Strings s = { }; + Strings expected = { }; + + ASSERT_EQ(quoteStrings(s), expected); + } + + TEST(quoteStrings, emptyStrings) { + Strings s = { "", "", "" }; + Strings expected = { "''", "''", "''" }; + ASSERT_EQ(quoteStrings(s), expected); + + } + + TEST(quoteStrings, trivialQuote) { + Strings s = { "foo", "bar", "baz" }; + Strings expected = { "'foo'", "'bar'", "'baz'" }; + + ASSERT_EQ(quoteStrings(s), expected); + } + + TEST(quoteStrings, quotedStrings) { + Strings s = { "'foo'", "'bar'", "'baz'" }; + Strings expected = { "''foo''", "''bar''", "''baz''" }; + + ASSERT_EQ(quoteStrings(s), expected); + } + + /* ---------------------------------------------------------------------------- + * tokenizeString + * --------------------------------------------------------------------------*/ + + TEST(tokenizeString, empty) { + Strings expected = { }; + + ASSERT_EQ(tokenizeString(""), expected); + } + + TEST(tokenizeString, tokenizeSpacesWithDefaults) { + auto s = "foo bar baz"; + Strings expected = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString(s), expected); + } + + TEST(tokenizeString, tokenizeTabsWithDefaults) { + auto s = "foo\tbar\tbaz"; + Strings expected = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString(s), expected); + } + + TEST(tokenizeString, tokenizeTabsSpacesWithDefaults) { + auto s = "foo\t bar\t baz"; + Strings expected = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString(s), expected); + } + + TEST(tokenizeString, tokenizeTabsSpacesNewlineWithDefaults) { + auto s = "foo\t\n bar\t\n baz"; + Strings expected = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString(s), expected); + } + + TEST(tokenizeString, tokenizeTabsSpacesNewlineRetWithDefaults) { + auto s = "foo\t\n\r bar\t\n\r baz"; + Strings expected = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString(s), expected); + + auto s2 = "foo \t\n\r bar \t\n\r baz"; + Strings expected2 = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString(s2), expected2); + } + + TEST(tokenizeString, tokenizeWithCustomSep) { + auto s = "foo\n,bar\n,baz\n"; + Strings expected = { "foo\n", "bar\n", "baz\n" }; + + ASSERT_EQ(tokenizeString(s, ","), expected); + } + + /* ---------------------------------------------------------------------------- + * get + * --------------------------------------------------------------------------*/ + + TEST(get, emptyContainer) { + StringMap s = { }; + auto expected = std::nullopt; + + ASSERT_EQ(get(s, "one"), expected); + } + + TEST(get, getFromContainer) { + StringMap s; + s["one"] = "yi"; + s["two"] = "er"; + auto expected = "yi"; + + ASSERT_EQ(get(s, "one"), expected); + } + + /* ---------------------------------------------------------------------------- + * filterANSIEscapes + * --------------------------------------------------------------------------*/ + + TEST(filterANSIEscapes, emptyString) { + auto s = ""; + auto expected = ""; + + ASSERT_EQ(filterANSIEscapes(s), expected); + } + + TEST(filterANSIEscapes, doesntChangePrintableChars) { + auto s = "09 2q304ruyhr slk2-19024 kjsadh sar f"; + + ASSERT_EQ(filterANSIEscapes(s), s); + } + + TEST(filterANSIEscapes, filtersColorCodes) { + auto s = "\u001b[30m A \u001b[31m B \u001b[32m C \u001b[33m D \u001b[0m"; + + ASSERT_EQ(filterANSIEscapes(s, true, 2), " A" ); + ASSERT_EQ(filterANSIEscapes(s, true, 3), " A " ); + ASSERT_EQ(filterANSIEscapes(s, true, 4), " A " ); + ASSERT_EQ(filterANSIEscapes(s, true, 5), " A B" ); + ASSERT_EQ(filterANSIEscapes(s, true, 8), " A B C" ); + } + + TEST(filterANSIEscapes, expandsTabs) { + auto s = "foo\tbar\tbaz"; + + ASSERT_EQ(filterANSIEscapes(s, true), "foo bar baz" ); + } +} diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc new file mode 100644 index 00000000000..80646ad3e32 --- /dev/null +++ b/src/libutil/tests/url.cc @@ -0,0 +1,266 @@ +#include "url.hh" +#include + +namespace nix { + +/* ----------- tests for url.hh --------------------------------------------------*/ + + string print_map(std::map m) { + std::map::iterator it; + string s = "{ "; + for (it = m.begin(); it != m.end(); ++it) { + s += "{ "; + s += it->first; + s += " = "; + s += it->second; + s += " } "; + } + s += "}"; + return s; + } + + + std::ostream& operator<<(std::ostream& os, const ParsedURL& p) { + return os << "\n" + << "url: " << p.url << "\n" + << "base: " << p.base << "\n" + << "scheme: " << p.scheme << "\n" + << "authority: " << p.authority.value() << "\n" + << "path: " << p.path << "\n" + << "query: " << print_map(p.query) << "\n" + << "fragment: " << p.fragment << "\n"; + } + + TEST(parseURL, parsesSimpleHttpUrl) { + auto s = "http://www.example.org/file.tar.gz"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "http://www.example.org/file.tar.gz", + .base = "http://www.example.org/file.tar.gz", + .scheme = "http", + .authority = "www.example.org", + .path = "/file.tar.gz", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, parsesSimpleHttpsUrl) { + auto s = "https://www.example.org/file.tar.gz"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "https://www.example.org/file.tar.gz", + .base = "https://www.example.org/file.tar.gz", + .scheme = "https", + .authority = "www.example.org", + .path = "/file.tar.gz", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, parsesSimpleHttpUrlWithQueryAndFragment) { + auto s = "https://www.example.org/file.tar.gz?download=fast&when=now#hello"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "https://www.example.org/file.tar.gz", + .base = "https://www.example.org/file.tar.gz", + .scheme = "https", + .authority = "www.example.org", + .path = "/file.tar.gz", + .query = (StringMap) { { "download", "fast" }, { "when", "now" } }, + .fragment = "hello", + }; + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, parsesSimpleHttpUrlWithComplexFragment) { + auto s = "http://www.example.org/file.tar.gz?field=value#?foo=bar%23"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "http://www.example.org/file.tar.gz", + .base = "http://www.example.org/file.tar.gz", + .scheme = "http", + .authority = "www.example.org", + .path = "/file.tar.gz", + .query = (StringMap) { { "field", "value" } }, + .fragment = "?foo=bar#", + }; + + ASSERT_EQ(parsed, expected); + } + + + TEST(parseURL, parseIPv4Address) { + auto s = "http://127.0.0.1:8080/file.tar.gz?download=fast&when=now#hello"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "http://127.0.0.1:8080/file.tar.gz", + .base = "https://127.0.0.1:8080/file.tar.gz", + .scheme = "http", + .authority = "127.0.0.1:8080", + .path = "/file.tar.gz", + .query = (StringMap) { { "download", "fast" }, { "when", "now" } }, + .fragment = "hello", + }; + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, parseIPv6Address) { + auto s = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", + .base = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", + .scheme = "http", + .authority = "[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", + .path = "", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + + } + + TEST(parseURL, parseEmptyQueryParams) { + auto s = "http://127.0.0.1:8080/file.tar.gz?&&&&&"; + auto parsed = parseURL(s); + ASSERT_EQ(parsed.query, (StringMap) { }); + } + + TEST(parseURL, parseUserPassword) { + auto s = "http://user:pass@www.example.org:8080/file.tar.gz"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "http://user:pass@www.example.org/file.tar.gz", + .base = "http://user:pass@www.example.org/file.tar.gz", + .scheme = "http", + .authority = "user:pass@www.example.org:8080", + .path = "/file.tar.gz", + .query = (StringMap) { }, + .fragment = "", + }; + + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, parseFileURLWithQueryAndFragment) { + auto s = "file:///none/of/your/business"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "", + .base = "", + .scheme = "file", + .authority = "", + .path = "/none/of/your/business", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + + } + + TEST(parseURL, parsedUrlsIsEqualToItself) { + auto s = "http://www.example.org/file.tar.gz"; + auto url = parseURL(s); + + ASSERT_TRUE(url == url); + } + + TEST(parseURL, parseFTPUrl) { + auto s = "ftp://ftp.nixos.org/downloads/nixos.iso"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "ftp://ftp.nixos.org/downloads/nixos.iso", + .base = "ftp://ftp.nixos.org/downloads/nixos.iso", + .scheme = "ftp", + .authority = "ftp.nixos.org", + .path = "/downloads/nixos.iso", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, parsesAnythingInUriFormat) { + auto s = "whatever://github.com/NixOS/nixpkgs.git"; + auto parsed = parseURL(s); + } + + TEST(parseURL, parsesAnythingInUriFormatWithoutDoubleSlash) { + auto s = "whatever:github.com/NixOS/nixpkgs.git"; + auto parsed = parseURL(s); + } + + TEST(parseURL, emptyStringIsInvalidURL) { + ASSERT_THROW(parseURL(""), Error); + } + + /* ---------------------------------------------------------------------------- + * decodeQuery + * --------------------------------------------------------------------------*/ + + TEST(decodeQuery, emptyStringYieldsEmptyMap) { + auto d = decodeQuery(""); + ASSERT_EQ(d, (StringMap) { }); + } + + TEST(decodeQuery, simpleDecode) { + auto d = decodeQuery("yi=one&er=two"); + ASSERT_EQ(d, ((StringMap) { { "yi", "one" }, { "er", "two" } })); + } + + TEST(decodeQuery, decodeUrlEncodedArgs) { + auto d = decodeQuery("arg=%3D%3D%40%3D%3D"); + ASSERT_EQ(d, ((StringMap) { { "arg", "==@==" } })); + } + + TEST(decodeQuery, decodeArgWithEmptyValue) { + auto d = decodeQuery("arg="); + ASSERT_EQ(d, ((StringMap) { { "arg", ""} })); + } + + /* ---------------------------------------------------------------------------- + * percentDecode + * --------------------------------------------------------------------------*/ + + TEST(percentDecode, decodesUrlEncodedString) { + string s = "==@=="; + string d = percentDecode("%3D%3D%40%3D%3D"); + ASSERT_EQ(d, s); + } + + TEST(percentDecode, multipleDecodesAreIdempotent) { + string once = percentDecode("%3D%3D%40%3D%3D"); + string twice = percentDecode(once); + + ASSERT_EQ(once, twice); + } + + TEST(percentDecode, trailingPercent) { + string s = "==@==%"; + string d = percentDecode("%3D%3D%40%3D%3D%25"); + + ASSERT_EQ(d, s); + } + +} diff --git a/src/libutil/tests/xml-writer.cc b/src/libutil/tests/xml-writer.cc new file mode 100644 index 00000000000..adcde25c9f1 --- /dev/null +++ b/src/libutil/tests/xml-writer.cc @@ -0,0 +1,105 @@ +#include "xml-writer.hh" +#include +#include + +namespace nix { + + /* ---------------------------------------------------------------------------- + * XMLWriter + * --------------------------------------------------------------------------*/ + + TEST(XMLWriter, emptyObject) { + std::stringstream out; + { + XMLWriter t(false, out); + } + + ASSERT_EQ(out.str(), "\n"); + } + + TEST(XMLWriter, objectWithEmptyElement) { + std::stringstream out; + { + XMLWriter t(false, out); + t.openElement("foobar"); + } + + ASSERT_EQ(out.str(), "\n"); + } + + TEST(XMLWriter, objectWithElementWithAttrs) { + std::stringstream out; + { + XMLWriter t(false, out); + XMLAttrs attrs = { + { "foo", "bar" } + }; + t.openElement("foobar", attrs); + } + + ASSERT_EQ(out.str(), "\n"); + } + + TEST(XMLWriter, objectWithElementWithEmptyAttrs) { + std::stringstream out; + { + XMLWriter t(false, out); + XMLAttrs attrs = {}; + t.openElement("foobar", attrs); + } + + ASSERT_EQ(out.str(), "\n"); + } + + TEST(XMLWriter, objectWithElementWithAttrsEscaping) { + std::stringstream out; + { + XMLWriter t(false, out); + XMLAttrs attrs = { + { "", "" } + }; + t.openElement("foobar", attrs); + } + + // XXX: While "" is escaped, "" isn't which I think is a bug. + ASSERT_EQ(out.str(), "\n=\"<value>\">"); + } + + TEST(XMLWriter, objectWithElementWithAttrsIndented) { + std::stringstream out; + { + XMLWriter t(true, out); + XMLAttrs attrs = { + { "foo", "bar" } + }; + t.openElement("foobar", attrs); + } + + ASSERT_EQ(out.str(), "\n\n\n"); + } + + TEST(XMLWriter, writeEmptyElement) { + std::stringstream out; + { + XMLWriter t(false, out); + t.writeEmptyElement("foobar"); + } + + ASSERT_EQ(out.str(), "\n"); + } + + TEST(XMLWriter, writeEmptyElementWithAttributes) { + std::stringstream out; + { + XMLWriter t(false, out); + XMLAttrs attrs = { + { "foo", "bar" } + }; + t.writeEmptyElement("foobar", attrs); + + } + + ASSERT_EQ(out.str(), "\n"); + } + +} diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index 0a3a407240f..857ee91f87d 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -13,19 +13,26 @@ ThreadPool::ThreadPool(size_t _maxThreads) if (!maxThreads) maxThreads = 1; } - debug(format("starting pool of %d threads") % maxThreads); + debug("starting pool of %d threads", maxThreads - 1); } ThreadPool::~ThreadPool() +{ + shutdown(); +} + +void ThreadPool::shutdown() { std::vector workers; { auto state(state_.lock()); - state->quit = true; + quit = true; std::swap(workers, state->workers); } - debug(format("reaping %d worker threads") % workers.size()); + if (workers.empty()) return; + + debug("reaping %d worker threads", workers.size()); work.notify_all(); @@ -36,63 +43,108 @@ ThreadPool::~ThreadPool() void ThreadPool::enqueue(const work_t & t) { auto state(state_.lock()); - if (state->quit) + if (quit) throw ThreadPoolShutDown("cannot enqueue a work item while the thread pool is shutting down"); - state->left.push(t); - if (state->left.size() > state->workers.size() && state->workers.size() < maxThreads) - state->workers.emplace_back(&ThreadPool::workerEntry, this); + state->pending.push(t); + /* Note: process() also executes items, so count it as a worker. */ + if (state->pending.size() > state->workers.size() + 1 && state->workers.size() + 1 < maxThreads) + state->workers.emplace_back(&ThreadPool::doWork, this, false); work.notify_one(); } void ThreadPool::process() { - while (true) { + state_.lock()->draining = true; + + /* Do work until no more work is pending or active. */ + try { + doWork(true); + auto state(state_.lock()); + + assert(quit); + if (state->exception) std::rethrow_exception(state->exception); - if (state->left.empty() && !state->pending) break; - state.wait(done); + + } catch (...) { + /* In the exceptional case, some workers may still be + active. They may be referencing the stack frame of the + caller. So wait for them to finish. (~ThreadPool also does + this, but it might be destroyed after objects referenced by + the work item lambdas.) */ + shutdown(); + throw; } } -void ThreadPool::workerEntry() +void ThreadPool::doWork(bool mainThread) { + if (!mainThread) + interruptCheck = [&]() { return (bool) quit; }; + bool didWork = false; + std::exception_ptr exc; while (true) { work_t w; { auto state(state_.lock()); + + if (didWork) { + assert(state->active); + state->active--; + + if (exc) { + + if (!state->exception) { + state->exception = exc; + // Tell the other workers to quit. + quit = true; + work.notify_all(); + } else { + /* Print the exception, since we can't + propagate it. */ + try { + std::rethrow_exception(exc); + } catch (std::exception & e) { + if (!dynamic_cast(&e) && + !dynamic_cast(&e)) + ignoreException(); + } catch (...) { + } + } + } + } + + /* Wait until a work item is available or we're asked to + quit. */ while (true) { - if (state->quit || state->exception) return; - if (didWork) { - assert(state->pending); - state->pending--; - didWork = false; + if (quit) return; + + if (!state->pending.empty()) break; + + /* If there are no active or pending items, and the + main thread is running process(), then no new items + can be added. So exit. */ + if (!state->active && state->draining) { + quit = true; + work.notify_all(); + return; } - if (!state->left.empty()) break; - if (!state->pending) - done.notify_all(); + state.wait(work); } - w = state->left.front(); - state->left.pop(); - state->pending++; + + w = std::move(state->pending.front()); + state->pending.pop(); + state->active++; } try { w(); - } catch (std::exception & e) { - auto state(state_.lock()); - if (state->exception) { - if (!dynamic_cast(&e) && - !dynamic_cast(&e)) - printError(format("error: %s") % e.what()); - } else { - state->exception = std::current_exception(); - work.notify_all(); - done.notify_all(); - } + } catch (...) { + exc = std::current_exception(); } didWork = true; diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh index b64dc52d473..b22e0d16225 100644 --- a/src/libutil/thread-pool.hh +++ b/src/libutil/thread-pool.hh @@ -7,10 +7,11 @@ #include #include #include +#include namespace nix { -MakeError(ThreadPoolShutDown, Error) +MakeError(ThreadPoolShutDown, Error); /* A simple thread pool that executes a queue of work items (lambdas). */ @@ -43,18 +44,22 @@ private: struct State { - std::queue left; - size_t pending = 0; + std::queue pending; + size_t active = 0; std::exception_ptr exception; std::vector workers; - bool quit = false; + bool draining = false; }; + std::atomic_bool quit{false}; + Sync state_; - std::condition_variable work, done; + std::condition_variable work; + + void doWork(bool mainThread); - void workerEntry(); + void shutdown(); }; /* Process in parallel a set of items of type T that have a partial @@ -70,50 +75,69 @@ void processGraph( struct Graph { std::set left; std::map> refs, rrefs; - std::function wrap; }; - ref> graph_ = make_ref>(); + Sync graph_(Graph{nodes, {}, {}}); + + std::function worker; + + worker = [&](const T & node) { + + { + auto graph(graph_.lock()); + auto i = graph->refs.find(node); + if (i == graph->refs.end()) + goto getRefs; + goto doWork; + } - auto wrapWork = [&pool, graph_, processNode](const T & node) { + getRefs: + { + auto refs = getEdges(node); + refs.erase(node); + + { + auto graph(graph_.lock()); + for (auto & ref : refs) + if (graph->left.count(ref)) { + graph->refs[node].insert(ref); + graph->rrefs[ref].insert(node); + } + if (graph->refs[node].empty()) + goto doWork; + } + } + + return; + + doWork: processNode(node); - /* Enqueue work for all nodes that were waiting on this one. */ + /* Enqueue work for all nodes that were waiting on this one + and have no unprocessed dependencies. */ { - auto graph(graph_->lock()); - graph->left.erase(node); + auto graph(graph_.lock()); for (auto & rref : graph->rrefs[node]) { auto & refs(graph->refs[rref]); auto i = refs.find(node); assert(i != refs.end()); refs.erase(i); if (refs.empty()) - pool.enqueue(std::bind(graph->wrap, rref)); + pool.enqueue(std::bind(worker, rref)); } + graph->left.erase(node); + graph->refs.erase(node); + graph->rrefs.erase(node); } }; - { - auto graph(graph_->lock()); - graph->left = nodes; - graph->wrap = wrapWork; - } - - /* Build the dependency graph; enqueue all nodes with no - dependencies. */ - for (auto & node : nodes) { - auto refs = getEdges(node); - { - auto graph(graph_->lock()); - for (auto & ref : refs) - if (ref != node && graph->left.count(ref)) { - graph->refs[node].insert(ref); - graph->rrefs[ref].insert(node); - } - if (graph->refs[node].empty()) - pool.enqueue(std::bind(graph->wrap, node)); - } - } + for (auto & node : nodes) + pool.enqueue(std::bind(worker, std::ref(node))); + + pool.process(); + + if (!graph_.lock()->left.empty()) + throw Error("graph processing incomplete (cyclic reference?)"); } } diff --git a/src/libutil/types.hh b/src/libutil/types.hh index b9a93d27d2a..3af485fa0fe 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -1,153 +1,34 @@ #pragma once -#include "config.h" - #include "ref.hh" -#include #include #include -#include - -#include - -/* Before 4.7, gcc's std::exception uses empty throw() specifiers for - * its (virtual) destructor and what() in c++11 mode, in violation of spec - */ -#ifdef __GNUC__ -#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7) -#define EXCEPTION_NEEDS_THROW_SPEC -#endif -#endif - +#include +#include namespace nix { - -/* Inherit some names from other namespaces for convenience. */ -using std::string; using std::list; using std::set; using std::vector; -using boost::format; - - -struct FormatOrString -{ - string s; - FormatOrString(const string & s) : s(s) { }; - FormatOrString(const format & f) : s(f.str()) { }; - FormatOrString(const char * s) : s(s) { }; -}; - - -/* A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is - equivalent to ‘boost::format(format) % a_0 % ... % - ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion - takes place). */ - -inline void formatHelper(boost::format & f) -{ -} - -template -inline void formatHelper(boost::format & f, T x, Args... args) -{ - formatHelper(f % x, args...); -} - -inline std::string fmt(const std::string & s) -{ - return s; -} - -inline std::string fmt(const char * s) -{ - return s; -} - -inline std::string fmt(const FormatOrString & fs) -{ - return fs.s; -} - -template -inline std::string fmt(const std::string & fs, Args... args) -{ - boost::format f(fs); - formatHelper(f, args...); - return f.str(); -} - - -/* BaseError should generally not be caught, as it has Interrupted as - a subclass. Catch Error instead. */ -class BaseError : public std::exception -{ -protected: - string prefix_; // used for location traces etc. - string err; -public: - unsigned int status = 1; // exit status - - template - BaseError(unsigned int status, Args... args) - : err(fmt(args...)) - , status(status) - { - } - - template - BaseError(Args... args) - : err(fmt(args...)) - { - } - -#ifdef EXCEPTION_NEEDS_THROW_SPEC - ~BaseError() throw () { }; - const char * what() const throw () { return err.c_str(); } -#else - const char * what() const noexcept { return err.c_str(); } -#endif - - const string & msg() const { return err; } - const string & prefix() const { return prefix_; } - BaseError & addPrefix(const FormatOrString & fs); -}; - -#define MakeError(newClass, superClass) \ - class newClass : public superClass \ - { \ - public: \ - using superClass::superClass; \ - }; - -MakeError(Error, BaseError) - -class SysError : public Error -{ -public: - int errNo; - - template - SysError(Args... args) - : Error(addErrno(fmt(args...))) - { } - -private: - - std::string addErrno(const std::string & s); -}; - +using std::string; typedef list Strings; typedef set StringSet; - +typedef std::map StringMap; /* Paths are just strings. */ + typedef string Path; typedef list Paths; typedef set PathSet; +/* Helper class to run code at startup. */ +template +struct OnStartup +{ + OnStartup(T && t) { t(); } +}; } diff --git a/src/libutil/url.cc b/src/libutil/url.cc new file mode 100644 index 00000000000..88c09eef9d7 --- /dev/null +++ b/src/libutil/url.cc @@ -0,0 +1,138 @@ +#include "url.hh" +#include "util.hh" + +namespace nix { + +std::regex refRegex(refRegexS, std::regex::ECMAScript); +std::regex badGitRefRegex(badGitRefRegexS, std::regex::ECMAScript); +std::regex revRegex(revRegexS, std::regex::ECMAScript); +std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript); + +ParsedURL parseURL(const std::string & url) +{ + static std::regex uriRegex( + "((" + schemeRegex + "):" + + "(?:(?://(" + authorityRegex + ")(" + absPathRegex + "))|(/?" + pathRegex + ")))" + + "(?:\\?(" + queryRegex + "))?" + + "(?:#(" + queryRegex + "))?", + std::regex::ECMAScript); + + std::smatch match; + + if (std::regex_match(url, match, uriRegex)) { + auto & base = match[1]; + std::string scheme = match[2]; + auto authority = match[3].matched + ? std::optional(match[3]) : std::nullopt; + std::string path = match[4].matched ? match[4] : match[5]; + auto & query = match[6]; + auto & fragment = match[7]; + + auto isFile = scheme.find("file") != std::string::npos; + + if (authority && *authority != "" && isFile) + throw Error("file:// URL '%s' has unexpected authority '%s'", + url, *authority); + + if (isFile && path.empty()) + path = "/"; + + return ParsedURL{ + .url = url, + .base = base, + .scheme = scheme, + .authority = authority, + .path = path, + .query = decodeQuery(query), + .fragment = percentDecode(std::string(fragment)) + }; + } + + else + throw BadURL("'%s' is not a valid URL", url); +} + +std::string percentDecode(std::string_view in) +{ + std::string decoded; + for (size_t i = 0; i < in.size(); ) { + if (in[i] == '%') { + if (i + 2 >= in.size()) + throw BadURL("invalid URI parameter '%s'", in); + try { + decoded += std::stoul(std::string(in, i + 1, 2), 0, 16); + i += 3; + } catch (...) { + throw BadURL("invalid URI parameter '%s'", in); + } + } else + decoded += in[i++]; + } + return decoded; +} + +std::map decodeQuery(const std::string & query) +{ + std::map result; + + for (auto s : tokenizeString(query, "&")) { + auto e = s.find('='); + if (e != std::string::npos) + result.emplace( + s.substr(0, e), + percentDecode(std::string_view(s).substr(e + 1))); + } + + return result; +} + +std::string percentEncode(std::string_view s) +{ + std::string res; + for (auto & c : s) + if ((c >= 'a' && c <= 'z') + || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9') + || strchr("-._~!$&'()*+,;=:@", c)) + res += c; + else + res += fmt("%%%02x", (unsigned int) c); + return res; +} + +std::string encodeQuery(const std::map & ss) +{ + std::string res; + bool first = true; + for (auto & [name, value] : ss) { + if (!first) res += '&'; + first = false; + res += percentEncode(name); + res += '='; + res += percentEncode(value); + } + return res; +} + +std::string ParsedURL::to_string() const +{ + return + scheme + + ":" + + (authority ? "//" + *authority : "") + + path + + (query.empty() ? "" : "?" + encodeQuery(query)) + + (fragment.empty() ? "" : "#" + percentEncode(fragment)); +} + +bool ParsedURL::operator ==(const ParsedURL & other) const +{ + return + scheme == other.scheme + && authority == other.authority + && path == other.path + && query == other.query + && fragment == other.fragment; +} + +} diff --git a/src/libutil/url.hh b/src/libutil/url.hh new file mode 100644 index 00000000000..2ef88ef2a1e --- /dev/null +++ b/src/libutil/url.hh @@ -0,0 +1,68 @@ +#pragma once + +#include "error.hh" + +#include + +namespace nix { + +struct ParsedURL +{ + std::string url; + std::string base; // URL without query/fragment + std::string scheme; + std::optional authority; + std::string path; + std::map query; + std::string fragment; + + std::string to_string() const; + + bool operator ==(const ParsedURL & other) const; +}; + +MakeError(BadURL, Error); + +std::string percentDecode(std::string_view in); + +std::map decodeQuery(const std::string & query); + +ParsedURL parseURL(const std::string & url); + +// URI stuff. +const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; +const static std::string schemeRegex = "(?:[a-z+]+)"; +const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])"; +const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; +const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; +const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)"; +const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")"; +const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)"; +const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?"; +const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])"; +const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*"; +const static std::string segmentRegex = "(?:" + pcharRegex + "+)"; +const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; +const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; + +// A Git ref (i.e. branch or tag name). +const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check +extern std::regex refRegex; + +// Instead of defining what a good Git Ref is, we define what a bad Git Ref is +// This is because of the definition of a ref in refs.c in https://github.com/git/git +// See tests/fetchGitRefs.sh for the full definition +const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$"; +extern std::regex badGitRefRegex; + +// A Git revision (a SHA-1 commit hash). +const static std::string revRegexS = "[0-9a-fA-F]{40}"; +extern std::regex revRegex; + +// A ref or revision, or a ref followed by a revision. +const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; + +const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; +extern std::regex flakeIdRegex; + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 52608ac2a01..667dd2edb83 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,22 +1,30 @@ -#include "config.h" - +#include "lazy.hh" #include "util.hh" #include "affinity.hh" #include "sync.hh" +#include "finally.hh" +#include "serialise.hh" #include #include #include #include #include +#include #include #include #include +#include +#include +#include +#include +#include +#include +#include #include +#include #include -#include -#include #ifdef __APPLE__ #include @@ -32,25 +40,11 @@ extern char * * environ; namespace nix { - -BaseError & BaseError::addPrefix(const FormatOrString & fs) -{ - prefix_ = fs.s + prefix_; - return *this; -} - - -std::string SysError::addErrno(const std::string & s) -{ - errNo = errno; - return s + ": " + strerror(errNo); -} - - -string getEnv(const string & key, const string & def) +std::optional getEnv(const std::string & key) { char * value = getenv(key.c_str()); - return value ? string(value) : def; + if (!value) return {}; + return std::string(value); } @@ -69,10 +63,26 @@ std::map getEnv() } -Path absPath(Path path, Path dir) +void clearEnv() +{ + for (auto & name : getEnv()) + unsetenv(name.first.c_str()); +} + +void replaceEnv(std::map newEnv) +{ + clearEnv(); + for (auto newEnvVar : newEnv) + { + setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); + } +} + + +Path absPath(Path path, std::optional dir) { if (path[0] != '/') { - if (dir == "") { + if (!dir) { #ifdef __GNU__ /* GNU (aka. GNU/Hurd) doesn't have any limitation on path lengths and doesn't define `PATH_MAX'. */ @@ -88,7 +98,7 @@ Path absPath(Path path, Path dir) free(buf); #endif } - path = dir + "/" + path; + path = *dir + "/" + path; } return canonPath(path); } @@ -96,10 +106,12 @@ Path absPath(Path path, Path dir) Path canonPath(const Path & path, bool resolveSymlinks) { + assert(path != ""); + string s; if (path[0] != '/') - throw Error(format("not an absolute path: ‘%1%’") % path); + throw Error("not an absolute path: '%1%'", path); string::const_iterator i = path.begin(), end = path.end(); string temp; @@ -135,7 +147,7 @@ Path canonPath(const Path & path, bool resolveSymlinks) the symlink target might contain new symlinks). */ if (resolveSymlinks && isLink(s)) { if (++followCount >= maxFollow) - throw Error(format("infinite symlink recursion in path ‘%1%’") % path); + throw Error("infinite symlink recursion in path '%1%'", path); temp = absPath(readLink(s), dirOf(s)) + string(i, end); i = temp.begin(); /* restart */ @@ -153,27 +165,27 @@ Path dirOf(const Path & path) { Path::size_type pos = path.rfind('/'); if (pos == string::npos) - throw Error(format("invalid file name ‘%1%’") % path); + return "."; return pos == 0 ? "/" : Path(path, 0, pos); } -string baseNameOf(const Path & path) +std::string_view baseNameOf(std::string_view path) { if (path.empty()) return ""; - Path::size_type last = path.length() - 1; + auto last = path.size() - 1; if (path[last] == '/' && last > 0) last -= 1; - Path::size_type pos = path.rfind('/', last); + auto pos = path.rfind('/', last); if (pos == string::npos) pos = 0; else pos += 1; - return string(path, pos, last - pos + 1); + return path.substr(pos, last - pos + 1); } @@ -186,11 +198,17 @@ bool isInDir(const Path & path, const Path & dir) } +bool isDirOrInDir(const Path & path, const Path & dir) +{ + return path == dir || isInDir(path, dir); +} + + struct stat lstat(const Path & path) { struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting status of ‘%1%’") % path); + throw SysError("getting status of '%1%'", path); return st; } @@ -202,7 +220,7 @@ bool pathExists(const Path & path) res = lstat(path.c_str(), &st); if (!res) return true; if (errno != ENOENT && errno != ENOTDIR) - throw SysError(format("getting status of %1%") % path); + throw SysError("getting status of %1%", path); return false; } @@ -210,17 +228,18 @@ bool pathExists(const Path & path) Path readLink(const Path & path) { checkInterrupt(); - struct stat st = lstat(path); - if (!S_ISLNK(st.st_mode)) - throw Error(format("‘%1%’ is not a symlink") % path); - char buf[st.st_size]; - ssize_t rlsize = readlink(path.c_str(), buf, st.st_size); - if (rlsize == -1) - throw SysError(format("reading symbolic link ‘%1%’") % path); - else if (rlsize > st.st_size) - throw Error(format("symbolic link ‘%1%’ size overflow %2% > %3%") - % path % rlsize % st.st_size); - return string(buf, rlsize); + std::vector buf; + for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) { + buf.resize(bufSize); + ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize); + if (rlSize == -1) + if (errno == EINVAL) + throw Error("'%1%' is not a symlink", path); + else + throw SysError("reading symbolic link '%1%'", path); + else if (rlSize < bufSize) + return string(buf.data(), rlSize); + } } @@ -231,16 +250,13 @@ bool isLink(const Path & path) } -DirEntries readDirectory(const Path & path) +DirEntries readDirectory(DIR *dir, const Path & path) { DirEntries entries; entries.reserve(64); - AutoCloseDir dir(opendir(path.c_str())); - if (!dir) throw SysError(format("opening directory ‘%1%’") % path); - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir.get())) { /* sic */ + while (errno = 0, dirent = readdir(dir)) { /* sic */ checkInterrupt(); string name = dirent->d_name; if (name == "." || name == "..") continue; @@ -252,11 +268,19 @@ DirEntries readDirectory(const Path & path) #endif ); } - if (errno) throw SysError(format("reading directory ‘%1%’") % path); + if (errno) throw SysError("reading directory '%1%'", path); return entries; } +DirEntries readDirectory(const Path & path) +{ + AutoCloseDir dir(opendir(path.c_str())); + if (!dir) throw SysError("opening directory '%1%'", path); + + return readDirectory(dir.get(), path); +} + unsigned char getFileType(const Path & path) { @@ -274,37 +298,61 @@ string readFile(int fd) if (fstat(fd, &st) == -1) throw SysError("statting file"); - auto buf = std::make_unique(st.st_size); - readFull(fd, buf.get(), st.st_size); + return drainFD(fd, true, st.st_size); +} + - return string((char *) buf.get(), st.st_size); +string readFile(const Path & path) +{ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%1%'", path); + return readFile(fd.get()); } -string readFile(const Path & path, bool drain) +void readFile(const Path & path, Sink & sink) { AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); if (!fd) - throw SysError(format("opening file ‘%1%’") % path); - return drain ? drainFD(fd.get()) : readFile(fd.get()); + throw SysError("opening file '%s'", path); + drainFD(fd.get(), sink); } -void writeFile(const Path & path, const string & s) +void writeFile(const Path & path, const string & s, mode_t mode) { - AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, 0666); + AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); if (!fd) - throw SysError(format("opening file ‘%1%’") % path); + throw SysError("opening file '%1%'", path); writeFull(fd.get(), s); } +void writeFile(const Path & path, Source & source, mode_t mode) +{ + AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); + if (!fd) + throw SysError("opening file '%1%'", path); + + std::vector buf(64 * 1024); + + while (true) { + try { + auto n = source.read(buf.data(), buf.size()); + writeFull(fd.get(), (unsigned char *) buf.data(), n); + } catch (EndOfFile &) { break; } + } +} + + string readLine(int fd) { string s; while (1) { checkInterrupt(); char ch; + // FIXME: inefficient ssize_t rd = read(fd, &ch, 1); if (rd == -1) { if (errno != EINTR) @@ -326,35 +374,62 @@ void writeLine(int fd, string s) } -static void _deletePath(const Path & path, unsigned long long & bytesFreed) +static void _deletePath(int parentfd, const Path & path, unsigned long long & bytesFreed) { checkInterrupt(); + string name(baseNameOf(path)); + struct stat st; - if (lstat(path.c_str(), &st) == -1) { + if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { if (errno == ENOENT) return; - throw SysError(format("getting status of ‘%1%’") % path); + throw SysError("getting status of '%1%'", path); } if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) - bytesFreed += st.st_blocks * 512; + bytesFreed += st.st_size; if (S_ISDIR(st.st_mode)) { /* Make the directory accessible. */ const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; if ((st.st_mode & PERM_MASK) != PERM_MASK) { - if (chmod(path.c_str(), st.st_mode | PERM_MASK) == -1) - throw SysError(format("chmod ‘%1%’") % path); + if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) + throw SysError("chmod '%1%'", path); } - for (auto & i : readDirectory(path)) - _deletePath(path + "/" + i.name, bytesFreed); + int fd = openat(parentfd, path.c_str(), O_RDONLY); + if (!fd) + throw SysError("opening directory '%1%'", path); + AutoCloseDir dir(fdopendir(fd)); + if (!dir) + throw SysError("opening directory '%1%'", path); + for (auto & i : readDirectory(dir.get(), path)) + _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed); + } + + int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; + if (unlinkat(parentfd, name.c_str(), flags) == -1) { + if (errno == ENOENT) return; + throw SysError("cannot unlink '%1%'", path); } +} + +static void _deletePath(const Path & path, unsigned long long & bytesFreed) +{ + Path dir = dirOf(path); + if (dir == "") + dir = "/"; - if (remove(path.c_str()) == -1) { + AutoCloseFD dirfd(open(dir.c_str(), O_RDONLY)); + if (!dirfd) { + // This really shouldn't fail silently, but it's left this way + // for backwards compatibility. if (errno == ENOENT) return; - throw SysError(format("cannot unlink ‘%1%’") % path); + + throw SysError("opening directory '%1%'", path); } + + _deletePath(dirfd.get(), path, bytesFreed); } @@ -367,7 +442,7 @@ void deletePath(const Path & path) void deletePath(const Path & path, unsigned long long & bytesFreed) { - Activity act(*logger, lvlDebug, format("recursively deleting path ‘%1%’") % path); + //Activity act(*logger, lvlDebug, format("recursively deleting path '%1%'") % path); bytesFreed = 0; _deletePath(path, bytesFreed); } @@ -376,7 +451,7 @@ void deletePath(const Path & path, unsigned long long & bytesFreed) static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, int & counter) { - tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR", "/tmp") : tmpRoot, true); + tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); if (includePid) return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str(); else @@ -405,25 +480,81 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix, "wheel", then "tar" will fail to unpack archives that have the setgid bit set on directories. */ if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) - throw SysError(format("setting group of directory ‘%1%’") % tmpDir); + throw SysError("setting group of directory '%1%'", tmpDir); #endif return tmpDir; } if (errno != EEXIST) - throw SysError(format("creating directory ‘%1%’") % tmpDir); + throw SysError("creating directory '%1%'", tmpDir); } } -Path getCacheDir() +std::pair createTempFile(const Path & prefix) { - Path cacheDir = getEnv("XDG_CACHE_HOME"); - if (cacheDir.empty()) { - Path homeDir = getEnv("HOME"); - if (homeDir.empty()) throw Error("$XDG_CACHE_HOME and $HOME are not set"); - cacheDir = homeDir + "/.cache"; + Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); + // Strictly speaking, this is UB, but who cares... + AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + if (!fd) + throw SysError("creating temporary file '%s'", tmpl); + return {std::move(fd), tmpl}; +} + + +std::string getUserName() +{ + auto pw = getpwuid(geteuid()); + std::string name = pw ? pw->pw_name : getEnv("USER").value_or(""); + if (name.empty()) + throw Error("cannot figure out user name"); + return name; +} + + +static Lazy getHome2([]() { + auto homeDir = getEnv("HOME"); + if (!homeDir) { + std::vector buf(16384); + struct passwd pwbuf; + struct passwd * pw; + if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0 + || !pw || !pw->pw_dir || !pw->pw_dir[0]) + throw Error("cannot determine user's home directory"); + homeDir = pw->pw_dir; } - return cacheDir; + return *homeDir; +}); + +Path getHome() { return getHome2(); } + + +Path getCacheDir() +{ + auto cacheDir = getEnv("XDG_CACHE_HOME"); + return cacheDir ? *cacheDir : getHome() + "/.cache"; +} + + +Path getConfigDir() +{ + auto configDir = getEnv("XDG_CONFIG_HOME"); + return configDir ? *configDir : getHome() + "/.config"; +} + +std::vector getConfigDirs() +{ + Path configHome = getConfigDir(); + string configDirs = getEnv("XDG_CONFIG_DIRS").value_or(""); + std::vector result = tokenizeString>(configDirs, ":"); + result.insert(result.begin(), configHome); + return result; +} + + +Path getDataDir() +{ + auto dataDir = getEnv("XDG_DATA_HOME"); + return dataDir ? *dataDir : getHome() + "/.local/share"; } @@ -436,15 +567,15 @@ Paths createDirs(const Path & path) if (lstat(path.c_str(), &st) == -1) { created = createDirs(dirOf(path)); if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) - throw SysError(format("creating directory ‘%1%’") % path); + throw SysError("creating directory '%1%'", path); st = lstat(path); created.push_back(path); } if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) - throw SysError(format("statting symlink ‘%1%’") % path); + throw SysError("statting symlink '%1%'", path); - if (!S_ISDIR(st.st_mode)) throw Error(format("‘%1%’ is not a directory") % path); + if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path); return created; } @@ -453,18 +584,27 @@ Paths createDirs(const Path & path) void createSymlink(const Path & target, const Path & link) { if (symlink(target.c_str(), link.c_str())) - throw SysError(format("creating symlink from ‘%1%’ to ‘%2%’") % link % target); + throw SysError("creating symlink from '%1%' to '%2%'", link, target); } void replaceSymlink(const Path & target, const Path & link) { - Path tmp = canonPath(dirOf(link) + "/.new_" + baseNameOf(link)); + for (unsigned int n = 0; true; n++) { + Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); + + try { + createSymlink(target, tmp); + } catch (SysError & e) { + if (e.errNo == EEXIST) continue; + throw; + } - createSymlink(target, tmp); + if (rename(tmp.c_str(), link.c_str()) != 0) + throw SysError("renaming '%1%' to '%2%'", tmp, link); - if (rename(tmp.c_str(), link.c_str()) != 0) - throw SysError(format("renaming ‘%1%’ to ‘%2%’") % tmp % link); + break; + } } @@ -487,6 +627,7 @@ void readFull(int fd, unsigned char * buf, size_t count) void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts) { while (count) { + if (allowInterrupts) checkInterrupt(); ssize_t res = write(fd, (char *) buf, count); if (res == -1 && errno != EINTR) throw SysError("writing to file"); @@ -494,7 +635,6 @@ void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterr count -= res; buf += res; } - if (allowInterrupts) checkInterrupt(); } } @@ -505,21 +645,44 @@ void writeFull(int fd, const string & s, bool allowInterrupts) } -string drainFD(int fd) +string drainFD(int fd, bool block, const size_t reserveSize) +{ + StringSink sink(reserveSize); + drainFD(fd, sink, block); + return std::move(*sink.s); +} + + +void drainFD(int fd, Sink & sink, bool block) { - string result; - unsigned char buffer[4096]; + int saved; + + Finally finally([&]() { + if (!block) { + if (fcntl(fd, F_SETFL, saved) == -1) + throw SysError("making file descriptor blocking"); + } + }); + + if (!block) { + saved = fcntl(fd, F_GETFL); + if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) + throw SysError("making file descriptor non-blocking"); + } + + std::vector buf(64 * 1024); while (1) { checkInterrupt(); - ssize_t rd = read(fd, buffer, sizeof buffer); + ssize_t rd = read(fd, buf.data(), buf.size()); if (rd == -1) { + if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) + break; if (errno != EINTR) throw SysError("reading from file"); } else if (rd == 0) break; - else result.append((char *) buffer, rd); + else sink(buf.data(), rd); } - return result; } @@ -543,7 +706,7 @@ AutoDelete::~AutoDelete() deletePath(path); else { if (remove(path.c_str()) == -1) - throw SysError(format("cannot unlink ‘%1%’") % path); + throw SysError("cannot unlink '%1%'", path); } } } catch (...) { @@ -609,7 +772,7 @@ void AutoCloseFD::close() if (fd != -1) { if (::close(fd) == -1) /* This should never happen. */ - throw SysError(format("closing file descriptor %1%") % fd); + throw SysError("closing file descriptor %1%", fd); } } @@ -678,18 +841,24 @@ Pid::operator pid_t() } -int Pid::kill(bool quiet) +int Pid::kill() { assert(pid != -1); - if (!quiet) - printError(format("killing process %1%") % pid); + debug("killing process %1%", pid); /* Send the requested signal to the child. If it has its own process group, send the signal to every process in the child process group (which hopefully includes *all* its children). */ - if (::kill(separatePG ? -pid : pid, killSignal) != 0) - printError((SysError(format("killing process %1%") % pid).msg())); + if (::kill(separatePG ? -pid : pid, killSignal) != 0) { + /* On BSDs, killing a process group will return EPERM if all + processes in the group are zombies (or something like + that). So try to detect and ignore that situation. */ +#if __FreeBSD__ || __APPLE__ + if (errno != EPERM || ::kill(pid, 0) != 0) +#endif + logError(SysError("killing process %d", pid).info()); + } return wait(); } @@ -734,7 +903,7 @@ pid_t Pid::release() void killUser(uid_t uid) { - debug(format("killing all processes running under uid ‘%1%’") % uid); + debug("killing all processes running under uid '%1%'", uid); assert(uid != 0); /* just to be safe... */ @@ -763,7 +932,7 @@ void killUser(uid_t uid) #endif if (errno == ESRCH) break; /* no more processes */ if (errno != EINTR) - throw SysError(format("cannot kill processes for uid ‘%1%’") % uid); + throw SysError("cannot kill processes for uid '%1%'", uid); } _exit(0); @@ -771,7 +940,7 @@ void killUser(uid_t uid) int status = pid.wait(); if (status != 0) - throw Error(format("cannot kill processes for uid ‘%1%’: %2%") % uid % statusToString(status)); + throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status)); /* !!! We should really do some check to make sure that there are no processes left running under `uid', but there is no portable @@ -803,7 +972,7 @@ pid_t startProcess(std::function fun, const ProcessOptions & options) { auto wrapper = [&]() { if (!options.allowVfork) - logger = makeDefaultLogger(); + logger = makeSimpleLogger(); try { #if __linux__ if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) @@ -839,63 +1008,166 @@ std::vector stringsToCharPtrs(const Strings & ss) string runProgram(Path program, bool searchPath, const Strings & args, - const string & input) + const std::optional & input) +{ + RunOptions opts(program, args); + opts.searchPath = searchPath; + opts.input = input; + + auto res = runProgram(opts); + + if (!statusOk(res.first)) + throw ExecError(res.first, fmt("program '%1%' %2%", program, statusToString(res.first))); + + return res.second; +} + +std::pair runProgram(const RunOptions & options_) +{ + RunOptions options(options_); + StringSink sink; + options.standardOut = &sink; + + int status = 0; + + try { + runProgram2(options); + } catch (ExecError & e) { + status = e.status; + } + + return {status, std::move(*sink.s)}; +} + +void runProgram2(const RunOptions & options) { checkInterrupt(); + assert(!(options.standardIn && options.input)); + + std::unique_ptr source_; + Source * source = options.standardIn; + + if (options.input) { + source_ = std::make_unique(*options.input); + source = source_.get(); + } + /* Create a pipe. */ Pipe out, in; - out.create(); - if (!input.empty()) in.create(); + if (options.standardOut) out.create(); + if (source) in.create(); + + ProcessOptions processOptions; + // vfork implies that the environment of the main process and the fork will + // be shared (technically this is undefined, but in practice that's the + // case), so we can't use it if we alter the environment + if (options.environment) + processOptions.allowVfork = false; /* Fork. */ Pid pid = startProcess([&]() { - if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) + if (options.environment) + replaceEnv(*options.environment); + if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("dupping stdout"); - if (!input.empty()) { - if (dup2(in.readSide.get(), STDIN_FILENO) == -1) - throw SysError("dupping stdin"); - } + if (options.mergeStderrToStdout) + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); + if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) + throw SysError("dupping stdin"); + + if (options.chdir && chdir((*options.chdir).c_str()) == -1) + throw SysError("chdir failed"); + if (options.gid && setgid(*options.gid) == -1) + throw SysError("setgid failed"); + /* Drop all other groups if we're setgid. */ + if (options.gid && setgroups(0, 0) == -1) + throw SysError("setgroups failed"); + if (options.uid && setuid(*options.uid) == -1) + throw SysError("setuid failed"); + + Strings args_(options.args); + args_.push_front(options.program); + + restoreSignals(); + + if (options.searchPath) + execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); + else + execv(options.program.c_str(), stringsToCharPtrs(args_).data()); - Strings args_(args); - args_.push_front(program); + throw SysError("executing '%1%'", options.program); + }, processOptions); - if (searchPath) - execvp(program.c_str(), stringsToCharPtrs(args_).data()); - else - execv(program.c_str(), stringsToCharPtrs(args_).data()); + out.writeSide = -1; - throw SysError(format("executing ‘%1%’") % program); + std::thread writerThread; + + std::promise promise; + + Finally doJoin([&]() { + if (writerThread.joinable()) + writerThread.join(); }); - out.writeSide = -1; - /* FIXME: This can deadlock if the input is too long. */ - if (!input.empty()) { + if (source) { in.readSide = -1; - writeFull(in.writeSide.get(), input); - in.writeSide = -1; + writerThread = std::thread([&]() { + try { + std::vector buf(8 * 1024); + while (true) { + size_t n; + try { + n = source->read(buf.data(), buf.size()); + } catch (EndOfFile &) { + break; + } + writeFull(in.writeSide.get(), buf.data(), n); + } + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + in.writeSide = -1; + }); } - string result = drainFD(out.readSide.get()); + if (options.standardOut) + drainFD(out.readSide.get(), *options.standardOut); /* Wait for the child to finish. */ int status = pid.wait(); - if (!statusOk(status)) - throw ExecError(status, format("program ‘%1%’ %2%") - % program % statusToString(status)); - return result; + /* Wait for the writer thread to finish. */ + if (source) promise.get_future().get(); + + if (status) + throw ExecError(status, fmt("program '%1%' %2%", options.program, statusToString(status))); } void closeMostFDs(const set & exceptions) { +#if __linux__ + try { + for (auto & s : readDirectory("/proc/self/fd")) { + auto fd = std::stoi(s.name); + if (!exceptions.count(fd)) { + debug("closing leaked FD %d", fd); + close(fd); + } + } + return; + } catch (SysError &) { + } +#endif + int maxFD = 0; maxFD = sysconf(_SC_OPEN_MAX); for (int fd = 0; fd < maxFD; ++fd) - if (fd != STDIN_FILENO && fd != STDOUT_FILENO && fd != STDERR_FILENO - && exceptions.find(fd) == exceptions.end()) + if (!exceptions.count(fd)) close(fd); /* ignore result */ } @@ -909,22 +1181,18 @@ void closeOnExec(int fd) } -void restoreSIGPIPE() -{ - struct sigaction act; - act.sa_handler = SIG_DFL; - act.sa_flags = 0; - sigemptyset(&act.sa_mask); - if (sigaction(SIGPIPE, &act, 0)) throw SysError("resetting SIGPIPE"); -} - - ////////////////////////////////////////////////////////////////////// bool _isInterrupted = false; -thread_local bool interruptThrown = false; +static thread_local bool interruptThrown = false; +thread_local std::function interruptCheck; + +void setInterruptThrown() +{ + interruptThrown = true; +} void _interrupted() { @@ -938,11 +1206,10 @@ void _interrupted() } - ////////////////////////////////////////////////////////////////////// -template C tokenizeString(const string & s, const string & separators) +template C tokenizeString(std::string_view s, const string & separators) { C result; string::size_type pos = s.find_first_not_of(separators, 0); @@ -956,31 +1223,9 @@ template C tokenizeString(const string & s, const string & separators) return result; } -template Strings tokenizeString(const string & s, const string & separators); -template StringSet tokenizeString(const string & s, const string & separators); -template vector tokenizeString(const string & s, const string & separators); - - -string concatStringsSep(const string & sep, const Strings & ss) -{ - string s; - for (auto & i : ss) { - if (s.size() != 0) s += sep; - s += i; - } - return s; -} - - -string concatStringsSep(const string & sep, const StringSet & ss) -{ - string s; - for (auto & i : ss) { - if (s.size() != 0) s += sep; - s += i; - } - return s; -} +template Strings tokenizeString(std::string_view s, const string & separators); +template StringSet tokenizeString(std::string_view s, const string & separators); +template vector tokenizeString(std::string_view s, const string & separators); string chomp(const string & s) @@ -1013,6 +1258,19 @@ string replaceStrings(const std::string & s, } +std::string rewriteStrings(const std::string & _s, const StringMap & rewrites) +{ + auto s = _s; + for (auto & i : rewrites) { + if (i.first == i.second) continue; + size_t j = 0; + while ((j = s.find(i.first, j)) != string::npos) + s.replace(j, i.first.size(), i.second); + } + return s; +} + + string statusToString(int status) { if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { @@ -1039,15 +1297,16 @@ bool statusOk(int status) } -bool hasPrefix(const string & s, const string & suffix) +bool hasPrefix(std::string_view s, std::string_view prefix) { - return s.compare(0, suffix.size(), suffix) == 0; + return s.compare(0, prefix.size(), prefix) == 0; } -bool hasSuffix(const string & s, const string & suffix) +bool hasSuffix(std::string_view s, std::string_view suffix) { - return s.size() >= suffix.size() && string(s, s.size() - suffix.size()) == suffix; + return s.size() >= suffix.size() + && s.substr(s.size() - suffix.size()) == suffix; } @@ -1060,17 +1319,12 @@ std::string toLower(const std::string & s) } -string decodeOctalEscaped(const string & s) +std::string shellEscape(const std::string & s) { - string r; - for (string::const_iterator i = s.begin(); i != s.end(); ) { - if (*i != '\\') { r += *i++; continue; } - unsigned char c = 0; - ++i; - while (i != s.end() && *i >= '0' && *i < '8') - c = c * 8 + (*i++ - '0'); - r += c; - } + std::string r = "'"; + for (auto & i : s) + if (i == '\'') r += "'\\''"; else r += i; + r += '\''; return r; } @@ -1080,41 +1334,56 @@ void ignoreException() try { throw; } catch (std::exception & e) { - printError(format("error (ignored): %1%") % e.what()); + printError("error (ignored): %1%", e.what()); } } -string filterANSIEscapes(const string & s, bool nixOnly) +std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned int width) { - string t, r; - enum { stTop, stEscape, stCSI } state = stTop; - for (auto c : s) { - if (state == stTop) { - if (c == '\e') { - state = stEscape; - r = c; - } else - t += c; - } else if (state == stEscape) { - r += c; - if (c == '[') - state = stCSI; - else { - t += r; - state = stTop; + std::string t, e; + size_t w = 0; + auto i = s.begin(); + + while (w < (size_t) width && i != s.end()) { + + if (*i == '\e') { + std::string e; + e += *i++; + char last = 0; + + if (i != s.end() && *i == '[') { + e += *i++; + // eat parameter bytes + while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; + // eat intermediate bytes + while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; + // eat final byte + if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; + } else { + if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; } - } else { - r += c; - if (c >= 0x40 && c != 0x7e) { - if (nixOnly && (c != 'p' && c != 'q' && c != 's' && c != 'a' && c != 'b')) - t += r; - state = stTop; - r.clear(); + + if (!filterAll && last == 'm') + t += e; + } + + else if (*i == '\t') { + i++; t += ' '; w++; + while (w < (size_t) width && w % 8) { + t += ' '; w++; } } + + else if (*i == '\r') + // do nothing for now + i++; + + else { + t += *i++; w++; + } } - t += r; + return t; } @@ -1122,7 +1391,7 @@ string filterANSIEscapes(const string & s, bool nixOnly) static char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; -string base64Encode(const string & s) +string base64Encode(std::string_view s) { string res; int data = 0, nbits = 0; @@ -1143,7 +1412,7 @@ string base64Encode(const string & s) } -string base64Decode(const string & s) +string base64Decode(std::string_view s) { bool init = false; char decode[256]; @@ -1178,17 +1447,26 @@ string base64Decode(const string & s) } -void callFailure(const std::function & failure, std::exception_ptr exc) +static Sync> windowSize{{0, 0}}; + + +static void updateWindowSize() { - try { - failure(exc); - } catch (std::exception & e) { - printError(format("uncaught exception: %s") % e.what()); - abort(); + struct winsize ws; + if (ioctl(2, TIOCGWINSZ, &ws) == 0) { + auto windowSize_(windowSize.lock()); + windowSize_->first = ws.ws_row; + windowSize_->second = ws.ws_col; } } +std::pair getWindowSize() +{ + return *windowSize.lock(); +} + + static Sync>> _interruptCallbacks; static void signalHandlerThread(sigset_t set) @@ -1197,36 +1475,59 @@ static void signalHandlerThread(sigset_t set) int signal = 0; sigwait(&set, &signal); - if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP) { - _isInterrupted = 1; + if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP) + triggerInterrupt(); - { - auto interruptCallbacks(_interruptCallbacks.lock()); - for (auto & callback : *interruptCallbacks) { - try { - callback(); - } catch (...) { - ignoreException(); - } - } + else if (signal == SIGWINCH) { + updateWindowSize(); + } + } +} + +void triggerInterrupt() +{ + _isInterrupted = true; + + { + auto interruptCallbacks(_interruptCallbacks.lock()); + for (auto & callback : *interruptCallbacks) { + try { + callback(); + } catch (...) { + ignoreException(); } } } } +static sigset_t savedSignalMask; + void startSignalHandlerThread() { + updateWindowSize(); + + if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) + throw SysError("quering signal mask"); + sigset_t set; sigemptyset(&set); sigaddset(&set, SIGINT); sigaddset(&set, SIGTERM); sigaddset(&set, SIGHUP); + sigaddset(&set, SIGPIPE); + sigaddset(&set, SIGWINCH); if (pthread_sigmask(SIG_BLOCK, &set, nullptr)) throw SysError("blocking signals"); std::thread(signalHandlerThread, set).detach(); } +void restoreSignals() +{ + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); +} + /* RAII helper to automatically deregister a callback. */ struct InterruptCallbackImpl : InterruptCallback { @@ -1249,4 +1550,37 @@ std::unique_ptr createInterruptCallback(std::function return std::unique_ptr(res.release()); } + +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) +{ + AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM + #ifdef SOCK_CLOEXEC + | SOCK_CLOEXEC + #endif + , 0); + if (!fdSocket) + throw SysError("cannot create Unix domain socket"); + + closeOnExec(fdSocket.get()); + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + if (path.size() >= sizeof(addr.sun_path)) + throw Error("socket path '%1%' is too long", path); + strcpy(addr.sun_path, path.c_str()); + + unlink(path.c_str()); + + if (bind(fdSocket.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot bind to socket '%1%'", path); + + if (chmod(path.c_str(), mode) == -1) + throw SysError("changing permissions on '%1%'", path); + + if (listen(fdSocket.get(), 5) == -1) + throw SysError("cannot listen on socket '%1%'", path); + + return fdSocket; +} + } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index b68d48582b3..3641daaec8c 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -1,7 +1,9 @@ #pragma once #include "types.hh" +#include "error.hh" #include "logging.hh" +#include "ansicolor.hh" #include #include @@ -13,6 +15,10 @@ #include #include #include +#include +#include +#include +#include #ifndef HAVE_STRUCT_DIRENT_D_TYPE #define DT_UNKNOWN 0 @@ -23,17 +29,27 @@ namespace nix { +struct Sink; +struct Source; + + +/* The system for which Nix is compiled. */ +extern const std::string nativeSystem; + /* Return an environment variable. */ -string getEnv(const string & key, const string & def = ""); +std::optional getEnv(const std::string & key); /* Get the entire environment. */ std::map getEnv(); +/* Clear the environment. */ +void clearEnv(); + /* Return an absolutized path, resolving paths relative to the specified directory, or the current directory otherwise. The path is also canonicalised. */ -Path absPath(Path path, Path dir = ""); +Path absPath(Path path, std::optional dir = {}); /* Canonicalise a path by removing all `.' or `..' components and double or trailing slashes. Optionally resolves all symlink @@ -43,18 +59,20 @@ Path canonPath(const Path & path, bool resolveSymlinks = false); /* Return the directory part of the given canonical path, i.e., everything before the final `/'. If the path is the root or an - immediate child thereof (e.g., `/foo'), this means an empty string - is returned. */ + immediate child thereof (e.g., `/foo'), this means `/' + is returned.*/ Path dirOf(const Path & path); /* Return the base name of the given canonical path, i.e., everything - following the final `/'. */ -string baseNameOf(const Path & path); + following the final `/' (trailing slashes are removed). */ +std::string_view baseNameOf(std::string_view path); -/* Check whether a given path is a descendant of the given - directory. */ +/* Check whether 'path' is a descendant of 'dir'. */ bool isInDir(const Path & path, const Path & dir); +/* Check whether 'path' is equal to 'dir' or a descendant of 'dir'. */ +bool isDirOrInDir(const Path & path, const Path & dir); + /* Get status of `path'. */ struct stat lstat(const Path & path); @@ -86,10 +104,13 @@ unsigned char getFileType(const Path & path); /* Read the contents of a file into a string. */ string readFile(int fd); -string readFile(const Path & path, bool drain = false); +string readFile(const Path & path); +void readFile(const Path & path, Sink & sink); /* Write a string to a file. */ -void writeFile(const Path & path, const string & s); +void writeFile(const Path & path, const string & s, mode_t mode = 0666); + +void writeFile(const Path & path, Source & source, mode_t mode = 0666); /* Read a line from a file descriptor. */ string readLine(int fd); @@ -104,13 +125,23 @@ void deletePath(const Path & path); void deletePath(const Path & path, unsigned long long & bytesFreed); -/* Create a temporary directory. */ -Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", - bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); +std::string getUserName(); -/* Return the path to $XDG_CACHE_HOME/.cache. */ +/* Return $HOME or the user's home directory from /etc/passwd. */ +Path getHome(); + +/* Return $XDG_CACHE_HOME or $HOME/.cache. */ Path getCacheDir(); +/* Return $XDG_CONFIG_HOME or $HOME/.config. */ +Path getConfigDir(); + +/* Return the directories to search for user configuration files */ +std::vector getConfigDirs(); + +/* Return $XDG_DATA_HOME or $HOME/.local/share. */ +Path getDataDir(); + /* Create a directory and all its parents, if necessary. Returns the list of created directories, in order of creation. */ Paths createDirs(const Path & path); @@ -128,12 +159,13 @@ void readFull(int fd, unsigned char * buf, size_t count); void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts = true); void writeFull(int fd, const string & s, bool allowInterrupts = true); -MakeError(EndOfFile, Error) +MakeError(EndOfFile, Error); /* Read a file descriptor until EOF occurs. */ -string drainFD(int fd); +string drainFD(int fd, bool block = true, const size_t reserveSize=0); +void drainFD(int fd, Sink & sink, bool block = true); /* Automatic cleanup of resources. */ @@ -172,6 +204,14 @@ public: }; +/* Create a temporary directory. */ +Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", + bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); + +/* Create a temporary file, returning a file handle and its path. */ +std::pair createTempFile(const Path & prefix = "nix"); + + class Pipe { public: @@ -201,7 +241,7 @@ public: ~Pid(); void operator =(pid_t pid); operator pid_t(); - int kill(bool quiet = false); + int kill(); int wait(); void setSeparatePG(bool separatePG); @@ -231,7 +271,34 @@ pid_t startProcess(std::function fun, const ProcessOptions & options = P /* Run a program and return its stdout in a string (i.e., like the shell backtick operator). */ string runProgram(Path program, bool searchPath = false, - const Strings & args = Strings(), const string & input = ""); + const Strings & args = Strings(), + const std::optional & input = {}); + +struct RunOptions +{ + std::optional uid; + std::optional gid; + std::optional chdir; + std::optional> environment; + Path program; + bool searchPath = true; + Strings args; + std::optional input; + Source * standardIn = nullptr; + Sink * standardOut = nullptr; + bool mergeStderrToStdout = false; + bool _killStderr = false; + + RunOptions(const Path & program, const Strings & args) + : program(program), args(args) { }; + + RunOptions & killStderr(bool v) { _killStderr = true; return *this; } +}; + +std::pair runProgram(const RunOptions & options); + +void runProgram2(const RunOptions & options); + class ExecError : public Error { @@ -239,7 +306,7 @@ public: int status; template - ExecError(int status, Args... args) + ExecError(int status, const Args & ... args) : Error(args...), status(status) { } }; @@ -249,45 +316,62 @@ public: list of strings. */ std::vector stringsToCharPtrs(const Strings & ss); -/* Close all file descriptors except stdin, stdout, stderr, and those - listed in the given set. Good practice in child processes. */ +/* Close all file descriptors except those listed in the given set. + Good practice in child processes. */ void closeMostFDs(const set & exceptions); /* Set the close-on-exec flag for the given file descriptor. */ void closeOnExec(int fd); -/* Restore default handling of SIGPIPE, otherwise some programs will - randomly say "Broken pipe". */ -void restoreSIGPIPE(); - /* User interruption. */ extern bool _isInterrupted; -extern thread_local bool interruptThrown; +extern thread_local std::function interruptCheck; + +void setInterruptThrown(); void _interrupted(); void inline checkInterrupt() { - if (_isInterrupted) _interrupted(); + if (_isInterrupted || (interruptCheck && interruptCheck())) + _interrupted(); } -MakeError(Interrupted, BaseError) +MakeError(Interrupted, BaseError); -MakeError(FormatError, Error) +MakeError(FormatError, Error); /* String tokenizer. */ -template C tokenizeString(const string & s, const string & separators = " \t\n\r"); +template C tokenizeString(std::string_view s, const string & separators = " \t\n\r"); /* Concatenate the given strings with a separator between the elements. */ -string concatStringsSep(const string & sep, const Strings & ss); -string concatStringsSep(const string & sep, const StringSet & ss); +template +string concatStringsSep(const string & sep, const C & ss) +{ + string s; + for (auto & i : ss) { + if (s.size() != 0) s += sep; + s += i; + } + return s; +} + + +/* Add quotes around a collection of strings. */ +template Strings quoteStrings(const C & c) +{ + Strings res; + for (auto & s : c) + res.push_back("'" + s + "'"); + return res; +} /* Remove trailing whitespace from a string. */ @@ -303,6 +387,9 @@ string replaceStrings(const std::string & s, const std::string & from, const std::string & to); +std::string rewriteStrings(const std::string & s, const StringMap & rewrites); + + /* Convert the exit status of a child as returned by wait() into an error string. */ string statusToString(int status); @@ -330,21 +417,19 @@ template bool string2Float(const string & s, N & n) /* Return true iff `s' starts with `prefix'. */ -bool hasPrefix(const string & s, const string & prefix); +bool hasPrefix(std::string_view s, std::string_view prefix); /* Return true iff `s' ends in `suffix'. */ -bool hasSuffix(const string & s, const string & suffix); +bool hasSuffix(std::string_view s, std::string_view suffix); /* Convert a string to lower case. */ std::string toLower(const std::string & s); -/* Escape a string that contains octal-encoded escape codes such as - used in /etc/fstab and /proc/mounts (e.g. "foo\040bar" decodes to - "foo bar"). */ -string decodeOctalEscaped(const string & s); +/* Escape a string as a shell word. */ +std::string shellEscape(const std::string & s); /* Exception handling in destructors: print an error message, then @@ -352,77 +437,85 @@ string decodeOctalEscaped(const string & s); void ignoreException(); -/* Some ANSI escape sequences. */ -#define ANSI_NORMAL "\e[0m" -#define ANSI_BOLD "\e[1m" -#define ANSI_RED "\e[31;1m" + +/* Tree formatting. */ +constexpr char treeConn[] = "├───"; +constexpr char treeLast[] = "└───"; +constexpr char treeLine[] = "│ "; +constexpr char treeNull[] = " "; -/* Filter out ANSI escape codes from the given string. If ‘nixOnly’ is - set, only filter escape codes generated by Nixpkgs' stdenv (used to - denote nesting etc.). */ -string filterANSIEscapes(const string & s, bool nixOnly = false); +/* Truncate a string to 'width' printable characters. If 'filterAll' + is true, all ANSI escape sequences are filtered out. Otherwise, + some escape sequences (such as colour setting) are copied but not + included in the character count. Also, tabs are expanded to + spaces. */ +std::string filterANSIEscapes(const std::string & s, + bool filterAll = false, + unsigned int width = std::numeric_limits::max()); /* Base64 encoding/decoding. */ -string base64Encode(const string & s); -string base64Decode(const string & s); +string base64Encode(std::string_view s); +string base64Decode(std::string_view s); -/* Get a value for the specified key from an associate container, or a - default value if the key doesn't exist. */ +/* Get a value for the specified key from an associate container. */ template -string get(const T & map, const string & key, const string & def = "") +std::optional get(const T & map, const typename T::key_type & key) { auto i = map.find(key); - return i == map.end() ? def : i->second; + if (i == map.end()) return {}; + return std::optional(i->second); } -/* Call ‘failure’ with the current exception as argument. If ‘failure’ - throws an exception, abort the program. */ -void callFailure(const std::function & failure, - std::exception_ptr exc = std::current_exception()); +/* A callback is a wrapper around a lambda that accepts a valid of + type T or an exception. (We abuse std::future to pass the value or + exception.) */ +template +class Callback +{ + std::function)> fun; + std::atomic_flag done = ATOMIC_FLAG_INIT; +public: -/* Evaluate the function ‘f’. If it returns a value, call ‘success’ - with that value as its argument. If it or ‘success’ throws an - exception, call ‘failure’. If ‘failure’ throws an exception, abort - the program. */ -template -void sync2async( - const std::function & success, - const std::function & failure, - const std::function & f) -{ - try { - success(f()); - } catch (...) { - callFailure(failure); + Callback(std::function)> fun) : fun(fun) { } + + Callback(Callback && callback) : fun(std::move(callback.fun)) + { + auto prev = callback.done.test_and_set(); + if (prev) done.test_and_set(); } -} + void operator()(T && t) noexcept + { + auto prev = done.test_and_set(); + assert(!prev); + std::promise promise; + promise.set_value(std::move(t)); + fun(promise.get_future()); + } -/* Call the function ‘success’. If it throws an exception, call - ‘failure’. If that throws an exception, abort the program. */ -template -void callSuccess( - const std::function & success, - const std::function & failure, - T && arg) -{ - try { - success(arg); - } catch (...) { - callFailure(failure); + void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept + { + auto prev = done.test_and_set(); + assert(!prev); + std::promise promise; + promise.set_exception(exc); + fun(promise.get_future()); } -} +}; /* Start a thread that handles various signals. Also block those signals on the current thread (and thus any threads created by it). */ void startSignalHandlerThread(); +/* Restore default signal handling. */ +void restoreSignals(); + struct InterruptCallback { virtual ~InterruptCallback() { }; @@ -433,5 +526,75 @@ struct InterruptCallback std::unique_ptr createInterruptCallback( std::function callback); +void triggerInterrupt(); + +/* A RAII class that causes the current thread to receive SIGUSR1 when + the signal handler thread receives SIGINT. That is, this allows + SIGINT to be multiplexed to multiple threads. */ +struct ReceiveInterrupts +{ + pthread_t target; + std::unique_ptr callback; + + ReceiveInterrupts() + : target(pthread_self()) + , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); })) + { } +}; + + + +/* A RAII helper that increments a counter on construction and + decrements it on destruction. */ +template +struct MaintainCount +{ + T & counter; + long delta; + MaintainCount(T & counter, long delta = 1) : counter(counter), delta(delta) { counter += delta; } + ~MaintainCount() { counter -= delta; } +}; + + +/* Return the number of rows and columns of the terminal. */ +std::pair getWindowSize(); + + +/* Used in various places. */ +typedef std::function PathFilter; + +extern PathFilter defaultPathFilter; + + +/* Create a Unix domain socket in listen mode. */ +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); + + +// A Rust/Python-like enumerate() iterator adapter. +// Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. +template ())), + typename = decltype(std::end(std::declval()))> +constexpr auto enumerate(T && iterable) +{ + struct iterator + { + size_t i; + TIter iter; + bool operator != (const iterator & other) const { return iter != other.iter; } + void operator ++ () { ++i; ++iter; } + auto operator * () const { return std::tie(i, *iter); } + }; + + struct iterable_wrapper + { + T iterable; + auto begin() { return iterator{ 0, std::begin(iterable) }; } + auto end() { return iterator{ 0, std::end(iterable) }; } + }; + + return iterable_wrapper{ std::forward(iterable) }; +} + } diff --git a/src/libutil/xml-writer.cc b/src/libutil/xml-writer.cc index 98bd058d18b..68857e34dc1 100644 --- a/src/libutil/xml-writer.cc +++ b/src/libutil/xml-writer.cc @@ -1,10 +1,10 @@ -#include +#include #include "xml-writer.hh" namespace nix { - + XMLWriter::XMLWriter(bool indent, std::ostream & output) : output(output), indent(indent) @@ -28,7 +28,7 @@ void XMLWriter::close() } -void XMLWriter::indent_(unsigned int depth) +void XMLWriter::indent_(size_t depth) { if (!indent) return; output << string(depth * 2, ' '); @@ -75,7 +75,7 @@ void XMLWriter::writeAttrs(const XMLAttrs & attrs) { for (auto & i : attrs) { output << " " << i.first << "=\""; - for (unsigned int j = 0; j < i.second.size(); ++j) { + for (size_t j = 0; j < i.second.size(); ++j) { char c = i.second[j]; if (c == '"') output << """; else if (c == '<') output << "<"; diff --git a/src/libutil/xml-writer.hh b/src/libutil/xml-writer.hh index 3cefe3712c0..b98b445265a 100644 --- a/src/libutil/xml-writer.hh +++ b/src/libutil/xml-writer.hh @@ -44,7 +44,7 @@ public: private: void writeAttrs(const XMLAttrs & attrs); - void indent_(unsigned int depth); + void indent_(size_t depth); }; diff --git a/src/nix-build/local.mk b/src/nix-build/local.mk deleted file mode 100644 index 91532411a50..00000000000 --- a/src/nix-build/local.mk +++ /dev/null @@ -1,9 +0,0 @@ -programs += nix-build - -nix-build_DIR := $(d) - -nix-build_SOURCES := $(d)/nix-build.cc - -nix-build_LIBS = libmain libstore libutil libformat - -$(eval $(call install-symlink, nix-build, $(bindir)/nix-shell)) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 3eb2d2c0b7a..da6f005a314 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -5,16 +5,21 @@ #include #include -#include - #include "store-api.hh" #include "globals.hh" #include "derivations.hh" #include "affinity.hh" #include "util.hh" #include "shared.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "get-drvs.hh" +#include "common-eval-args.hh" +#include "attr-path.hh" +#include "../nix/legacy.hh" using namespace nix; +using namespace std::string_literals; extern char * * environ; @@ -62,450 +67,472 @@ std::vector shellwords(const string & s) return res; } -static void maybePrintExecError(ExecError & e) +static void _main(int argc, char * * argv) { - if (WIFEXITED(e.status)) - throw Exit(WEXITSTATUS(e.status)); - else - throw e; -} + auto dryRun = false; + auto runEnv = std::regex_search(argv[0], std::regex("nix-shell$")); + auto pure = false; + auto fromArgs = false; + auto packages = false; + // Same condition as bash uses for interactive shells + auto interactive = isatty(STDIN_FILENO) && isatty(STDERR_FILENO); + Strings attrPaths; + Strings left; + RepairFlag repair = NoRepair; + Path gcRoot; + BuildMode buildMode = bmNormal; + bool readStdin = false; + + std::string envCommand; // interactive shell + Strings envExclude; + + auto myName = runEnv ? "nix-shell" : "nix-build"; + + auto inShebang = false; + std::string script; + std::vector savedArgs; + + AutoDelete tmpDir(createTempDir("", myName)); + + std::string outLink = "./result"; + + // List of environment variables kept for --pure + std::set keepVars{ + "HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", + "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL", + "http_proxy", "https_proxy", "ftp_proxy", "all_proxy", "no_proxy" + }; -int main(int argc, char ** argv) -{ - return handleExceptions(argv[0], [&]() { - initNix(); - auto store = openStore(); - auto dryRun = false; - auto verbose = false; - auto runEnv = std::regex_search(argv[0], std::regex("nix-shell$")); - auto pure = false; - auto fromArgs = false; - auto packages = false; - // Same condition as bash uses for interactive shells - auto interactive = isatty(STDIN_FILENO) && isatty(STDERR_FILENO); - - Strings instArgs; - Strings buildArgs; - Strings exprs; - - auto shell = getEnv("SHELL", "/bin/sh"); - std::string envCommand; // interactive shell - Strings envExclude; - - auto myName = runEnv ? "nix-shell" : "nix-build"; - - auto inShebang = false; - std::string script; - std::vector savedArgs; - - AutoDelete tmpDir(createTempDir("", myName)); - - std::string outLink = "./result"; - auto drvLink = (Path) tmpDir + "/derivation"; - - std::vector args; - for (int i = 1; i < argc; ++i) - args.push_back(argv[i]); - - // Heuristic to see if we're invoked as a shebang script, namely, if we - // have a single argument, it's the name of an executable file, and it - // starts with "#!". - if (runEnv && argc > 1 && !std::regex_search(argv[1], std::regex("nix-shell"))) { - script = argv[1]; - if (access(script.c_str(), F_OK) == 0 && access(script.c_str(), X_OK) == 0) { - auto lines = tokenizeString(readFile(script), "\n"); - if (std::regex_search(lines.front(), std::regex("^#!"))) { - lines.pop_front(); - inShebang = true; - for (int i = 2; i < argc; ++i) - savedArgs.push_back(argv[i]); - args.clear(); - for (auto line : lines) { - line = chomp(line); - std::smatch match; - if (std::regex_match(line, match, std::regex("^#!\\s*nix-shell (.*)$"))) - for (const auto & word : shellwords(match[1].str())) - args.push_back(word); - } + Strings args; + for (int i = 1; i < argc; ++i) + args.push_back(argv[i]); + + // Heuristic to see if we're invoked as a shebang script, namely, + // if we have at least one argument, it's the name of an + // executable file, and it starts with "#!". + if (runEnv && argc > 1) { + script = argv[1]; + try { + auto lines = tokenizeString(readFile(script), "\n"); + if (std::regex_search(lines.front(), std::regex("^#!"))) { + lines.pop_front(); + inShebang = true; + for (int i = 2; i < argc; ++i) + savedArgs.push_back(argv[i]); + args.clear(); + for (auto line : lines) { + line = chomp(line); + std::smatch match; + if (std::regex_match(line, match, std::regex("^#!\\s*nix-shell (.*)$"))) + for (const auto & word : shellwords(match[1].str())) + args.push_back(word); } } + } catch (SysError &) { } + } + + struct MyArgs : LegacyArgs, MixEvalArgs + { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(myName, [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") { + deletePath(tmpDir); + showManPage(myName); } - for (size_t n = 0; n < args.size(); ++n) { - auto arg = args[n]; + else if (*arg == "--version") + printVersion(myName); - if (arg == "--help") { - deletePath(tmpDir); - showManPage(myName); - } + else if (*arg == "--add-drv-link" || *arg == "--indirect") + ; // obsolete - else if (arg == "--version") - printVersion(myName); + else if (*arg == "--no-out-link" || *arg == "--no-link") + outLink = (Path) tmpDir + "/result"; - else if (arg == "--add-drv-link") { - drvLink = "./derivation"; - } + else if (*arg == "--attr" || *arg == "-A") + attrPaths.push_back(getArg(*arg, arg, end)); - else if (arg == "--no-out-link" || arg == "--no-link") { - outLink = (Path) tmpDir + "/result"; - } + else if (*arg == "--drv-link") + getArg(*arg, arg, end); // obsolete - else if (arg == "--drv-link") { - n++; - if (n >= args.size()) { - throw UsageError("--drv-link requires an argument"); - } - drvLink = args[n]; - } + else if (*arg == "--out-link" || *arg == "-o") + outLink = getArg(*arg, arg, end); - else if (arg == "--out-link" || arg == "-o") { - n++; - if (n >= args.size()) { - throw UsageError(format("%1% requires an argument") % arg); - } - outLink = args[n]; - } + else if (*arg == "--add-root") + gcRoot = getArg(*arg, arg, end); - else if (arg == "--attr" || arg == "-A" || arg == "-I") { - n++; - if (n >= args.size()) { - throw UsageError(format("%1% requires an argument") % arg); - } - instArgs.push_back(arg); - instArgs.push_back(args[n]); - } + else if (*arg == "--dry-run") + dryRun = true; - else if (arg == "--arg" || arg == "--argstr") { - if (n + 2 >= args.size()) { - throw UsageError(format("%1% requires two arguments") % arg); - } - instArgs.push_back(arg); - instArgs.push_back(args[n + 1]); - instArgs.push_back(args[n + 2]); - n += 2; - } + else if (*arg == "--repair") { + repair = Repair; + buildMode = bmRepair; + } - else if (arg == "--option") { - if (n + 2 >= args.size()) { - throw UsageError(format("%1% requires two arguments") % arg); - } - instArgs.push_back(arg); - instArgs.push_back(args[n + 1]); - instArgs.push_back(args[n + 2]); - buildArgs.push_back(arg); - buildArgs.push_back(args[n + 1]); - buildArgs.push_back(args[n + 2]); - n += 2; - } + else if (*arg == "--run-env") // obsolete + runEnv = true; - else if (arg == "--max-jobs" || arg == "-j" || arg == "--max-silent-time" || arg == "--cores" || arg == "--timeout" || arg == "--add-root") { - n++; - if (n >= args.size()) { - throw UsageError(format("%1% requires an argument") % arg); - } - buildArgs.push_back(arg); - buildArgs.push_back(args[n]); - } + else if (*arg == "--command" || *arg == "--run") { + if (*arg == "--run") + interactive = false; + envCommand = getArg(*arg, arg, end) + "\nexit"; + } - else if (arg == "--dry-run") { - buildArgs.push_back("--dry-run"); - dryRun = true; - } + else if (*arg == "--check") + buildMode = bmCheck; - else if (arg == "--show-trace") { - instArgs.push_back(arg); - } + else if (*arg == "--exclude") + envExclude.push_back(getArg(*arg, arg, end)); - else if (arg == "-") { - exprs = Strings{"-"}; - } + else if (*arg == "--expr" || *arg == "-E") + fromArgs = true; - else if (arg == "--verbose" || (arg.size() >= 2 && arg.substr(0, 2) == "-v")) { - buildArgs.push_back(arg); - instArgs.push_back(arg); - verbose = true; - } + else if (*arg == "--pure") pure = true; + else if (*arg == "--impure") pure = false; - else if (arg == "--quiet" || arg == "--repair") { - buildArgs.push_back(arg); - instArgs.push_back(arg); - } + else if (*arg == "--packages" || *arg == "-p") + packages = true; - else if (arg == "--check") { - buildArgs.push_back(arg); - } + else if (inShebang && *arg == "-i") { + auto interpreter = getArg(*arg, arg, end); + interactive = false; + auto execArgs = ""; - else if (arg == "--run-env") { // obsolete - runEnv = true; - } + // Überhack to support Perl. Perl examines the shebang and + // executes it unless it contains the string "perl" or "indir", + // or (undocumented) argv[0] does not contain "perl". Exploit + // the latter by doing "exec -a". + if (std::regex_search(interpreter, std::regex("perl"))) + execArgs = "-a PERL"; - else if (arg == "--command" || arg == "--run") { - n++; - if (n >= args.size()) { - throw UsageError(format("%1% requires an argument") % arg); - } - envCommand = args[n] + "\nexit"; - if (arg == "--run") - interactive = false; + std::ostringstream joined; + for (const auto & i : savedArgs) + joined << shellEscape(i) << ' '; + + if (std::regex_search(interpreter, std::regex("ruby"))) { + // Hack for Ruby. Ruby also examines the shebang. It tries to + // read the shebang to understand which packages to read from. Since + // this is handled via nix-shell -p, we wrap our ruby script execution + // in ruby -e 'load' which ignores the shebangs. + envCommand = (format("exec %1% %2% -e 'load(\"%3%\")' -- %4%") % execArgs % interpreter % script % joined.str()).str(); + } else { + envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % script % joined.str()).str(); } + } - else if (arg == "--exclude") { - n++; - if (n >= args.size()) { - throw UsageError(format("%1% requires an argument") % arg); - } - envExclude.push_back(args[n]); - } + else if (*arg == "--keep") + keepVars.insert(getArg(*arg, arg, end)); - else if (arg == "--pure") { pure = true; } - else if (arg == "--impure") { pure = false; } + else if (*arg == "-") + readStdin = true; - else if (arg == "--expr" || arg == "-E") { - fromArgs = true; - instArgs.push_back("--expr"); - } + else if (*arg != "" && arg->at(0) == '-') + return false; - else if (arg == "--packages" || arg == "-p") { - packages = true; - } + else + left.push_back(*arg); - else if (inShebang && arg == "-i") { - n++; - if (n >= args.size()) { - throw UsageError(format("%1% requires an argument") % arg); - } - interactive = false; - auto interpreter = args[n]; - auto execArgs = ""; - - auto shellEscape = [](const string & s) { - return "'" + std::regex_replace(s, std::regex("'"), "'\\''") + "'"; - }; - - // Überhack to support Perl. Perl examines the shebang and - // executes it unless it contains the string "perl" or "indir", - // or (undocumented) argv[0] does not contain "perl". Exploit - // the latter by doing "exec -a". - if (std::regex_search(interpreter, std::regex("perl"))) - execArgs = "-a PERL"; - - std::ostringstream joined; - for (const auto & i : savedArgs) - joined << shellEscape(i) << ' '; - - if (std::regex_search(interpreter, std::regex("ruby"))) { - // Hack for Ruby. Ruby also examines the shebang. It tries to - // read the shebang to understand which packages to read from. Since - // this is handled via nix-shell -p, we wrap our ruby script execution - // in ruby -e 'load' which ignores the shebangs. - envCommand = (format("exec %1% %2% -e 'load(\"%3%\") -- %4%") % execArgs % interpreter % script % joined.str()).str(); - } else { - envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % script % joined.str()).str(); - } - } + return true; + }); - else if (!arg.empty() && arg[0] == '-') { - buildArgs.push_back(arg); - } + myArgs.parseCmdline(args); - else if (arg == "-Q" || arg == "--no-build-output") { - buildArgs.push_back(arg); - instArgs.push_back(arg); - } + initPlugins(); + if (packages && fromArgs) + throw UsageError("'-p' and '-E' are mutually exclusive"); + + auto store = openStore(); + + auto state = std::make_unique(myArgs.searchPath, store); + state->repair = repair; + + auto autoArgs = myArgs.getAutoArgs(*state); + + if (runEnv) { + auto newArgs = state->allocBindings(autoArgs->size() + 1); + auto tru = state->allocValue(); + mkBool(*tru, true); + newArgs->push_back(Attr(state->symbols.create("inNixShell"), tru)); + for (auto & i : *autoArgs) newArgs->push_back(i); + newArgs->sort(); + autoArgs = newArgs; + } + + if (packages) { + std::ostringstream joined; + joined << "with import { }; (pkgs.runCommandCC or pkgs.runCommand) \"shell\" { buildInputs = [ "; + for (const auto & i : left) + joined << '(' << i << ") "; + joined << "]; } \"\""; + fromArgs = true; + left = {joined.str()}; + } else if (!fromArgs) { + if (left.empty() && runEnv && pathExists("shell.nix")) + left = {"shell.nix"}; + if (left.empty()) + left = {"default.nix"}; + } + + if (runEnv) + setenv("IN_NIX_SHELL", pure ? "pure" : "impure", 1); + + DrvInfos drvs; + + /* Parse the expressions. */ + std::vector exprs; + + if (readStdin) + exprs = {state->parseStdin()}; + else + for (auto i : left) { + if (fromArgs) + exprs.push_back(state->parseExprFromString(i, absPath("."))); else { - exprs.push_back(arg); + auto absolute = i; + try { + absolute = canonPath(absPath(i), true); + } catch (Error & e) {}; + auto [path, outputNames] = parsePathWithOutputs(absolute); + if (store->isStorePath(path) && hasSuffix(path, ".drv")) + drvs.push_back(DrvInfo(*state, store, absolute)); + else + /* If we're in a #! script, interpret filenames + relative to the script. */ + exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, + inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))))); } } - if (packages && fromArgs) { - throw UsageError("‘-p’ and ‘-E’ are mutually exclusive"); - } + /* Evaluate them into derivations. */ + if (attrPaths.empty()) attrPaths = {""}; - if (packages) { - instArgs.push_back("--expr"); - std::ostringstream joined; - joined << "with import { }; runCommand \"shell\" { buildInputs = [ "; - for (const auto & i : exprs) - joined << '(' << i << ") "; - joined << "]; } \"\""; - exprs = Strings{joined.str()}; - } else if (!fromArgs) { - if (exprs.empty() && runEnv && access("shell.nix", F_OK) == 0) - exprs.push_back("shell.nix"); - if (exprs.empty()) - exprs.push_back("default.nix"); + for (auto e : exprs) { + Value vRoot; + state->eval(e, vRoot); + + for (auto & i : attrPaths) { + Value & v(*findAlongAttrPath(*state, i, *autoArgs, vRoot).first); + state->forceValue(v); + getDerivations(*state, v, "", *autoArgs, drvs, false); } + } - if (runEnv) - setenv("IN_NIX_SHELL", pure ? "pure" : "impure", 1); - - for (auto & expr : exprs) { - // Instantiate. - std::vector drvPaths; - if (!std::regex_match(expr, std::regex("^/.*\\.drv$"))) { - // If we're in a #! script, interpret filenames relative to the - // script. - if (inShebang && !packages) - expr = absPath(expr, dirOf(script)); - - Strings instantiateArgs{"--add-root", drvLink, "--indirect"}; - for (const auto & arg : instArgs) - instantiateArgs.push_back(arg); - instantiateArgs.push_back(expr); - try { - auto instOutput = runProgram(settings.nixBinDir + "/nix-instantiate", false, instantiateArgs); - drvPaths = tokenizeString>(instOutput); - } catch (ExecError & e) { - maybePrintExecError(e); - } - } else { - drvPaths.push_back(expr); - } + state->printStats(); - if (runEnv) { - if (drvPaths.size() != 1) - throw UsageError("a single derivation is required"); - auto drvPath = drvPaths[0]; - drvPath = drvPath.substr(0, drvPath.find_first_of('!')); - if (isLink(drvPath)) - drvPath = readLink(drvPath); - auto drv = store->derivationFromPath(drvPath); - - // Build or fetch all dependencies of the derivation. - Strings nixStoreArgs{"-r", "--no-output", "--no-gc-warning"}; - for (const auto & arg : buildArgs) - nixStoreArgs.push_back(arg); - for (const auto & input : drv.inputDrvs) - if (std::all_of(envExclude.cbegin(), envExclude.cend(), [&](const string & exclude) { return !std::regex_search(input.first, std::regex(exclude)); })) - nixStoreArgs.push_back(input.first); - for (const auto & src : drv.inputSrcs) - nixStoreArgs.push_back(src); + auto buildPaths = [&](const std::vector & paths) { + /* Note: we do this even when !printMissing to efficiently + fetch binary cache data. */ + unsigned long long downloadSize, narSize; + StorePathSet willBuild, willSubstitute, unknown; + store->queryMissing(paths, + willBuild, willSubstitute, unknown, downloadSize, narSize); - try { - runProgram(settings.nixBinDir + "/nix-store", false, nixStoreArgs); - } catch (ExecError & e) { - maybePrintExecError(e); - } + if (settings.printMissing) + printMissing(ref(store), willBuild, willSubstitute, unknown, downloadSize, narSize); - // Set the environment. - auto env = getEnv(); + if (!dryRun) + store->buildPaths(paths, buildMode); + }; - auto tmp = getEnv("TMPDIR", getEnv("XDG_RUNTIME_DIR", "/tmp")); + if (runEnv) { + if (drvs.size() != 1) + throw UsageError("nix-shell requires a single derivation"); - if (pure) { - std::set keepVars{"HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL"}; - decltype(env) newEnv; - for (auto & i : env) - if (keepVars.count(i.first)) - newEnv.emplace(i); - env = newEnv; - // NixOS hack: prevent /etc/bashrc from sourcing /etc/profile. - env["__ETC_PROFILE_SOURCED"] = "1"; - } + auto & drvInfo = drvs.front(); + auto drv = store->derivationFromPath(store->parseStorePath(drvInfo.queryDrvPath())); - env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmp; - env["NIX_STORE"] = store->storeDir; - - for (auto & var : drv.env) - env.emplace(var); - - restoreAffinity(); - - // Run a shell using the derivation's environment. For - // convenience, source $stdenv/setup to setup additional - // environment variables and shell functions. Also don't lose - // the current $PATH directories. - auto rcfile = (Path) tmpDir + "/rc"; - writeFile(rcfile, fmt( - "rm -rf '%1%'; " - "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc; " - "%2%" - "dontAddDisableDepTrack=1; " - "[ -e $stdenv/setup ] && source $stdenv/setup; " - "%3%" - "set +e; " - "[ -n \"$PS1\" ] && PS1=\"\\n\\[\\033[1;32m\\][nix-shell:\\w]$\\[\\033[0m\\] \"; " - "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; " - "unset NIX_ENFORCE_PURITY; " - "unset NIX_INDENT_MAKE; " - "shopt -u nullglob; " - "unset TZ; %4%" - "%5%", - (Path) tmpDir, - (pure ? "" : "p=$PATH; "), - (pure ? "" : "PATH=$PATH:$p; unset p; "), - (getenv("TZ") ? (string("export TZ='") + getenv("TZ") + "'; ") : ""), - envCommand)); - - Strings envStrs; - for (auto & i : env) - envStrs.push_back(i.first + "=" + i.second); - - auto args = interactive - ? Strings{"bash", "--rcfile", rcfile} - : Strings{"bash", rcfile}; - - auto envPtrs = stringsToCharPtrs(envStrs); - - environ = envPtrs.data(); - - auto argPtrs = stringsToCharPtrs(args); - - execvp(getEnv("NIX_BUILD_SHELL", "bash").c_str(), argPtrs.data()); - - throw SysError("executing shell"); - } + std::vector pathsToBuild; + + /* Figure out what bash shell to use. If $NIX_BUILD_SHELL + is not set, then build bashInteractive from + . */ + auto shell = getEnv("NIX_BUILD_SHELL"); + + if (!shell) { - // Ugly hackery to make "nix-build -A foo.all" produce symlinks - // ./result, ./result-dev, and so on, rather than ./result, - // ./result-2-dev, and so on. This combines multiple derivation - // paths into one "/nix/store/drv-path!out1,out2,..." argument. - std::string prevDrvPath; - Strings drvPaths2; - for (const auto & drvPath : drvPaths) { - auto p = drvPath; - std::string output = "out"; - std::smatch match; - if (std::regex_match(drvPath, match, std::regex("(.*)!(.*)"))) { - p = match[1].str(); - output = match[2].str(); - } - auto target = readLink(p); - if (verbose) - std::cerr << "derivation is " << target << '\n'; - if (target == prevDrvPath) { - auto last = drvPaths2.back(); - drvPaths2.pop_back(); - drvPaths2.push_back(last + "," + output); - } else { - drvPaths2.push_back(target + "!" + output); - prevDrvPath = target; - } - } - // Build. - Strings outPaths; - Strings nixStoreArgs{"--add-root", outLink, "--indirect", "-r"}; - for (const auto & arg : buildArgs) - nixStoreArgs.push_back(arg); - for (const auto & path : drvPaths2) - nixStoreArgs.push_back(path); - - std::string nixStoreRes; try { - nixStoreRes = runProgram(settings.nixBinDir + "/nix-store", false, nixStoreArgs); - } catch (ExecError & e) { - maybePrintExecError(e); + auto expr = state->parseExprFromString("(import {}).bashInteractive", absPath(".")); + + Value v; + state->eval(expr, v); + + auto drv = getDerivation(*state, v, false); + if (!drv) + throw Error("the 'bashInteractive' attribute in did not evaluate to a derivation"); + + pathsToBuild.emplace_back(store->parseStorePath(drv->queryDrvPath())); + + shell = drv->queryOutPath() + "/bin/bash"; + + } catch (Error & e) { + logWarning({ + .name = "bashInteractive", + .hint = hintfmt("%s; will use bash from your environment", + (e.info().hint ? e.info().hint->str() : "")) + }); + shell = "bash"; } + } + + // Build or fetch all dependencies of the derivation. + for (const auto & input : drv.inputDrvs) + if (std::all_of(envExclude.cbegin(), envExclude.cend(), + [&](const string & exclude) { return !std::regex_search(store->printStorePath(input.first), std::regex(exclude)); })) + pathsToBuild.emplace_back(input.first, input.second); + for (const auto & src : drv.inputSrcs) + pathsToBuild.emplace_back(src); + + buildPaths(pathsToBuild); - for (const auto & outpath : tokenizeString>(nixStoreRes)) - outPaths.push_back(chomp(outpath)); + if (dryRun) return; - if (dryRun) - continue; + // Set the environment. + auto env = getEnv(); - for (const auto & outPath : outPaths) - std::cout << readLink(outPath) << '\n'; + auto tmp = getEnv("TMPDIR"); + if (!tmp) tmp = getEnv("XDG_RUNTIME_DIR").value_or("/tmp"); + + if (pure) { + decltype(env) newEnv; + for (auto & i : env) + if (keepVars.count(i.first)) + newEnv.emplace(i); + env = newEnv; + // NixOS hack: prevent /etc/bashrc from sourcing /etc/profile. + env["__ETC_PROFILE_SOURCED"] = "1"; } - }); + + env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = *tmp; + env["NIX_STORE"] = store->storeDir; + env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores); + + auto passAsFile = tokenizeString(get(drv.env, "passAsFile").value_or("")); + + bool keepTmp = false; + int fileNr = 0; + + for (auto & var : drv.env) + if (passAsFile.count(var.first)) { + keepTmp = true; + string fn = ".attr-" + std::to_string(fileNr++); + Path p = (Path) tmpDir + "/" + fn; + writeFile(p, var.second); + env[var.first + "Path"] = p; + } else + env[var.first] = var.second; + + restoreAffinity(); + + /* Run a shell using the derivation's environment. For + convenience, source $stdenv/setup to setup additional + environment variables and shell functions. Also don't + lose the current $PATH directories. */ + auto rcfile = (Path) tmpDir + "/rc"; + writeFile(rcfile, fmt( + R"(_nix_shell_clean_tmpdir() { rm -rf %1%; }; )"s + + (keepTmp ? + "trap _nix_shell_clean_tmpdir EXIT; " + "exitHooks+=(_nix_shell_clean_tmpdir); " + "failureHooks+=(_nix_shell_clean_tmpdir); ": + "_nix_shell_clean_tmpdir; ") + + (pure ? "" : "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;") + + "%2%" + "dontAddDisableDepTrack=1; " + "[ -e $stdenv/setup ] && source $stdenv/setup; " + "%3%" + "PATH=%4%:\"$PATH\"; " + "SHELL=%5%; " + "set +e; " + R"s([ -n "$PS1" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s" + "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; " + "unset NIX_ENFORCE_PURITY; " + "shopt -u nullglob; " + "unset TZ; %6%" + "%7%", + shellEscape(tmpDir), + (pure ? "" : "p=$PATH; "), + (pure ? "" : "PATH=$PATH:$p; unset p; "), + shellEscape(dirOf(*shell)), + shellEscape(*shell), + (getenv("TZ") ? (string("export TZ=") + shellEscape(getenv("TZ")) + "; ") : ""), + envCommand)); + + Strings envStrs; + for (auto & i : env) + envStrs.push_back(i.first + "=" + i.second); + + auto args = interactive + ? Strings{"bash", "--rcfile", rcfile} + : Strings{"bash", rcfile}; + + auto envPtrs = stringsToCharPtrs(envStrs); + + environ = envPtrs.data(); + + auto argPtrs = stringsToCharPtrs(args); + + restoreSignals(); + + logger->stop(); + + execvp(shell->c_str(), argPtrs.data()); + + throw SysError("executing shell '%s'", *shell); + } + + else { + + std::vector pathsToBuild; + + std::map drvPrefixes; + std::map resultSymlinks; + std::vector outPaths; + + for (auto & drvInfo : drvs) { + auto drvPath = drvInfo.queryDrvPath(); + auto outPath = drvInfo.queryOutPath(); + + auto outputName = drvInfo.queryOutputName(); + if (outputName == "") + throw Error("derivation '%s' lacks an 'outputName' attribute", drvPath); + + pathsToBuild.emplace_back(store->parseStorePath(drvPath), StringSet{outputName}); + + std::string drvPrefix; + auto i = drvPrefixes.find(drvPath); + if (i != drvPrefixes.end()) + drvPrefix = i->second; + else { + drvPrefix = outLink; + if (drvPrefixes.size()) + drvPrefix += fmt("-%d", drvPrefixes.size() + 1); + drvPrefixes[drvPath] = drvPrefix; + } + + std::string symlink = drvPrefix; + if (outputName != "out") symlink += "-" + outputName; + + resultSymlinks[symlink] = outPath; + outPaths.push_back(outPath); + } + + buildPaths(pathsToBuild); + + if (dryRun) return; + + for (auto & symlink : resultSymlinks) + if (auto store2 = store.dynamic_pointer_cast()) + store2->addPermRoot(store->parseStorePath(symlink.second), absPath(symlink.first), true); + + logger->stop(); + + for (auto & path : outPaths) + std::cout << path << '\n'; + } } + +static RegisterLegacyCommand s1("nix-build", _main); +static RegisterLegacyCommand s2("nix-shell", _main); diff --git a/src/nix-channel/local.mk b/src/nix-channel/local.mk deleted file mode 100644 index 49fc105c6f7..00000000000 --- a/src/nix-channel/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-channel - -nix-channel_DIR := $(d) - -nix-channel_LIBS = libmain libutil libformat libstore - -nix-channel_SOURCES := $(d)/nix-channel.cc diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 36162782312..3ccf620c946 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -1,17 +1,20 @@ #include "shared.hh" #include "globals.hh" -#include "download.hh" +#include "filetransfer.hh" +#include "store-api.hh" +#include "../nix/legacy.hh" +#include "fetchers.hh" + #include #include -#include "store-api.hh" #include using namespace nix; typedef std::map Channels; -static auto channels = Channels{}; -static auto channelsList = Path{}; +static Channels channels; +static Path channelsList; // Reads the list of channels. static void readChannels() @@ -25,7 +28,7 @@ static void readChannels() continue; auto split = tokenizeString>(line, " "); auto url = std::regex_replace(split[0], std::regex("/*$"), ""); - auto name = split.size() > 1 ? split[1] : baseNameOf(url); + auto name = split.size() > 1 ? split[1] : std::string(baseNameOf(url)); channels[name] = url; } } @@ -35,7 +38,7 @@ static void writeChannels() { auto channelsFD = AutoCloseFD{open(channelsList.c_str(), O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC, 0644)}; if (!channelsFD) - throw SysError(format("opening ‘%1%’ for writing") % channelsList); + throw SysError("opening '%1%' for writing", channelsList); for (const auto & channel : channels) writeFull(channelsFD.get(), channel.second + " " + channel.first + "\n"); } @@ -44,15 +47,15 @@ static void writeChannels() static void addChannel(const string & url, const string & name) { if (!regex_search(url, std::regex("^(file|http|https)://"))) - throw Error(format("invalid channel URL ‘%1%’") % url); + throw Error("invalid channel URL '%1%'", url); if (!regex_search(name, std::regex("^[a-zA-Z0-9_][a-zA-Z0-9_\\.-]*$"))) - throw Error(format("invalid channel identifier ‘%1%’") % name); + throw Error("invalid channel identifier '%1%'", name); readChannels(); channels[name] = url; writeChannels(); } -static auto profile = Path{}; +static Path profile; // Remove a channel. static void removeChannel(const string & name) @@ -64,7 +67,7 @@ static void removeChannel(const string & name) runProgram(settings.nixBinDir + "/nix-env", true, { "--profile", profile, "--uninstall", name }); } -static auto nixDefExpr = Path{}; +static Path nixDefExpr; // Fetch Nix expressions and binary cache URLs from the subscribed channels. static void update(const StringSet & channelNames) @@ -74,7 +77,7 @@ static void update(const StringSet & channelNames) auto store = openStore(); // Download each channel. - auto exprs = Strings{}; + Strings exprs; for (const auto & channel : channels) { auto name = channel.first; auto url = channel.second; @@ -84,53 +87,35 @@ static void update(const StringSet & channelNames) // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. - auto effectiveUrl = string{}; - auto dl = getDownloader(); - auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl); - url = chomp(std::move(effectiveUrl)); + auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)), false); + auto filename = store->toRealPath(result.storePath); + url = result.effectiveUrl; // If the URL contains a version number, append it to the name // attribute (so that "nix-env -q" on the channels profile // shows something useful). auto cname = name; std::smatch match; - auto urlBase = baseNameOf(url); - if (std::regex_search(urlBase, match, std::regex("(-\\d.*)$"))) { + auto urlBase = std::string(baseNameOf(url)); + if (std::regex_search(urlBase, match, std::regex("(-\\d.*)$"))) cname = cname + (string) match[1]; - } - auto extraAttrs = string{}; + std::string extraAttrs; - auto unpacked = false; + bool unpacked = false; if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) { - try { - runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import " - "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" }); - unpacked = true; - } catch (ExecError & e) { - } + runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import " + "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" }); + unpacked = true; } if (!unpacked) { - // The URL doesn't unpack directly, so let's try treating it like a full channel folder with files in it - // Check if the channel advertises a binary cache. - DownloadRequest request(url + "/binary-cache-url"); - request.showProgress = DownloadRequest::no; - try { - auto dlRes = dl->download(request); - extraAttrs = "binaryCacheURL = \"" + *dlRes.data + "\";"; - } catch (DownloadError & e) { - } - // Download the channel tarball. - auto fullURL = url + "/nixexprs.tar.xz"; try { - filename = dl->downloadCached(store, fullURL, false); - } catch (DownloadError & e) { - fullURL = url + "/nixexprs.tar.bz2"; - filename = dl->downloadCached(store, fullURL, false); + filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath); + } catch (FileTransferError & e) { + filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath); } - chomp(filename); } // Regardless of where it came from, add the expression representing this channel to accumulated expression @@ -140,7 +125,7 @@ static void update(const StringSet & channelNames) // Unpack the channel tarballs into the Nix store and install them // into the channels profile. std::cerr << "unpacking channels...\n"; - auto envArgs = Strings{ "--profile", profile, "--file", "", "--install", "--from-expression" }; + Strings envArgs{ "--profile", profile, "--file", "", "--install", "--from-expression" }; for (auto & expr : exprs) envArgs.push_back(std::move(expr)); envArgs.push_back("--quiet"); @@ -152,43 +137,25 @@ static void update(const StringSet & channelNames) if (S_ISLNK(st.st_mode)) // old-skool ~/.nix-defexpr if (unlink(nixDefExpr.c_str()) == -1) - throw SysError(format("unlinking %1%") % nixDefExpr); + throw SysError("unlinking %1%", nixDefExpr); } else if (errno != ENOENT) { - throw SysError(format("getting status of %1%") % nixDefExpr); + throw SysError("getting status of %1%", nixDefExpr); } createDirs(nixDefExpr); auto channelLink = nixDefExpr + "/channels"; replaceSymlink(profile, channelLink); } -int main(int argc, char ** argv) +static int _main(int argc, char ** argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - - // Turn on caching in nix-prefetch-url. - auto channelCache = settings.nixStateDir + "/channel-cache"; - createDirs(channelCache); - setenv("NIX_DOWNLOAD_CACHE", channelCache.c_str(), 1); - + { // Figure out the name of the `.nix-channels' file to use - auto home = getEnv("HOME"); - if (home.empty()) - throw Error("$HOME not set"); + auto home = getHome(); channelsList = home + "/.nix-channels"; nixDefExpr = home + "/.nix-defexpr"; // Figure out the name of the channels profile. - auto name = string{}; - auto pw = getpwuid(getuid()); - if (!pw) - name = getEnv("USER", ""); - else - name = pw->pw_name; - if (name.empty()) - throw Error("cannot figure out user name"); - profile = settings.nixStateDir + "/profiles/per-user/" + name + "/channels"; - createDirs(dirOf(profile)); + profile = fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()); enum { cNone, @@ -198,7 +165,7 @@ int main(int argc, char ** argv) cUpdate, cRollback } cmd = cNone; - auto args = std::vector{}; + std::vector args; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--help") { showManPage("nix-channel"); @@ -215,19 +182,24 @@ int main(int argc, char ** argv) } else if (*arg == "--rollback") { cmd = cRollback; } else { + if (hasPrefix(*arg, "-")) + throw UsageError("unsupported argument '%s'", *arg); args.push_back(std::move(*arg)); } return true; }); + + initPlugins(); + switch (cmd) { case cNone: throw UsageError("no command specified"); case cAdd: if (args.size() < 1 || args.size() > 2) - throw UsageError("‘--add’ requires one or two arguments"); + throw UsageError("'--add' requires one or two arguments"); { auto url = args[0]; - auto name = string{}; + std::string name; if (args.size() == 2) { name = args[1]; } else { @@ -240,12 +212,12 @@ int main(int argc, char ** argv) break; case cRemove: if (args.size() != 1) - throw UsageError("‘--remove’ requires one argument"); + throw UsageError("'--remove' requires one argument"); removeChannel(args[0]); break; case cList: if (!args.empty()) - throw UsageError("‘--list’ expects no arguments"); + throw UsageError("'--list' expects no arguments"); readChannels(); for (const auto & channel : channels) std::cout << channel.first << ' ' << channel.second << '\n'; @@ -255,8 +227,8 @@ int main(int argc, char ** argv) break; case cRollback: if (args.size() > 1) - throw UsageError("‘--rollback’ has at most one argument"); - auto envArgs = Strings{"--profile", profile}; + throw UsageError("'--rollback' has at most one argument"); + Strings envArgs{"--profile", profile}; if (args.size() == 1) { envArgs.push_back("--switch-generation"); envArgs.push_back(args[0]); @@ -266,5 +238,9 @@ int main(int argc, char ** argv) runProgram(settings.nixBinDir + "/nix-env", false, envArgs); break; } - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-channel", _main); diff --git a/src/nix-collect-garbage/local.mk b/src/nix-collect-garbage/local.mk deleted file mode 100644 index 02d14cf6219..00000000000 --- a/src/nix-collect-garbage/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-collect-garbage - -nix-collect-garbage_DIR := $(d) - -nix-collect-garbage_SOURCES := $(d)/nix-collect-garbage.cc - -nix-collect-garbage_LIBS = libmain libstore libutil libformat diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index cc663a96924..aa5ada3a62b 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -2,6 +2,7 @@ #include "profiles.hh" #include "shared.hh" #include "globals.hh" +#include "../nix/legacy.hh" #include #include @@ -48,12 +49,10 @@ void removeOldGenerations(std::string dir) } } -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - bool removeOld = false; - - return handleExceptions(argv[0], [&]() { - initNix(); + { + bool removeOld = false; GCOptions options; @@ -77,6 +76,8 @@ int main(int argc, char * * argv) return true; }); + initPlugins(); + auto profilesDir = settings.nixStateDir + "/profiles"; if (removeOld) removeOldGenerations(profilesDir); @@ -88,5 +89,9 @@ int main(int argc, char * * argv) PrintFreed freed(true, results); store->collectGarbage(options, results); } - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-collect-garbage", _main); diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc new file mode 100755 index 00000000000..b10184718d1 --- /dev/null +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -0,0 +1,68 @@ +#include "shared.hh" +#include "store-api.hh" +#include "../nix/legacy.hh" + +using namespace nix; + +static int _main(int argc, char ** argv) +{ + { + auto gzip = false; + auto toMode = true; + auto includeOutputs = false; + auto dryRun = false; + auto useSubstitutes = NoSubstitute; + std::string sshHost; + PathSet storePaths; + + parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") + showManPage("nix-copy-closure"); + else if (*arg == "--version") + printVersion("nix-copy-closure"); + else if (*arg == "--gzip" || *arg == "--bzip2" || *arg == "--xz") { + if (*arg != "--gzip") + printMsg(lvlError, format("Warning: '%1%' is not implemented, falling back to gzip") % *arg); + gzip = true; + } else if (*arg == "--from") + toMode = false; + else if (*arg == "--to") + toMode = true; + else if (*arg == "--include-outputs") + includeOutputs = true; + else if (*arg == "--show-progress") + printMsg(lvlError, "Warning: '--show-progress' is not implemented"); + else if (*arg == "--dry-run") + dryRun = true; + else if (*arg == "--use-substitutes" || *arg == "-s") + useSubstitutes = Substitute; + else if (sshHost.empty()) + sshHost = *arg; + else + storePaths.insert(*arg); + return true; + }); + + initPlugins(); + + if (sshHost.empty()) + throw UsageError("no host name specified"); + + auto remoteUri = "ssh://" + sshHost + (gzip ? "?compress=true" : ""); + auto to = toMode ? openStore(remoteUri) : openStore(); + auto from = toMode ? openStore() : openStore(remoteUri); + + StorePathSet storePaths2; + for (auto & path : storePaths) + storePaths2.insert(from->followLinksToStorePath(path)); + + StorePathSet closure; + from->computeFSClosure(storePaths2, closure, false, includeOutputs); + + copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes); + + return 0; + } +} + +static RegisterLegacyCommand s1("nix-copy-closure", _main); diff --git a/src/nix-daemon/local.mk b/src/nix-daemon/local.mk deleted file mode 100644 index 5a4474465b3..00000000000 --- a/src/nix-daemon/local.mk +++ /dev/null @@ -1,13 +0,0 @@ -programs += nix-daemon - -nix-daemon_DIR := $(d) - -nix-daemon_SOURCES := $(d)/nix-daemon.cc - -nix-daemon_LIBS = libmain libstore libutil libformat - -nix-daemon_LDFLAGS = -pthread - -ifeq ($(OS), SunOS) - nix-daemon_LDFLAGS += -lsocket -endif diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 90a7301873c..bcb86cbce65 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -2,16 +2,17 @@ #include "local-store.hh" #include "util.hh" #include "serialise.hh" -#include "worker-protocol.hh" #include "archive.hh" -#include "affinity.hh" #include "globals.hh" -#include "monitor-fd.hh" #include "derivations.hh" +#include "finally.hh" +#include "../nix/legacy.hh" +#include "daemon.hh" #include - +#include #include + #include #include #include @@ -29,19 +30,20 @@ #endif using namespace nix; +using namespace nix::daemon; #ifndef __linux__ #define SPLICE_F_MOVE 0 static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t len, unsigned int flags) { - /* We ignore most parameters, we just have them for conformance with the linux syscall */ - char buf[8192]; - auto read_count = read(fd_in, buf, sizeof(buf)); + // We ignore most parameters, we just have them for conformance with the linux syscall + std::vector buf(8192); + auto read_count = read(fd_in, buf.data(), buf.size()); if (read_count == -1) return read_count; auto write_count = decltype(read_count)(0); while (write_count < read_count) { - auto res = write(fd_out, buf + write_count, read_count - write_count); + auto res = write(fd_out, buf.data() + write_count, read_count - write_count); if (res == -1) return res; write_count += res; @@ -50,665 +52,14 @@ static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t } #endif -static FdSource from(STDIN_FILENO); -static FdSink to(STDOUT_FILENO); - -static bool canSendStderr; - -static Logger * defaultLogger; - - -/* Logger that forwards log messages to the client, *if* we're in a - state where the protocol allows it (i.e., when canSendStderr is - true). */ -class TunnelLogger : public Logger -{ - void log(Verbosity lvl, const FormatOrString & fs) override - { - if (lvl > verbosity) return; - - if (canSendStderr) { - try { - to << STDERR_NEXT << (fs.s + "\n"); - to.flush(); - } catch (...) { - /* Write failed; that means that the other side is - gone. */ - canSendStderr = false; - throw; - } - } else - defaultLogger->log(lvl, fs); - } - - void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override - { - log(lvl, fs); - } - - void stopActivity(Activity & activity) override - { - } -}; - - -/* startWork() means that we're starting an operation for which we - want to send out stderr to the client. */ -static void startWork() -{ - canSendStderr = true; -} - - -/* stopWork() means that we're done; stop sending stderr to the - client. */ -static void stopWork(bool success = true, const string & msg = "", unsigned int status = 0) -{ - canSendStderr = false; - - if (success) - to << STDERR_LAST; - else { - to << STDERR_ERROR << msg; - if (status != 0) to << status; - } -} - - -struct TunnelSink : Sink -{ - Sink & to; - TunnelSink(Sink & to) : to(to) { } - virtual void operator () (const unsigned char * data, size_t len) - { - to << STDERR_WRITE; - writeString(data, len, to); - } -}; - - -struct TunnelSource : BufferedSource -{ - Source & from; - TunnelSource(Source & from) : from(from) { } - size_t readUnbuffered(unsigned char * data, size_t len) - { - to << STDERR_READ << len; - to.flush(); - size_t n = readString(data, len, from); - if (n == 0) throw EndOfFile("unexpected end-of-file"); - return n; - } -}; - - -/* If the NAR archive contains a single file at top-level, then save - the contents of the file to `s'. Otherwise barf. */ -struct RetrieveRegularNARSink : ParseSink -{ - bool regular; - string s; - - RetrieveRegularNARSink() : regular(true) { } - - void createDirectory(const Path & path) - { - regular = false; - } - - void receiveContents(unsigned char * data, unsigned int len) - { - s.append((const char *) data, len); - } - - void createSymlink(const Path & path, const string & target) - { - regular = false; - } -}; - - -/* Adapter class of a Source that saves all data read to `s'. */ -struct SavingSourceAdapter : Source -{ - Source & orig; - string s; - SavingSourceAdapter(Source & orig) : orig(orig) { } - size_t read(unsigned char * data, size_t len) - { - size_t n = orig.read(data, len); - s.append((const char *) data, n); - return n; - } -}; - - -static void performOp(ref store, bool trusted, unsigned int clientVersion, - Source & from, Sink & to, unsigned int op) -{ - switch (op) { - - case wopIsValidPath: { - /* 'readStorePath' could raise an error leading to the connection - being closed. To be able to recover from an invalid path error, - call 'startWork' early, and do 'assertStorePath' afterwards so - that the 'Error' exception handler doesn't close the - connection. */ - Path path = readString(from); - startWork(); - store->assertStorePath(path); - bool result = store->isValidPath(path); - stopWork(); - to << result; - break; - } - - case wopQueryValidPaths: { - PathSet paths = readStorePaths(*store, from); - startWork(); - PathSet res = store->queryValidPaths(paths); - stopWork(); - to << res; - break; - } - - case wopHasSubstitutes: { - Path path = readStorePath(*store, from); - startWork(); - PathSet res = store->querySubstitutablePaths({path}); - stopWork(); - to << (res.find(path) != res.end()); - break; - } - - case wopQuerySubstitutablePaths: { - PathSet paths = readStorePaths(*store, from); - startWork(); - PathSet res = store->querySubstitutablePaths(paths); - stopWork(); - to << res; - break; - } - - case wopQueryPathHash: { - Path path = readStorePath(*store, from); - startWork(); - auto hash = store->queryPathInfo(path)->narHash; - stopWork(); - to << printHash(hash); - break; - } - - case wopQueryReferences: - case wopQueryReferrers: - case wopQueryValidDerivers: - case wopQueryDerivationOutputs: { - Path path = readStorePath(*store, from); - startWork(); - PathSet paths; - if (op == wopQueryReferences) - paths = store->queryPathInfo(path)->references; - else if (op == wopQueryReferrers) - store->queryReferrers(path, paths); - else if (op == wopQueryValidDerivers) - paths = store->queryValidDerivers(path); - else paths = store->queryDerivationOutputs(path); - stopWork(); - to << paths; - break; - } - - case wopQueryDerivationOutputNames: { - Path path = readStorePath(*store, from); - startWork(); - StringSet names; - names = store->queryDerivationOutputNames(path); - stopWork(); - to << names; - break; - } - - case wopQueryDeriver: { - Path path = readStorePath(*store, from); - startWork(); - auto deriver = store->queryPathInfo(path)->deriver; - stopWork(); - to << deriver; - break; - } - - case wopQueryPathFromHashPart: { - string hashPart = readString(from); - startWork(); - Path path = store->queryPathFromHashPart(hashPart); - stopWork(); - to << path; - break; - } - - case wopAddToStore: { - string baseName = readString(from); - bool fixed = readInt(from) == 1; /* obsolete */ - bool recursive = readInt(from) == 1; - string s = readString(from); - /* Compatibility hack. */ - if (!fixed) { - s = "sha256"; - recursive = true; - } - HashType hashAlgo = parseHashType(s); - - SavingSourceAdapter savedNAR(from); - RetrieveRegularNARSink savedRegular; - - if (recursive) { - /* Get the entire NAR dump from the client and save it to - a string so that we can pass it to - addToStoreFromDump(). */ - ParseSink sink; /* null sink; just parse the NAR */ - parseDump(sink, savedNAR); - } else - parseDump(savedRegular, from); - - startWork(); - if (!savedRegular.regular) throw Error("regular file expected"); - Path path = store->addToStoreFromDump(recursive ? savedNAR.s : savedRegular.s, baseName, recursive, hashAlgo); - stopWork(); - - to << path; - break; - } - - case wopAddTextToStore: { - string suffix = readString(from); - string s = readString(from); - PathSet refs = readStorePaths(*store, from); - startWork(); - Path path = store->addTextToStore(suffix, s, refs, false); - stopWork(); - to << path; - break; - } - - case wopExportPath: { - Path path = readStorePath(*store, from); - readInt(from); // obsolete - startWork(); - TunnelSink sink(to); - store->exportPath(path, sink); - stopWork(); - to << 1; - break; - } - - case wopImportPaths: { - startWork(); - TunnelSource source(from); - Paths paths = store->importPaths(source, 0, trusted); - stopWork(); - to << paths; - break; - } - - case wopBuildPaths: { - PathSet drvs = readStorePaths(*store, from); - BuildMode mode = bmNormal; - if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { - mode = (BuildMode)readInt(from); - - /* Repairing is not atomic, so disallowed for "untrusted" - clients. */ - if (mode == bmRepair && !trusted) - throw Error("repairing is not supported when building through the Nix daemon"); - } - startWork(); - store->buildPaths(drvs, mode); - stopWork(); - to << 1; - break; - } - - case wopBuildDerivation: { - Path drvPath = readStorePath(*store, from); - BasicDerivation drv; - readDerivation(from, *store, drv); - BuildMode buildMode = (BuildMode) readInt(from); - startWork(); - if (!trusted) - throw Error("you are not privileged to build derivations"); - auto res = store->buildDerivation(drvPath, drv, buildMode); - stopWork(); - to << res.status << res.errorMsg; - break; - } - - case wopEnsurePath: { - Path path = readStorePath(*store, from); - startWork(); - store->ensurePath(path); - stopWork(); - to << 1; - break; - } - - case wopAddTempRoot: { - Path path = readStorePath(*store, from); - startWork(); - store->addTempRoot(path); - stopWork(); - to << 1; - break; - } - - case wopAddIndirectRoot: { - Path path = absPath(readString(from)); - startWork(); - store->addIndirectRoot(path); - stopWork(); - to << 1; - break; - } - - case wopSyncWithGC: { - startWork(); - store->syncWithGC(); - stopWork(); - to << 1; - break; - } - - case wopFindRoots: { - startWork(); - Roots roots = store->findRoots(); - stopWork(); - to << roots.size(); - for (auto & i : roots) - to << i.first << i.second; - break; - } - - case wopCollectGarbage: { - GCOptions options; - options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = readStorePaths(*store, from); - options.ignoreLiveness = readInt(from); - options.maxFreed = readLongLong(from); - // obsolete fields - readInt(from); - readInt(from); - readInt(from); - - GCResults results; - - startWork(); - if (options.ignoreLiveness) - throw Error("you are not allowed to ignore liveness"); - store->collectGarbage(options, results); - stopWork(); - - to << results.paths << results.bytesFreed << 0 /* obsolete */; - - break; - } - - case wopSetOptions: { - settings.keepFailed = readInt(from) != 0; - settings.keepGoing = readInt(from) != 0; - settings.set("build-fallback", readInt(from) ? "true" : "false"); - verbosity = (Verbosity) readInt(from); - settings.set("build-max-jobs", std::to_string(readInt(from))); - settings.set("build-max-silent-time", std::to_string(readInt(from))); - settings.useBuildHook = readInt(from) != 0; - settings.verboseBuild = lvlError == (Verbosity) readInt(from); - readInt(from); // obsolete logType - readInt(from); // obsolete printBuildTrace - settings.set("build-cores", std::to_string(readInt(from))); - settings.set("build-use-substitutes", readInt(from) ? "true" : "false"); - if (GET_PROTOCOL_MINOR(clientVersion) >= 12) { - unsigned int n = readInt(from); - for (unsigned int i = 0; i < n; i++) { - string name = readString(from); - string value = readString(from); - if (name == "build-timeout" || name == "use-ssh-substituter") - settings.set(name, value); - else - settings.set(trusted ? name : "untrusted-" + name, value); - } - } - settings.update(); - startWork(); - stopWork(); - break; - } - - case wopQuerySubstitutablePathInfo: { - Path path = absPath(readString(from)); - startWork(); - SubstitutablePathInfos infos; - store->querySubstitutablePathInfos({path}, infos); - stopWork(); - SubstitutablePathInfos::iterator i = infos.find(path); - if (i == infos.end()) - to << 0; - else { - to << 1 << i->second.deriver << i->second.references << i->second.downloadSize << i->second.narSize; - } - break; - } - - case wopQuerySubstitutablePathInfos: { - PathSet paths = readStorePaths(*store, from); - startWork(); - SubstitutablePathInfos infos; - store->querySubstitutablePathInfos(paths, infos); - stopWork(); - to << infos.size(); - for (auto & i : infos) { - to << i.first << i.second.deriver << i.second.references - << i.second.downloadSize << i.second.narSize; - } - break; - } - - case wopQueryAllValidPaths: { - startWork(); - PathSet paths = store->queryAllValidPaths(); - stopWork(); - to << paths; - break; - } - - case wopQueryPathInfo: { - Path path = readStorePath(*store, from); - std::shared_ptr info; - startWork(); - try { - info = store->queryPathInfo(path); - } catch (InvalidPath &) { - if (GET_PROTOCOL_MINOR(clientVersion) < 17) throw; - } - stopWork(); - if (info) { - if (GET_PROTOCOL_MINOR(clientVersion) >= 17) - to << 1; - to << info->deriver << printHash(info->narHash) << info->references - << info->registrationTime << info->narSize; - if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { - to << info->ultimate - << info->sigs - << info->ca; - } - } else { - assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); - to << 0; - } - break; - } - - case wopOptimiseStore: - startWork(); - store->optimiseStore(); - stopWork(); - to << 1; - break; - - case wopVerifyStore: { - bool checkContents = readInt(from) != 0; - bool repair = readInt(from) != 0; - startWork(); - if (repair && !trusted) - throw Error("you are not privileged to repair paths"); - bool errors = store->verifyStore(checkContents, repair); - stopWork(); - to << errors; - break; - } - - case wopAddSignatures: { - Path path = readStorePath(*store, from); - StringSet sigs = readStrings(from); - startWork(); - if (!trusted) - throw Error("you are not privileged to add signatures"); - store->addSignatures(path, sigs); - stopWork(); - to << 1; - break; - } - - case wopNarFromPath: { - auto path = readStorePath(*store, from); - startWork(); - stopWork(); - dumpPath(path, to); - break; - } - - case wopAddToStoreNar: { - ValidPathInfo info; - info.path = readStorePath(*store, from); - info.deriver = readString(from); - if (!info.deriver.empty()) - store->assertStorePath(info.deriver); - info.narHash = parseHash(htSHA256, readString(from)); - info.references = readStorePaths(*store, from); - info.registrationTime = readInt(from); - info.narSize = readLongLong(from); - info.ultimate = readLongLong(from); - info.sigs = readStrings(from); - auto nar = make_ref(readString(from)); - auto repair = readInt(from) ? true : false; - auto dontCheckSigs = readInt(from) ? true : false; - if (!trusted && dontCheckSigs) - dontCheckSigs = false; - startWork(); - store->addToStore(info, nar, repair, dontCheckSigs, nullptr); - stopWork(); - break; - } - - default: - throw Error(format("invalid operation %1%") % op); - } -} - - -static void processConnection(bool trusted) -{ - MonitorFdHup monitor(from.fd); - - canSendStderr = false; - defaultLogger = logger; - logger = new TunnelLogger(); - - /* Exchange the greeting. */ - unsigned int magic = readInt(from); - if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); - to << WORKER_MAGIC_2 << PROTOCOL_VERSION; - to.flush(); - unsigned int clientVersion = readInt(from); - - if (clientVersion < 0x10a) - throw Error("the Nix client version is too old"); - - if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) - setAffinityTo(readInt(from)); - - readInt(from); // obsolete reserveSpace - - /* Send startup error messages to the client. */ - startWork(); - - try { - - /* If we can't accept clientVersion, then throw an error - *here* (not above). */ - -#if 0 - /* Prevent users from doing something very dangerous. */ - if (geteuid() == 0 && - querySetting("build-users-group", "") == "") - throw Error("if you run ‘nix-daemon’ as root, then you MUST set ‘build-users-group’!"); -#endif - - /* Open the store. */ - auto store = make_ref(Store::Params()); // FIXME: get params from somewhere - - stopWork(); - to.flush(); - - /* Process client requests. */ - unsigned int opCount = 0; - - while (true) { - WorkerOp op; - try { - op = (WorkerOp) readInt(from); - } catch (Interrupted & e) { - break; - } catch (EndOfFile & e) { - break; - } - - opCount++; - - try { - performOp(store, trusted, clientVersion, from, to, op); - } catch (Error & e) { - /* If we're not in a state where we can send replies, then - something went wrong processing the input of the - client. This can happen especially if I/O errors occur - during addTextToStore() / importPath(). If that - happens, just send the error message and exit. */ - bool errorAllowed = canSendStderr; - stopWork(false, e.msg(), e.status); - if (!errorAllowed) throw; - } catch (std::bad_alloc & e) { - stopWork(false, "Nix daemon out of memory", 1); - throw; - } - - to.flush(); - - assert(!canSendStderr); - }; - - canSendStderr = false; - _isInterrupted = false; - debug(format("%1% operations") % opCount); - - } catch (Error & e) { - stopWork(false, e.msg(), 1); - to.flush(); - return; - } -} - static void sigChldHandler(int sigNo) { - /* Reap all dead children. */ + // Ensure we don't modify errno of whatever we've interrupted + auto saved_errno = errno; + // Reap all dead children. while (waitpid(-1, 0, WNOHANG) > 0) ; + errno = saved_errno; } @@ -755,7 +106,7 @@ struct PeerInfo }; -/* Get the identity of the caller, if possible. */ +// Get the identity of the caller, if possible. static PeerInfo getPeerInfo(int remote) { PeerInfo peer = { false, 0, false, 0, false, 0 }; @@ -789,74 +140,45 @@ static PeerInfo getPeerInfo(int remote) #define SD_LISTEN_FDS_START 3 +static ref openUncachedStore() +{ + Store::Params params; // FIXME: get params from somewhere + // Disable caching since the client already does that. + params["path-info-cache-size"] = "0"; + return openStore(settings.storeUri, params); +} + + static void daemonLoop(char * * argv) { if (chdir("/") == -1) throw SysError("cannot change current directory"); - /* Get rid of children automatically; don't let them become - zombies. */ + // Get rid of children automatically; don't let them become zombies. setSigChldAction(true); AutoCloseFD fdSocket; - /* Handle socket-based activation by systemd. */ - if (getEnv("LISTEN_FDS") != "") { - if (getEnv("LISTEN_PID") != std::to_string(getpid()) || getEnv("LISTEN_FDS") != "1") + // Handle socket-based activation by systemd. + auto listenFds = getEnv("LISTEN_FDS"); + if (listenFds) { + if (getEnv("LISTEN_PID") != std::to_string(getpid()) || listenFds != "1") throw Error("unexpected systemd environment variables"); fdSocket = SD_LISTEN_FDS_START; + closeOnExec(fdSocket.get()); } - /* Otherwise, create and bind to a Unix domain socket. */ + // Otherwise, create and bind to a Unix domain socket. else { - - /* Create and bind to a Unix domain socket. */ - fdSocket = socket(PF_UNIX, SOCK_STREAM, 0); - if (!fdSocket) - throw SysError("cannot create Unix domain socket"); - - string socketPath = settings.nixDaemonSocketFile; - - createDirs(dirOf(socketPath)); - - /* Urgh, sockaddr_un allows path names of only 108 characters. - So chdir to the socket directory so that we can pass a - relative path name. */ - if (chdir(dirOf(socketPath).c_str()) == -1) - throw SysError("cannot change current directory"); - Path socketPathRel = "./" + baseNameOf(socketPath); - - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - if (socketPathRel.size() >= sizeof(addr.sun_path)) - throw Error(format("socket path ‘%1%’ is too long") % socketPathRel); - strcpy(addr.sun_path, socketPathRel.c_str()); - - unlink(socketPath.c_str()); - - /* Make sure that the socket is created with 0666 permission - (everybody can connect --- provided they have access to the - directory containing the socket). */ - mode_t oldMode = umask(0111); - int res = bind(fdSocket.get(), (struct sockaddr *) &addr, sizeof(addr)); - umask(oldMode); - if (res == -1) - throw SysError(format("cannot bind to socket ‘%1%’") % socketPath); - - if (chdir("/") == -1) /* back to the root */ - throw SysError("cannot change current directory"); - - if (listen(fdSocket.get(), 5) == -1) - throw SysError(format("cannot listen on socket ‘%1%’") % socketPath); + createDirs(dirOf(settings.nixDaemonSocketFile)); + fdSocket = createUnixDomainSocket(settings.nixDaemonSocketFile, 0666); } - closeOnExec(fdSocket.get()); - - /* Loop accepting connections. */ + // Loop accepting connections. while (1) { try { - /* Accept a connection. */ + // Accept a connection. struct sockaddr_un remoteAddr; socklen_t remoteAddrLen = sizeof(remoteAddr); @@ -870,7 +192,7 @@ static void daemonLoop(char * * argv) closeOnExec(remote.get()); - bool trusted = false; + TrustedFlag trusted = NotTrusted; PeerInfo peer = getPeerInfo(remote.get()); struct passwd * pw = peer.uidKnown ? getpwuid(peer.uid) : 0; @@ -879,20 +201,20 @@ static void daemonLoop(char * * argv) struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; string group = gr ? gr->gr_name : std::to_string(peer.gid); - Strings trustedUsers = settings.get("trusted-users", Strings({"root"})); - Strings allowedUsers = settings.get("allowed-users", Strings({"*"})); + Strings trustedUsers = settings.trustedUsers; + Strings allowedUsers = settings.allowedUsers; if (matchUser(user, group, trustedUsers)) - trusted = true; + trusted = Trusted; - if (!trusted && !matchUser(user, group, allowedUsers)) - throw Error(format("user ‘%1%’ is not allowed to connect to the Nix daemon") % user); + if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) + throw Error("user '%1%' is not allowed to connect to the Nix daemon", user); printInfo(format((string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : "")) % (peer.pidKnown ? std::to_string(peer.pid) : "") % (peer.uidKnown ? user : "")); - /* Fork a child to handle the connection. */ + // Fork a child to handle the connection. ProcessOptions options; options.errorPrefix = "unexpected Nix daemon error: "; options.dieWithParent = false; @@ -901,46 +223,47 @@ static void daemonLoop(char * * argv) startProcess([&]() { fdSocket = -1; - /* Background the daemon. */ + // Background the daemon. if (setsid() == -1) - throw SysError(format("creating a new session")); + throw SysError("creating a new session"); - /* Restore normal handling of SIGCHLD. */ + // Restore normal handling of SIGCHLD. setSigChldAction(false); - /* For debugging, stuff the pid into argv[1]. */ + // For debugging, stuff the pid into argv[1]. if (peer.pidKnown && argv[1]) { string processName = std::to_string(peer.pid); strncpy(argv[1], processName.c_str(), strlen(argv[1])); } - /* Handle the connection. */ - from.fd = remote.get(); - to.fd = remote.get(); - processConnection(trusted); + // Handle the connection. + FdSource from(remote.get()); + FdSink to(remote.get()); + processConnection(openUncachedStore(), from, to, trusted, NotRecursive, user, peer.uid); exit(0); }, options); } catch (Interrupted & e) { - throw; - } catch (Error & e) { - printError(format("error processing connection: %1%") % e.msg()); + return; + } catch (Error & error) { + ErrorInfo ei = error.info(); + ei.hint = std::optional(hintfmt("error processing connection: %1%", + (error.info().hint.has_value() ? error.info().hint->str() : ""))); + logError(ei); } } } -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - + { auto stdio = false; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--daemon") - ; /* ignored for backwards compatibility */ + ; // ignored for backwards compatibility else if (*arg == "--help") showManPage("nix-daemon"); else if (*arg == "--version") @@ -951,9 +274,11 @@ int main(int argc, char * * argv) return true; }); + initPlugins(); + if (stdio) { if (getStoreType() == tDaemon) { - /* Forward on this connection to the real daemon */ + // Forward on this connection to the real daemon auto socketPath = settings.nixDaemonSocketFile; auto s = socket(PF_UNIX, SOCK_STREAM, 0); if (s == -1) @@ -961,17 +286,17 @@ int main(int argc, char * * argv) auto socketDir = dirOf(socketPath); if (chdir(socketDir.c_str()) == -1) - throw SysError(format("changing to socket directory ‘%1%’") % socketDir); + throw SysError("changing to socket directory '%1%'", socketDir); - auto socketName = baseNameOf(socketPath); + auto socketName = std::string(baseNameOf(socketPath)); auto addr = sockaddr_un{}; addr.sun_family = AF_UNIX; if (socketName.size() + 1 >= sizeof(addr.sun_path)) - throw Error(format("socket name %1% is too long") % socketName); + throw Error("socket name %1% is too long", socketName); strcpy(addr.sun_path, socketName.c_str()); if (connect(s, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError(format("cannot connect to daemon at %1%") % socketPath); + throw SysError("cannot connect to daemon at %1%", socketPath); auto nfds = (s > STDIN_FILENO ? s : STDIN_FILENO) + 1; while (true) { @@ -982,25 +307,31 @@ int main(int argc, char * * argv) if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) throw SysError("waiting for data from client or server"); if (FD_ISSET(s, &fds)) { - auto res = splice(s, nullptr, STDOUT_FILENO, nullptr, SIZE_MAX, SPLICE_F_MOVE); + auto res = splice(s, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); if (res == -1) throw SysError("splicing data from daemon socket to stdout"); else if (res == 0) throw EndOfFile("unexpected EOF from daemon socket"); } if (FD_ISSET(STDIN_FILENO, &fds)) { - auto res = splice(STDIN_FILENO, nullptr, s, nullptr, SIZE_MAX, SPLICE_F_MOVE); + auto res = splice(STDIN_FILENO, nullptr, s, nullptr, SSIZE_MAX, SPLICE_F_MOVE); if (res == -1) throw SysError("splicing data from stdin to daemon socket"); else if (res == 0) - return; + return 0; } } } else { - processConnection(true); + FdSource from(STDIN_FILENO); + FdSink to(STDOUT_FILENO); + processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, "root", 0); } } else { daemonLoop(argv); } - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-daemon", _main); diff --git a/src/nix-env/buildenv.nix b/src/nix-env/buildenv.nix new file mode 100644 index 00000000000..0bac4c44b48 --- /dev/null +++ b/src/nix-env/buildenv.nix @@ -0,0 +1,25 @@ +{ derivations, manifest }: + +derivation { + name = "user-environment"; + system = "builtin"; + builder = "builtin:buildenv"; + + inherit manifest; + + # !!! grmbl, need structured data for passing this in a clean way. + derivations = + map (d: + [ (d.meta.active or "true") + (d.meta.priority or 5) + (builtins.length d.outputs) + ] ++ map (output: builtins.getAttr output d) d.outputs) + derivations; + + # Building user environments remotely just causes huge amounts of + # network traffic, so don't do that. + preferLocalBuild = true; + + # Also don't bother substituting. + allowSubstitutes = false; +} diff --git a/src/nix-env/local.mk b/src/nix-env/local.mk deleted file mode 100644 index e80719cd76f..00000000000 --- a/src/nix-env/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-env - -nix-env_DIR := $(d) - -nix-env_SOURCES := $(wildcard $(d)/*.cc) - -nix-env_LIBS = libexpr libmain libstore libutil libformat diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 908c09bc8c8..2c27d97f5bb 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1,5 +1,5 @@ #include "attr-path.hh" -#include "common-opts.hh" +#include "common-eval-args.hh" #include "derivations.hh" #include "eval.hh" #include "get-drvs.hh" @@ -13,6 +13,7 @@ #include "json.hh" #include "value-to-json.hh" #include "xml-writer.hh" +#include "../nix/legacy.hh" #include #include @@ -24,7 +25,6 @@ #include #include - using namespace nix; using std::cout; @@ -69,8 +69,7 @@ typedef void (* Operation) (Globals & globals, static string needArg(Strings::iterator & i, Strings & args, const string & arg) { - if (i == args.end()) throw UsageError( - format("‘%1%’ requires an argument") % arg); + if (i == args.end()) throw UsageError("'%1%' requires an argument", arg); return *i++; } @@ -123,17 +122,19 @@ static void getAllExprs(EvalState & state, string attrName = i; if (hasSuffix(attrName, ".nix")) attrName = string(attrName, 0, attrName.size() - 4); - if (attrs.find(attrName) != attrs.end()) { - printError(format("warning: name collision in input Nix expressions, skipping ‘%1%’") % path2); + if (!attrs.insert(attrName).second) { + logError({ + .name = "Name collision", + .hint = hintfmt("warning: name collision in input Nix expressions, skipping '%1%'", path2) + }); continue; } - attrs.insert(attrName); /* Load the expression on demand. */ Value & vFun = state.getBuiltin("import"); Value & vArg(*state.allocValue()); mkString(vArg, path2); if (v.attrs->size() == v.attrs->capacity()) - throw Error(format("too many Nix expressions in directory ‘%1%’") % path); + throw Error("too many Nix expressions in directory '%1%'", path); mkApp(*state.allocAttr(v, state.symbols.create(attrName)), vFun, vArg); } else if (S_ISDIR(st.st_mode)) @@ -144,16 +145,15 @@ static void getAllExprs(EvalState & state, } + static void loadSourceExpr(EvalState & state, const Path & path, Value & v) { struct stat st; if (stat(path.c_str(), &st) == -1) - throw SysError(format("getting information about ‘%1%’") % path); + throw SysError("getting information about '%1%'", path); - if (isNixExpr(path, st)) { + if (isNixExpr(path, st)) state.evalFile(path, v); - return; - } /* The path is a directory. Put the Nix expressions in the directory in a set, with the file name of each expression as @@ -161,13 +161,15 @@ static void loadSourceExpr(EvalState & state, const Path & path, Value & v) set flat, not nested, to make it easier for a user to have a ~/.nix-defexpr directory that includes some system-wide directory). */ - if (S_ISDIR(st.st_mode)) { + else if (S_ISDIR(st.st_mode)) { state.mkAttrs(v, 1024); state.mkList(*state.allocAttr(v, state.symbols.create("_combineChannels")), 0); StringSet attrs; getAllExprs(state, path, attrs, v); v.attrs->sort(); } + + else throw Error("path '%s' is not a directory or a Nix expression", path); } @@ -178,7 +180,7 @@ static void loadDerivations(EvalState & state, Path nixExprPath, Value vRoot; loadSourceExpr(state, nixExprPath, vRoot); - Value & v(*findAlongAttrPath(state, pathPrefix, autoArgs, vRoot)); + Value & v(*findAlongAttrPath(state, pathPrefix, autoArgs, vRoot).first); getDerivations(state, v, pathPrefix, autoArgs, elems, true); @@ -186,33 +188,19 @@ static void loadDerivations(EvalState & state, Path nixExprPath, system. */ for (DrvInfos::iterator i = elems.begin(), j; i != elems.end(); i = j) { j = i; j++; - if (systemFilter != "*" && i->system != systemFilter) + if (systemFilter != "*" && i->querySystem() != systemFilter) elems.erase(i); } } -static Path getHomeDir() -{ - Path homeDir(getEnv("HOME", "")); - if (homeDir == "") throw Error("HOME environment variable not set"); - return homeDir; -} - - -static Path getDefNixExprPath() -{ - return getHomeDir() + "/.nix-defexpr"; -} - - -static int getPriority(EvalState & state, DrvInfo & drv) +static long getPriority(EvalState & state, DrvInfo & drv) { return drv.queryMetaInt("priority", 0); } -static int comparePriorities(EvalState & state, DrvInfo & drv1, DrvInfo & drv2) +static long comparePriorities(EvalState & state, DrvInfo & drv1, DrvInfo & drv2) { return getPriority(state, drv2) - getPriority(state, drv1); } @@ -222,10 +210,11 @@ static int comparePriorities(EvalState & state, DrvInfo & drv1, DrvInfo & drv2) // at a time. static bool isPrebuilt(EvalState & state, DrvInfo & elem) { - Path path = elem.queryOutPath(); + auto path = state.store->parseStorePath(elem.queryOutPath()); if (state.store->isValidPath(path)) return true; - PathSet ps = state.store->querySubstitutablePaths({path}); - return ps.find(path) != ps.end(); + StorePathSet paths; + paths.insert(path.clone()); // FIXME: why doesn't StorePathSet{path.clone()} work? + return state.store->querySubstitutablePaths(paths).count(path); } @@ -234,7 +223,7 @@ static void checkSelectorUse(DrvNames & selectors) /* Check that all selectors have been used. */ for (auto & i : selectors) if (i.hits == 0 && i.fullName != "*") - throw Error(format("selector ‘%1%’ matches no derivations") % i.fullName); + throw Error("selector '%1%' matches no derivations", i.fullName); } @@ -255,7 +244,7 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, for (DrvInfos::const_iterator j = allElems.begin(); j != allElems.end(); ++j, ++n) { - DrvName drvName(j->name); + DrvName drvName(j->queryName()); if (i.matches(drvName)) { i.hits++; matches.push_back(std::pair(*j, n)); @@ -277,36 +266,36 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, StringSet multiple; for (auto & j : matches) { - DrvName drvName(j.first.name); - int d = 1; + DrvName drvName(j.first.queryName()); + long d = 1; Newest::iterator k = newest.find(drvName.name); if (k != newest.end()) { - d = j.first.system == k->second.first.system ? 0 : - j.first.system == settings.thisSystem ? 1 : - k->second.first.system == settings.thisSystem ? -1 : 0; + d = j.first.querySystem() == k->second.first.querySystem() ? 0 : + j.first.querySystem() == settings.thisSystem ? 1 : + k->second.first.querySystem() == settings.thisSystem ? -1 : 0; if (d == 0) d = comparePriorities(state, j.first, k->second.first); if (d == 0) - d = compareVersions(drvName.version, DrvName(k->second.first.name).version); + d = compareVersions(drvName.version, DrvName(k->second.first.queryName()).version); } if (d > 0) { newest.erase(drvName.name); newest.insert(Newest::value_type(drvName.name, j)); - multiple.erase(j.first.name); + multiple.erase(j.first.queryName()); } else if (d == 0) { - multiple.insert(j.first.name); + multiple.insert(j.first.queryName()); } } matches.clear(); for (auto & j : newest) { - if (multiple.find(j.second.first.name) != multiple.end()) + if (multiple.find(j.second.first.queryName()) != multiple.end()) printInfo( - format("warning: there are multiple derivations named ‘%1%’; using the first one") - % j.second.first.name); + "warning: there are multiple derivations named '%1%'; using the first one", + j.second.first.queryName()); matches.push_back(j.second); } } @@ -314,10 +303,8 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, /* Insert only those elements in the final list that we haven't inserted before. */ for (auto & j : matches) - if (done.find(j.second) == done.end()) { - done.insert(j.second); + if (done.insert(j.second).second) elems.push_back(j.first); - } } checkSelectorUse(selectors); @@ -387,23 +374,21 @@ static void queryInstSources(EvalState & state, case srcStorePaths: { for (auto & i : args) { - Path path = state.store->followLinksToStorePath(i); + auto path = state.store->followLinksToStorePath(i); - string name = baseNameOf(path); - string::size_type dash = name.find('-'); - if (dash != string::npos) - name = string(name, dash + 1); + std::string name(path.name()); - DrvInfo elem(state, name, "", "", 0); + DrvInfo elem(state, "", nullptr); + elem.setName(name); - if (isDerivation(path)) { - elem.setDrvPath(path); - elem.setOutPath(state.store->derivationFromPath(path).findOutput("out")); + if (path.isDerivation()) { + elem.setDrvPath(state.store->printStorePath(path)); + elem.setOutPath(state.store->printStorePath(state.store->derivationFromPath(path).findOutput("out"))); if (name.size() >= drvExtension.size() && string(name, name.size() - drvExtension.size()) == drvExtension) name = string(name, 0, name.size() - drvExtension.size()); } - else elem.setOutPath(path); + else elem.setOutPath(state.store->printStorePath(path)); elems.push_back(elem); } @@ -425,7 +410,7 @@ static void queryInstSources(EvalState & state, Value vRoot; loadSourceExpr(state, instSource.nixExprPath, vRoot); for (auto & i : args) { - Value & v(*findAlongAttrPath(state, i, *instSource.autoArgs, vRoot)); + Value & v(*findAlongAttrPath(state, i, *instSource.autoArgs, vRoot).first); getDerivations(state, v, "", *instSource.autoArgs, elems, true); } break; @@ -436,13 +421,13 @@ static void queryInstSources(EvalState & state, static void printMissing(EvalState & state, DrvInfos & elems) { - PathSet targets; + std::vector targets; for (auto & i : elems) { Path drvPath = i.queryDrvPath(); if (drvPath != "") - targets.insert(drvPath); + targets.emplace_back(state.store->parseStorePath(drvPath)); else - targets.insert(i.queryOutPath()); + targets.emplace_back(state.store->parseStorePath(i.queryOutPath())); } printMissing(state.store, targets); @@ -476,8 +461,8 @@ static void installDerivations(Globals & globals, path is not the one we want (e.g., `java-front' versus `java-front-0.9pre15899'). */ if (globals.forceName != "") - i.name = globals.forceName; - newNames.insert(DrvName(i.name).name); + i.setName(globals.forceName); + newNames.insert(DrvName(i.queryName()).name); } @@ -492,17 +477,17 @@ static void installDerivations(Globals & globals, DrvInfos installedElems = queryInstalled(*globals.state, profile); for (auto & i : installedElems) { - DrvName drvName(i.name); + DrvName drvName(i.queryName()); if (!globals.preserveInstalled && newNames.find(drvName.name) != newNames.end() && !keep(i)) - printInfo(format("replacing old ‘%1%’") % i.name); + printInfo("replacing old '%s'", i.queryName()); else allElems.push_back(i); } for (auto & i : newElems) - printInfo(format("installing ‘%1%’") % i.name); + printInfo("installing '%s'", i.queryName()); } printMissing(*globals.state, newElems); @@ -524,7 +509,7 @@ static void opInstall(Globals & globals, Strings opFlags, Strings opArgs) globals.preserveInstalled = true; else if (arg == "--remove-all" || arg == "-r") globals.removeAll = true; - else throw UsageError(format("unknown flag ‘%1%’") % arg); + else throw UsageError("unknown flag '%1%'", arg); } installDerivations(globals, opArgs, globals.profile); @@ -556,7 +541,7 @@ static void upgradeDerivations(Globals & globals, /* Go through all installed derivations. */ DrvInfos newElems; for (auto & i : installedElems) { - DrvName drvName(i.name); + DrvName drvName(i.queryName()); try { @@ -577,7 +562,7 @@ static void upgradeDerivations(Globals & globals, for (auto j = availElems.begin(); j != availElems.end(); ++j) { if (comparePriorities(*globals.state, i, *j) > 0) continue; - DrvName newName(j->name); + DrvName newName(j->queryName()); if (newName.name == drvName.name) { int d = compareVersions(drvName.version, newName.version); if ((upgradeType == utLt && d < 0) || @@ -585,7 +570,7 @@ static void upgradeDerivations(Globals & globals, (upgradeType == utEq && d == 0) || upgradeType == utAlways) { - int d2 = -1; + long d2 = -1; if (bestElem != availElems.end()) { d2 = comparePriorities(*globals.state, *bestElem, *j); if (d2 == 0) d2 = compareVersions(bestVersion, newName.version); @@ -604,14 +589,13 @@ static void upgradeDerivations(Globals & globals, { const char * action = compareVersions(drvName.version, bestVersion) <= 0 ? "upgrading" : "downgrading"; - printInfo( - format("%1% ‘%2%’ to ‘%3%’") - % action % i.name % bestElem->name); + printInfo("%1% '%2%' to '%3%'", + action, i.queryName(), bestElem->queryName()); newElems.push_back(*bestElem); } else newElems.push_back(i); } catch (Error & e) { - e.addPrefix(format("while trying to find an upgrade for ‘%1%’:\n") % i.name); + e.addPrefix(fmt("while trying to find an upgrade for '%s':\n", i.queryName())); throw; } } @@ -636,7 +620,7 @@ static void opUpgrade(Globals & globals, Strings opFlags, Strings opArgs) else if (arg == "--leq") upgradeType = utLeq; else if (arg == "--eq") upgradeType = utEq; else if (arg == "--always") upgradeType = utAlways; - else throw UsageError(format("unknown flag ‘%1%’") % arg); + else throw UsageError("unknown flag '%1%'", arg); } upgradeDerivations(globals, opArgs, upgradeType); @@ -655,9 +639,9 @@ static void setMetaFlag(EvalState & state, DrvInfo & drv, static void opSetFlag(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) - throw UsageError(format("unknown flag ‘%1%’") % opFlags.front()); + throw UsageError("unknown flag '%1%'", opFlags.front()); if (opArgs.size() < 2) - throw UsageError("not enough arguments to ‘--set-flag’"); + throw UsageError("not enough arguments to '--set-flag'"); Strings::iterator arg = opArgs.begin(); string flagName = *arg++; @@ -671,10 +655,10 @@ static void opSetFlag(Globals & globals, Strings opFlags, Strings opArgs) /* Update all matching derivations. */ for (auto & i : installedElems) { - DrvName drvName(i.name); + DrvName drvName(i.queryName()); for (auto & j : selectors) if (j.matches(drvName)) { - printInfo(format("setting flag on ‘%1%’") % i.name); + printInfo("setting flag on '%1%'", i.queryName()); j.hits++; setMetaFlag(*globals.state, i, flagName, flagValue); break; @@ -698,7 +682,7 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) { string arg = *i++; if (parseInstallSourceOptions(globals, i, opFlags, arg)) ; - else throw UsageError(format("unknown flag ‘%1%’") % arg); + else throw UsageError("unknown flag '%1%'", arg); } DrvInfos elems; @@ -710,18 +694,18 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) DrvInfo & drv(elems.front()); if (globals.forceName != "") - drv.name = globals.forceName; + drv.setName(globals.forceName); if (drv.queryDrvPath() != "") { - PathSet paths = {drv.queryDrvPath()}; + std::vector paths{globals.state->store->parseStorePath(drv.queryDrvPath())}; printMissing(globals.state->store, paths); if (globals.dryRun) return; globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal); - } - else { - printMissing(globals.state->store, {drv.queryOutPath()}); + } else { + printMissing(globals.state->store, + {globals.state->store->parseStorePath(drv.queryOutPath())}); if (globals.dryRun) return; - globals.state->store->ensurePath(drv.queryOutPath()); + globals.state->store->ensurePath(globals.state->store->parseStorePath(drv.queryOutPath())); } debug(format("switching to new user environment")); @@ -736,28 +720,39 @@ static void uninstallDerivations(Globals & globals, Strings & selectors, while (true) { string lockToken = optimisticLockProfile(profile); - DrvInfos installedElems = queryInstalled(*globals.state, profile); - DrvInfos newElems; + DrvInfos workingElems = queryInstalled(*globals.state, profile); - for (auto & i : installedElems) { - DrvName drvName(i.name); - bool found = false; - for (auto & j : selectors) - /* !!! the repeated calls to followLinksToStorePath() - are expensive, should pre-compute them. */ - if ((isPath(j) && i.queryOutPath() == globals.state->store->followLinksToStorePath(j)) - || DrvName(j).matches(drvName)) - { - printInfo(format("uninstalling ‘%1%’") % i.name); - found = true; - break; - } - if (!found) newElems.push_back(i); + for (auto & selector : selectors) { + DrvInfos::iterator split = workingElems.begin(); + if (isPath(selector)) { + StorePath selectorStorePath = globals.state->store->followLinksToStorePath(selector); + split = std::partition( + workingElems.begin(), workingElems.end(), + [&selectorStorePath, globals](auto &elem) { + return selectorStorePath != globals.state->store->parseStorePath(elem.queryOutPath()); + } + ); + } else { + DrvName selectorName(selector); + split = std::partition( + workingElems.begin(), workingElems.end(), + [&selectorName](auto &elem){ + DrvName elemName(elem.queryName()); + return !selectorName.matches(elemName); + } + ); + } + if (split == workingElems.end()) + warn("selector '%s' matched no installed derivations", selector); + for (auto removedElem = split; removedElem != workingElems.end(); removedElem++) { + printInfo("uninstalling '%s'", removedElem->queryName()); + } + workingElems.erase(split, workingElems.end()); } if (globals.dryRun) return; - if (createUserEnv(*globals.state, newElems, + if (createUserEnv(*globals.state, workingElems, profile, settings.envKeepDerivations, lockToken)) break; } } @@ -766,7 +761,7 @@ static void uninstallDerivations(Globals & globals, Strings & selectors, static void opUninstall(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) - throw UsageError(format("unknown flag ‘%1%’") % opFlags.front()); + throw UsageError("unknown flag '%1%'", opFlags.front()); uninstallDerivations(globals, opArgs, globals.profile); } @@ -779,9 +774,11 @@ static bool cmpChars(char a, char b) static bool cmpElemByName(const DrvInfo & a, const DrvInfo & b) { + auto a_name = a.queryName(); + auto b_name = b.queryName(); return lexicographical_compare( - a.name.begin(), a.name.end(), - b.name.begin(), b.name.end(), cmpChars); + a_name.begin(), a_name.end(), + b_name.begin(), b_name.end(), cmpChars); } @@ -790,22 +787,22 @@ typedef list Table; void printTable(Table & table) { - unsigned int nrColumns = table.size() > 0 ? table.front().size() : 0; + auto nrColumns = table.size() > 0 ? table.front().size() : 0; - vector widths; + vector widths; widths.resize(nrColumns); for (auto & i : table) { assert(i.size() == nrColumns); Strings::iterator j; - unsigned int column; + size_t column; for (j = i.begin(), column = 0; j != i.end(); ++j, ++column) if (j->size() > widths[column]) widths[column] = j->size(); } for (auto & i : table) { Strings::iterator j; - unsigned int column; + size_t column; for (j = i.begin(), column = 0; j != i.end(); ++j, ++column) { string s = *j; replace(s.begin(), s.end(), '\n', ' '); @@ -830,13 +827,13 @@ typedef enum { cvLess, cvEqual, cvGreater, cvUnavail } VersionDiff; static VersionDiff compareVersionAgainstSet( const DrvInfo & elem, const DrvInfos & elems, string & version) { - DrvName name(elem.name); + DrvName name(elem.queryName()); VersionDiff diff = cvUnavail; version = "?"; for (auto & i : elems) { - DrvName name2(i.name); + DrvName name2(i.queryName()); if (name.name == name2.name) { int d = compareVersions(name.version, name2.version); if (d < 0) { @@ -865,8 +862,11 @@ static void queryJSON(Globals & globals, vector & elems) for (auto & i : elems) { JSONObject pkgObj = topObj.object(i.attrPath); - pkgObj.attr("name", i.name); - pkgObj.attr("system", i.system); + auto drvName = DrvName(i.queryName()); + pkgObj.attr("name", drvName.fullName); + pkgObj.attr("pname", drvName.name); + pkgObj.attr("version", drvName.version); + pkgObj.attr("system", i.querySystem()); JSONObject metaObj = pkgObj.object("meta"); StringSet metaNames = i.queryMetaNames(); @@ -874,7 +874,11 @@ static void queryJSON(Globals & globals, vector & elems) auto placeholder = metaObj.placeholder(j); Value * v = i.queryMeta(j); if (!v) { - printError(format("derivation ‘%1%’ has invalid meta attribute ‘%2%’") % i.name % j); + logError({ + .name = "Invalid meta attribute", + .hint = hintfmt("derivation '%s' has invalid meta attribute '%s'", + i.queryName(), j) + }); placeholder.write(nullptr); } else { PathSet context; @@ -924,9 +928,11 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) else if (arg == "--attr" || arg == "-A") attrPath = needArg(i, opFlags, arg); else - throw UsageError(format("unknown flag ‘%1%’") % arg); + throw UsageError("unknown flag '%1%'", arg); } + if (printAttrPath && source != sAvailable) + throw UsageError("--attr-path(-P) only works with --available"); /* Obtain derivation information from the specified source. */ DrvInfos availElems, installedElems; @@ -964,14 +970,15 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) /* Query which paths have substitutes. */ - PathSet validPaths, substitutablePaths; + StorePathSet validPaths; + StorePathSet substitutablePaths; if (printStatus || globals.prebuiltOnly) { - PathSet paths; + StorePathSet paths; for (auto & i : elems) try { - paths.insert(i.queryOutPath()); + paths.insert(globals.state->store->parseStorePath(i.queryOutPath())); } catch (AssertionError & e) { - printMsg(lvlTalkative, format("skipping derivation named ‘%1%’ which gives an assertion failure") % i.name); + printMsg(lvlTalkative, "skipping derivation named '%s' which gives an assertion failure", i.queryName()); i.setFailed(); } validPaths = globals.state->store->queryValidPaths(paths); @@ -997,11 +1004,11 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) try { if (i.hasFailed()) continue; - Activity act(*logger, lvlDebug, format("outputting query result ‘%1%’") % i.attrPath); + //Activity act(*logger, lvlDebug, format("outputting query result '%1%'") % i.attrPath); if (globals.prebuiltOnly && - validPaths.find(i.queryOutPath()) == validPaths.end() && - substitutablePaths.find(i.queryOutPath()) == substitutablePaths.end()) + !validPaths.count(globals.state->store->parseStorePath(i.queryOutPath())) && + !substitutablePaths.count(globals.state->store->parseStorePath(i.queryOutPath()))) continue; /* For table output. */ @@ -1012,9 +1019,9 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) if (printStatus) { Path outPath = i.queryOutPath(); - bool hasSubs = substitutablePaths.find(outPath) != substitutablePaths.end(); + bool hasSubs = substitutablePaths.count(globals.state->store->parseStorePath(outPath)); bool isInstalled = installed.find(outPath) != installed.end(); - bool isValid = validPaths.find(outPath) != validPaths.end(); + bool isValid = validPaths.count(globals.state->store->parseStorePath(outPath)); if (xmlOutput) { attrs["installed"] = isInstalled ? "1" : "0"; attrs["valid"] = isValid ? "1" : "0"; @@ -1031,10 +1038,14 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) else if (printAttrPath) columns.push_back(i.attrPath); - if (xmlOutput) - attrs["name"] = i.name; - else if (printName) - columns.push_back(i.name); + if (xmlOutput) { + auto drvName = DrvName(i.queryName()); + attrs["name"] = drvName.fullName; + attrs["pname"] = drvName.name; + attrs["version"] = drvName.version; + } else if (printName) { + columns.push_back(i.queryName()); + } if (compareVersions) { /* Compare this element against the versions of the @@ -1067,10 +1078,10 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) } if (xmlOutput) { - if (i.system != "") attrs["system"] = i.system; + if (i.querySystem() != "") attrs["system"] = i.querySystem(); } else if (printSystem) - columns.push_back(i.system); + columns.push_back(i.querySystem()); if (printDrvPath) { string drvPath = i.queryDrvPath(); @@ -1118,7 +1129,12 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) attrs2["name"] = j; Value * v = i.queryMeta(j); if (!v) - printError(format("derivation ‘%1%’ has invalid meta attribute ‘%2%’") % i.name % j); + logError({ + .name = "Invalid meta attribute", + .hint = hintfmt( + "derivation '%s' has invalid meta attribute '%s'", + i.queryName(), j) + }); else { if (v->type == tString) { attrs2["type"] = "string"; @@ -1169,9 +1185,9 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) cout.flush(); } catch (AssertionError & e) { - printMsg(lvlTalkative, format("skipping derivation named ‘%1%’ which gives an assertion failure") % i.name); + printMsg(lvlTalkative, "skipping derivation named '%1%' which gives an assertion failure", i.queryName()); } catch (Error & e) { - e.addPrefix(format("while querying the derivation named ‘%1%’:\n") % i.name); + e.addPrefix(fmt("while querying the derivation named '%1%':\n", i.queryName())); throw; } } @@ -1183,12 +1199,12 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) static void opSwitchProfile(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) - throw UsageError(format("unknown flag ‘%1%’") % opFlags.front()); + throw UsageError("unknown flag '%1%'", opFlags.front()); if (opArgs.size() != 1) - throw UsageError(format("exactly one argument expected")); + throw UsageError("exactly one argument expected"); Path profile = absPath(opArgs.front()); - Path profileLink = getHomeDir() + "/.nix-profile"; + Path profileLink = getHome() + "/.nix-profile"; switchLink(profileLink, profile); } @@ -1213,10 +1229,10 @@ static void switchGeneration(Globals & globals, int dstGen) if (!dst) { if (dstGen == prevGen) - throw Error(format("no generation older than the current (%1%) exists") - % curGen); + throw Error("no generation older than the current (%1%) exists", + curGen); else - throw Error(format("generation %1% does not exist") % dstGen); + throw Error("generation %1% does not exist", dstGen); } printInfo(format("switching from generation %1% to %2%") @@ -1231,13 +1247,13 @@ static void switchGeneration(Globals & globals, int dstGen) static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) - throw UsageError(format("unknown flag ‘%1%’") % opFlags.front()); + throw UsageError("unknown flag '%1%'", opFlags.front()); if (opArgs.size() != 1) - throw UsageError(format("exactly one argument expected")); + throw UsageError("exactly one argument expected"); int dstGen; if (!string2Int(opArgs.front(), dstGen)) - throw UsageError(format("expected a generation number")); + throw UsageError("expected a generation number"); switchGeneration(globals, dstGen); } @@ -1246,9 +1262,9 @@ static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArg static void opRollback(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) - throw UsageError(format("unknown flag ‘%1%’") % opFlags.front()); + throw UsageError("unknown flag '%1%'", opFlags.front()); if (opArgs.size() != 0) - throw UsageError(format("no arguments expected")); + throw UsageError("no arguments expected"); switchGeneration(globals, prevGen); } @@ -1257,9 +1273,9 @@ static void opRollback(Globals & globals, Strings opFlags, Strings opArgs) static void opListGenerations(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) - throw UsageError(format("unknown flag ‘%1%’") % opFlags.front()); + throw UsageError("unknown flag '%1%'", opFlags.front()); if (opArgs.size() != 0) - throw UsageError(format("no arguments expected")); + throw UsageError("no arguments expected"); PathLocks lock; lockProfile(lock, globals.profile); @@ -1284,18 +1300,26 @@ static void opListGenerations(Globals & globals, Strings opFlags, Strings opArgs static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) - throw UsageError(format("unknown flag ‘%1%’") % opFlags.front()); + throw UsageError("unknown flag '%1%'", opFlags.front()); if (opArgs.size() == 1 && opArgs.front() == "old") { deleteOldGenerations(globals.profile, globals.dryRun); } else if (opArgs.size() == 1 && opArgs.front().find('d') != string::npos) { deleteGenerationsOlderThan(globals.profile, opArgs.front(), globals.dryRun); + } else if (opArgs.size() == 1 && opArgs.front().find('+') != string::npos) { + if(opArgs.front().size() < 2) + throw Error("invalid number of generations ‘%1%’", opArgs.front()); + string str_max = string(opArgs.front(), 1, opArgs.front().size()); + int max; + if (!string2Int(str_max, max) || max == 0) + throw Error("invalid number of generations to keep ‘%1%’", opArgs.front()); + deleteGenerationsGreaterThan(globals.profile, max, globals.dryRun); } else { std::set gens; for (auto & i : opArgs) { unsigned int n; if (!string2Int(i, n)) - throw UsageError(format("invalid generation number ‘%1%’") % i); + throw UsageError("invalid generation number '%1%'", i); gens.insert(n); } deleteGenerations(globals.profile, gens, globals.dryRun); @@ -1309,30 +1333,44 @@ static void opVersion(Globals & globals, Strings opFlags, Strings opArgs) } -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - initGC(); - - Strings opFlags, opArgs, searchPath; - std::map autoArgs_; + { + Strings opFlags, opArgs; Operation op = 0; - bool repair = false; + RepairFlag repair = NoRepair; string file; Globals globals; globals.instSource.type = srcUnknown; - globals.instSource.nixExprPath = getDefNixExprPath(); + globals.instSource.nixExprPath = getHome() + "/.nix-defexpr"; globals.instSource.systemFilter = "*"; + if (!pathExists(globals.instSource.nixExprPath)) { + try { + createDirs(globals.instSource.nixExprPath); + replaceSymlink( + fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()), + globals.instSource.nixExprPath + "/channels"); + if (getuid() != 0) + replaceSymlink( + fmt("%s/profiles/per-user/root/channels", settings.nixStateDir), + globals.instSource.nixExprPath + "/channels_root"); + } catch (Error &) { } + } + globals.dryRun = false; globals.preserveInstalled = false; globals.removeAll = false; globals.prebuiltOnly = false; - parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { + struct MyArgs : LegacyArgs, MixEvalArgs + { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) { Operation oldOp = op; if (*arg == "--help") @@ -1341,10 +1379,6 @@ int main(int argc, char * * argv) op = opVersion; else if (*arg == "--install" || *arg == "-i") op = opInstall; - else if (parseAutoArgs(arg, end, autoArgs_)) - ; - else if (parseSearchPathArg(arg, end, searchPath)) - ; else if (*arg == "--force-name") // undocumented flag for nix-install-package globals.forceName = getArg(*arg, arg, end); else if (*arg == "--uninstall" || *arg == "-e") @@ -1380,7 +1414,7 @@ int main(int argc, char * * argv) else if (*arg == "--prebuilt-only" || *arg == "-b") globals.prebuiltOnly = true; else if (*arg == "--repair") - repair = true; + repair = Repair; else if (*arg != "" && arg->at(0) == '-') { opFlags.push_back(*arg); /* FIXME: hacky */ @@ -1397,30 +1431,36 @@ int main(int argc, char * * argv) return true; }); + myArgs.parseCmdline(argvToStrings(argc, argv)); + + initPlugins(); + if (!op) throw UsageError("no operation specified"); auto store = openStore(); - globals.state = std::shared_ptr(new EvalState(searchPath, store)); + globals.state = std::shared_ptr(new EvalState(myArgs.searchPath, store)); globals.state->repair = repair; if (file != "") globals.instSource.nixExprPath = lookupFileArg(*globals.state, file); - globals.instSource.autoArgs = evalAutoArgs(*globals.state, autoArgs_); + globals.instSource.autoArgs = myArgs.getAutoArgs(*globals.state); if (globals.profile == "") - globals.profile = getEnv("NIX_PROFILE", ""); + globals.profile = getEnv("NIX_PROFILE").value_or(""); - if (globals.profile == "") { - Path profileLink = getHomeDir() + "/.nix-profile"; - globals.profile = pathExists(profileLink) - ? absPath(readLink(profileLink), dirOf(profileLink)) - : canonPath(settings.nixStateDir + "/profiles/default"); - } + if (globals.profile == "") + globals.profile = getDefaultProfile(); op(globals, opFlags, opArgs); globals.state->printStats(); - }); + + logger->stop(); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-env", _main); diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index e9997fae57b..f804b77a04e 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -15,6 +15,8 @@ namespace nix { DrvInfos queryInstalled(EvalState & state, const Path & userEnv) { DrvInfos elems; + if (pathExists(userEnv + "/manifest.json")) + throw Error("profile '%s' is incompatible with 'nix-env'; please use 'nix profile' instead", userEnv); Path manifestFile = userEnv + "/manifest.nix"; if (pathExists(manifestFile)) { Value v; @@ -32,16 +34,16 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, { /* Build the components in the user environment, if they don't exist already. */ - PathSet drvsToBuild; + std::vector drvsToBuild; for (auto & i : elems) if (i.queryDrvPath() != "") - drvsToBuild.insert(i.queryDrvPath()); + drvsToBuild.push_back({state.store->parseStorePath(i.queryDrvPath())}); debug(format("building user environment dependencies")); state.store->buildPaths(drvsToBuild, state.repair ? bmRepair : bmNormal); /* Construct the whole top level derivation. */ - PathSet references; + StorePathSet references; Value manifest; state.mkList(manifest, elems.size()); unsigned int n = 0; @@ -56,9 +58,10 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, state.mkAttrs(v, 16); mkString(*state.allocAttr(v, state.sType), "derivation"); - mkString(*state.allocAttr(v, state.sName), i.name); - if (!i.system.empty()) - mkString(*state.allocAttr(v, state.sSystem), i.system); + mkString(*state.allocAttr(v, state.sName), i.queryName()); + auto system = i.querySystem(); + if (!system.empty()) + mkString(*state.allocAttr(v, state.sSystem), system); mkString(*state.allocAttr(v, state.sOutPath), i.queryOutPath()); if (drvPath != "") mkString(*state.allocAttr(v, state.sDrvPath), i.queryDrvPath()); @@ -76,10 +79,10 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* This is only necessary when installing store paths, e.g., `nix-env -i /nix/store/abcd...-foo'. */ - state.store->addTempRoot(j.second); - state.store->ensurePath(j.second); + state.store->addTempRoot(state.store->parseStorePath(j.second)); + state.store->ensurePath(state.store->parseStorePath(j.second)); - references.insert(j.second); + references.insert(state.store->parseStorePath(j.second)); } // Copy the meta attributes. @@ -94,25 +97,27 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, vMeta.attrs->sort(); v.attrs->sort(); - if (drvPath != "") references.insert(drvPath); + if (drvPath != "") references.insert(state.store->parseStorePath(drvPath)); } /* Also write a copy of the list of user environment elements to the store; we need it for future modifications of the environment. */ - Path manifestFile = state.store->addTextToStore("env-manifest.nix", - (format("%1%") % manifest).str(), references); + auto manifestFile = state.store->addTextToStore("env-manifest.nix", + fmt("%s", manifest), references); /* Get the environment builder expression. */ Value envBuilder; - state.evalFile(state.findFile("nix/buildenv.nix"), envBuilder); + state.eval(state.parseExprFromString( + #include "buildenv.nix.gen.hh" + , "/"), envBuilder); /* Construct a Nix expression that calls the user environment builder with the manifest as argument. */ Value args, topLevel; state.mkAttrs(args, 3); mkString(*state.allocAttr(args, state.symbols.create("manifest")), - manifestFile, {manifestFile}); + state.store->printStorePath(manifestFile), {state.store->printStorePath(manifestFile)}); args.attrs->push_back(Attr(state.symbols.create("derivations"), &manifest)); args.attrs->sort(); mkApp(topLevel, envBuilder, args); @@ -122,13 +127,15 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, state.forceValue(topLevel); PathSet context; Attr & aDrvPath(*topLevel.attrs->find(state.sDrvPath)); - Path topLevelDrv = state.coerceToPath(aDrvPath.pos ? *(aDrvPath.pos) : noPos, *(aDrvPath.value), context); + auto topLevelDrv = state.store->parseStorePath(state.coerceToPath(aDrvPath.pos ? *(aDrvPath.pos) : noPos, *(aDrvPath.value), context)); Attr & aOutPath(*topLevel.attrs->find(state.sOutPath)); Path topLevelOut = state.coerceToPath(aOutPath.pos ? *(aOutPath.pos) : noPos, *(aOutPath.value), context); /* Realise the resulting store expression. */ debug("building user environment"); - state.store->buildPaths({topLevelDrv}, state.repair ? bmRepair : bmNormal); + std::vector topLevelDrvs; + topLevelDrvs.push_back(StorePathWithOutputs{topLevelDrv.clone()}); + state.store->buildPaths(topLevelDrvs, state.repair ? bmRepair : bmNormal); /* Switch the current user environment to the output path. */ auto store2 = state.store.dynamic_pointer_cast(); @@ -139,7 +146,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, Path lockTokenCur = optimisticLockProfile(profile); if (lockToken != lockTokenCur) { - printError(format("profile ‘%1%’ changed while we were busy; restarting") % profile); + printInfo("profile '%1%' changed while we were busy; restarting", profile); return false; } diff --git a/src/nix-instantiate/local.mk b/src/nix-instantiate/local.mk deleted file mode 100644 index 7d1bc5ec9df..00000000000 --- a/src/nix-instantiate/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-instantiate - -nix-instantiate_DIR := $(d) - -nix-instantiate_SOURCES := $(d)/nix-instantiate.cc - -nix-instantiate_LIBS = libexpr libmain libstore libutil libformat diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index c1b0b0ea092..bf353677a5b 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -8,7 +8,8 @@ #include "value-to-json.hh" #include "util.hh" #include "store-api.hh" -#include "common-opts.hh" +#include "common-eval-args.hh" +#include "../nix/legacy.hh" #include #include @@ -17,13 +18,6 @@ using namespace nix; -static Expr * parseStdin(EvalState & state) -{ - Activity act(*logger, lvlTalkative, format("parsing standard input")); - return state.parseExprFromString(drainFD(0), absPath(".")); -} - - static Path gcRoot; static int rootNr = 0; static bool indirectRoot = false; @@ -45,7 +39,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, state.eval(e, vRoot); for (auto & i : attrPaths) { - Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot)); + Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot).first); state.forceValue(v); PathSet context; @@ -72,31 +66,28 @@ void processExpr(EvalState & state, const Strings & attrPaths, /* What output do we want? */ string outputName = i.queryOutputName(); if (outputName == "") - throw Error(format("derivation ‘%1%’ lacks an ‘outputName’ attribute ") % drvPath); + throw Error("derivation '%1%' lacks an 'outputName' attribute ", drvPath); if (gcRoot == "") printGCWarning(); else { - Path rootName = gcRoot; + Path rootName = indirectRoot ? absPath(gcRoot) : gcRoot; if (++rootNr > 1) rootName += "-" + std::to_string(rootNr); auto store2 = state.store.dynamic_pointer_cast(); if (store2) - drvPath = store2->addPermRoot(drvPath, rootName, indirectRoot); + drvPath = store2->addPermRoot(store2->parseStorePath(drvPath), rootName, indirectRoot); } - std::cout << format("%1%%2%\n") % drvPath % (outputName != "out" ? "!" + outputName : ""); + std::cout << fmt("%s%s\n", drvPath, (outputName != "out" ? "!" + outputName : "")); } } } } -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - initGC(); - - Strings files, searchPath; + { + Strings files; bool readStdin = false; bool fromArgs = false; bool findFile = false; @@ -107,10 +98,14 @@ int main(int argc, char * * argv) bool strict = false; Strings attrPaths; bool wantsReadWrite = false; - std::map autoArgs_; - bool repair = false; + RepairFlag repair = NoRepair; + + struct MyArgs : LegacyArgs, MixEvalArgs + { + using LegacyArgs::LegacyArgs; + }; - parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { + MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--help") showManPage("nix-instantiate"); else if (*arg == "--version") @@ -129,10 +124,6 @@ int main(int argc, char * * argv) findFile = true; else if (*arg == "--attr" || *arg == "-A") attrPaths.push_back(getArg(*arg, arg, end)); - else if (parseAutoArgs(arg, end, autoArgs_)) - ; - else if (parseSearchPathArg(arg, end, searchPath)) - ; else if (*arg == "--add-root") gcRoot = getArg(*arg, arg, end); else if (*arg == "--indirect") @@ -146,7 +137,7 @@ int main(int argc, char * * argv) else if (*arg == "--strict") strict = true; else if (*arg == "--repair") - repair = true; + repair = Repair; else if (*arg == "--dry-run") settings.readOnlyMode = true; else if (*arg != "" && arg->at(0) == '-') @@ -156,42 +147,50 @@ int main(int argc, char * * argv) return true; }); + myArgs.parseCmdline(argvToStrings(argc, argv)); + + initPlugins(); + if (evalOnly && !wantsReadWrite) settings.readOnlyMode = true; auto store = openStore(); - EvalState state(searchPath, store); - state.repair = repair; + auto state = std::make_unique(myArgs.searchPath, store); + state->repair = repair; - Bindings & autoArgs(*evalAutoArgs(state, autoArgs_)); + Bindings & autoArgs = *myArgs.getAutoArgs(*state); - if (attrPaths.empty()) attrPaths.push_back(""); + if (attrPaths.empty()) attrPaths = {""}; if (findFile) { for (auto & i : files) { - Path p = state.findFile(i); - if (p == "") throw Error(format("unable to find ‘%1%’") % i); + Path p = state->findFile(i); + if (p == "") throw Error("unable to find '%1%'", i); std::cout << p << std::endl; } - return; + return 0; } if (readStdin) { - Expr * e = parseStdin(state); - processExpr(state, attrPaths, parseOnly, strict, autoArgs, + Expr * e = state->parseStdin(); + processExpr(*state, attrPaths, parseOnly, strict, autoArgs, evalOnly, outputKind, xmlOutputSourceLocation, e); } else if (files.empty() && !fromArgs) files.push_back("./default.nix"); for (auto & i : files) { Expr * e = fromArgs - ? state.parseExprFromString(i, absPath(".")) - : state.parseExprFromFile(resolveExprPath(lookupFileArg(state, i))); - processExpr(state, attrPaths, parseOnly, strict, autoArgs, + ? state->parseExprFromString(i, absPath(".")) + : state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, i)))); + processExpr(*state, attrPaths, parseOnly, strict, autoArgs, evalOnly, outputKind, xmlOutputSourceLocation, e); } - state.printStats(); - }); + state->printStats(); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-instantiate", _main); diff --git a/src/nix-prefetch-url/local.mk b/src/nix-prefetch-url/local.mk deleted file mode 100644 index 3e7735406af..00000000000 --- a/src/nix-prefetch-url/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-prefetch-url - -nix-prefetch-url_DIR := $(d) - -nix-prefetch-url_SOURCES := $(d)/nix-prefetch-url.cc - -nix-prefetch-url_LIBS = libmain libexpr libstore libutil libformat diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index acf60302569..55b72bda6e6 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -1,14 +1,22 @@ #include "hash.hh" #include "shared.hh" -#include "download.hh" +#include "filetransfer.hh" #include "store-api.hh" #include "eval.hh" #include "eval-inline.hh" -#include "common-opts.hh" +#include "common-eval-args.hh" #include "attr-path.hh" +#include "finally.hh" +#include "../nix/legacy.hh" +#include "progress-bar.hh" +#include "tarfile.hh" #include +#include +#include +#include + using namespace nix; @@ -29,34 +37,34 @@ string resolveMirrorUri(EvalState & state, string uri) auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); if (mirrorList == vMirrors.attrs->end()) - throw Error(format("unknown mirror name ‘%1%’") % mirrorName); + throw Error("unknown mirror name '%1%'", mirrorName); state.forceList(*mirrorList->value); if (mirrorList->value->listSize() < 1) - throw Error(format("mirror URI ‘%1%’ did not expand to anything") % uri); + throw Error("mirror URI '%1%' did not expand to anything", uri); string mirror = state.forceString(*mirrorList->value->listElems()[0]); return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1); } -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - initGC(); - + { HashType ht = htSHA256; std::vector args; - Strings searchPath; - bool printPath = getEnv("PRINT_PATH") != ""; + bool printPath = getEnv("PRINT_PATH") == "1"; bool fromExpr = false; string attrPath; - std::map autoArgs_; bool unpack = false; string name; - parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { + struct MyArgs : LegacyArgs, MixEvalArgs + { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--help") showManPage("nix-prefetch-url"); else if (*arg == "--version") @@ -65,7 +73,7 @@ int main(int argc, char * * argv) string s = getArg(*arg, arg, end); ht = parseHashType(s); if (ht == htUnknown) - throw UsageError(format("unknown hash type ‘%1%’") % s); + throw UsageError("unknown hash type '%1%'", s); } else if (*arg == "--print-path") printPath = true; @@ -77,10 +85,6 @@ int main(int argc, char * * argv) unpack = true; else if (*arg == "--name") name = getArg(*arg, arg, end); - else if (parseAutoArgs(arg, end, autoArgs_)) - ; - else if (parseSearchPathArg(arg, end, searchPath)) - ; else if (*arg != "" && arg->at(0) == '-') return false; else @@ -88,13 +92,22 @@ int main(int argc, char * * argv) return true; }); + myArgs.parseCmdline(argvToStrings(argc, argv)); + + initPlugins(); + if (args.size() > 2) throw UsageError("too many arguments"); + Finally f([]() { stopProgressBar(); }); + + if (isatty(STDERR_FILENO)) + startProgressBar(); + auto store = openStore(); - EvalState state(searchPath, store); + auto state = std::make_unique(myArgs.searchPath, store); - Bindings & autoArgs(*evalAutoArgs(state, autoArgs_)); + Bindings & autoArgs = *myArgs.getAutoArgs(*state); /* If -A is given, get the URI from the specified Nix expression. */ @@ -104,33 +117,33 @@ int main(int argc, char * * argv) throw UsageError("you must specify a URI"); uri = args[0]; } else { - Path path = resolveExprPath(lookupFileArg(state, args.empty() ? "." : args[0])); + Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); Value vRoot; - state.evalFile(path, vRoot); - Value & v(*findAlongAttrPath(state, attrPath, autoArgs, vRoot)); - state.forceAttrs(v); + state->evalFile(path, vRoot); + Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first); + state->forceAttrs(v); /* Extract the URI. */ - auto attr = v.attrs->find(state.symbols.create("urls")); + auto attr = v.attrs->find(state->symbols.create("urls")); if (attr == v.attrs->end()) - throw Error("attribute set does not contain a ‘urls’ attribute"); - state.forceList(*attr->value); + throw Error("attribute set does not contain a 'urls' attribute"); + state->forceList(*attr->value); if (attr->value->listSize() < 1) - throw Error("‘urls’ list is empty"); - uri = state.forceString(*attr->value->listElems()[0]); + throw Error("'urls' list is empty"); + uri = state->forceString(*attr->value->listElems()[0]); /* Extract the hash mode. */ - attr = v.attrs->find(state.symbols.create("outputHashMode")); + attr = v.attrs->find(state->symbols.create("outputHashMode")); if (attr == v.attrs->end()) printInfo("warning: this does not look like a fetchurl call"); else - unpack = state.forceString(*attr->value) == "recursive"; + unpack = state->forceString(*attr->value) == "recursive"; /* Extract the name. */ if (name.empty()) { - attr = v.attrs->find(state.symbols.create("name")); + attr = v.attrs->find(state->symbols.create("name")); if (attr != v.attrs->end()) - name = state.forceString(*attr->value); + name = state->forceString(*attr->value); } } @@ -138,42 +151,47 @@ int main(int argc, char * * argv) if (name.empty()) name = baseNameOf(uri); if (name.empty()) - throw Error(format("cannot figure out file name for ‘%1%’") % uri); + throw Error("cannot figure out file name for '%1%'", uri); /* If an expected hash is given, the file may already exist in the store. */ Hash hash, expectedHash(ht); - Path storePath; + std::optional storePath; if (args.size() == 2) { - expectedHash = parseHash16or32(ht, args[1]); - storePath = store->makeFixedOutputPath(unpack, expectedHash, name); - if (store->isValidPath(storePath)) + expectedHash = Hash(args[1], ht); + const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + storePath = store->makeFixedOutputPath(recursive, expectedHash, name); + if (store->isValidPath(*storePath)) hash = expectedHash; else - storePath.clear(); + storePath.reset(); } - if (storePath.empty()) { + if (!storePath) { - auto actualUri = resolveMirrorUri(state, uri); - - /* Download the file. */ - auto result = getDownloader()->download(DownloadRequest(actualUri)); + auto actualUri = resolveMirrorUri(*state, uri); AutoDelete tmpDir(createTempDir(), true); Path tmpFile = (Path) tmpDir + "/tmp"; - writeFile(tmpFile, *result.data); + + /* Download the file. */ + { + AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600); + if (!fd) throw SysError("creating temporary file '%s'", tmpFile); + + FdSink sink(fd.get()); + + FileTransferRequest req(actualUri); + req.decompress = false; + getFileTransfer()->download(std::move(req), sink); + } /* Optionally unpack the file. */ if (unpack) { printInfo("unpacking..."); Path unpacked = (Path) tmpDir + "/unpacked"; createDirs(unpacked); - if (hasSuffix(baseNameOf(uri), ".zip")) - runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked}, ""); - else - // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", tmpFile, "-C", unpacked}, ""); + unpackTarfile(tmpFile, unpacked); /* If the archive unpacks to a single file/directory, then use that as the top-level. */ @@ -186,25 +204,33 @@ int main(int argc, char * * argv) /* FIXME: inefficient; addToStore() will also hash this. */ - hash = unpack ? hashPath(ht, tmpFile).first : hashString(ht, *result.data); + hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile); if (expectedHash != Hash(ht) && expectedHash != hash) - throw Error(format("hash mismatch for ‘%1%’") % uri); + throw Error("hash mismatch for '%1%'", uri); + + const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; /* Copy the file to the Nix store. FIXME: if RemoteStore implemented addToStoreFromDump() and downloadFile() supported a sink, we could stream the download directly into the Nix store. */ - storePath = store->addToStore(name, tmpFile, unpack, ht); + storePath = store->addToStore(name, tmpFile, recursive, ht); - assert(storePath == store->makeFixedOutputPath(unpack, hash, name)); + assert(*storePath == store->makeFixedOutputPath(recursive, hash, name)); } + stopProgressBar(); + if (!printPath) - printInfo(format("path is ‘%1%’") % storePath); + printInfo("path is '%s'", store->printStorePath(*storePath)); std::cout << printHash16or32(hash) << std::endl; if (printPath) - std::cout << storePath << std::endl; - }); + std::cout << store->printStorePath(*storePath) << std::endl; + + return 0; + } } + +static RegisterLegacyCommand s1("nix-prefetch-url", _main); diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 356a8251012..667d917f510 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -10,9 +10,9 @@ using std::cout; namespace nix { -static string dotQuote(const string & s) +static string dotQuote(std::string_view s) { - return "\"" + s + "\""; + return "\"" + std::string(s) + "\""; } @@ -34,7 +34,7 @@ static string makeEdge(const string & src, const string & dst) } -static string makeNode(const string & id, const string & label, +static string makeNode(const string & id, std::string_view label, const string & colour) { format f = format("%1% [label = %2%, shape = box, " @@ -44,109 +44,26 @@ static string makeNode(const string & id, const string & label, } -static string symbolicName(const string & path) +void printDotGraph(ref store, StorePathSet && roots) { - string p = baseNameOf(path); - int dash = p.find('-'); - return string(p, dash + 1); -} - - -#if 0 -string pathLabel(const Path & nePath, const string & elemPath) -{ - return (string) nePath + "-" + elemPath; -} - - -void printClosure(const Path & nePath, const StoreExpr & fs) -{ - PathSet workList(fs.closure.roots); - PathSet doneSet; - - for (PathSet::iterator i = workList.begin(); i != workList.end(); ++i) { - cout << makeEdge(pathLabel(nePath, *i), nePath); - } - - while (!workList.empty()) { - Path path = *(workList.begin()); - workList.erase(path); - - if (doneSet.find(path) == doneSet.end()) { - doneSet.insert(path); - - ClosureElems::const_iterator elem = fs.closure.elems.find(path); - if (elem == fs.closure.elems.end()) - throw Error(format("bad closure, missing path ‘%1%’") % path); - - for (StringSet::const_iterator i = elem->second.refs.begin(); - i != elem->second.refs.end(); ++i) - { - workList.insert(*i); - cout << makeEdge(pathLabel(nePath, *i), pathLabel(nePath, path)); - } - - cout << makeNode(pathLabel(nePath, path), - symbolicName(path), "#ff0000"); - } - } -} -#endif - - -void printDotGraph(ref store, const PathSet & roots) -{ - PathSet workList(roots); - PathSet doneSet; + StorePathSet workList(std::move(roots)); + StorePathSet doneSet; cout << "digraph G {\n"; while (!workList.empty()) { - Path path = *(workList.begin()); - workList.erase(path); + auto path = std::move(workList.extract(workList.begin()).value()); - if (doneSet.find(path) != doneSet.end()) continue; - doneSet.insert(path); + if (!doneSet.insert(path.clone()).second) continue; - cout << makeNode(path, symbolicName(path), "#ff0000"); + cout << makeNode(std::string(path.to_string()), path.name(), "#ff0000"); for (auto & p : store->queryPathInfo(path)->references) { if (p != path) { - workList.insert(p); - cout << makeEdge(p, path); - } - } - -#if 0 - StoreExpr ne = storeExprFromPath(path); - - string label, colour; - - if (ne.type == StoreExpr::neDerivation) { - for (PathSet::iterator i = ne.derivation.inputs.begin(); - i != ne.derivation.inputs.end(); ++i) - { - workList.insert(*i); - cout << makeEdge(*i, path); + workList.insert(p.clone()); + cout << makeEdge(std::string(p.to_string()), std::string(path.to_string())); } - - label = "derivation"; - colour = "#00ff00"; - for (StringPairs::iterator i = ne.derivation.env.begin(); - i != ne.derivation.env.end(); ++i) - if (i->first == "name") label = i->second; } - - else if (ne.type == StoreExpr::neClosure) { - label = ""; - colour = "#00ffff"; - printClosure(path, ne); - } - - else abort(); - - cout << makeNode(path, label, colour); -#endif } cout << "}\n"; diff --git a/src/nix-store/dotgraph.hh b/src/nix-store/dotgraph.hh index e2b5fc72fbe..73b8d06b945 100644 --- a/src/nix-store/dotgraph.hh +++ b/src/nix-store/dotgraph.hh @@ -1,11 +1,9 @@ #pragma once -#include "types.hh" +#include "store-api.hh" namespace nix { -class Store; - -void printDotGraph(ref store, const PathSet & roots); +void printDotGraph(ref store, StorePathSet && roots); } diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc new file mode 100644 index 00000000000..34770885104 --- /dev/null +++ b/src/nix-store/graphml.cc @@ -0,0 +1,88 @@ +#include "graphml.hh" +#include "util.hh" +#include "store-api.hh" +#include "derivations.hh" + +#include + + +using std::cout; + +namespace nix { + + +static inline std::string_view xmlQuote(std::string_view s) +{ + // Luckily, store paths shouldn't contain any character that needs to be + // quoted. + return s; +} + + +static string symbolicName(const std::string & p) +{ + return string(p, p.find('-') + 1); +} + + +static string makeEdge(std::string_view src, std::string_view dst) +{ + return fmt(" \n", + xmlQuote(src), xmlQuote(dst)); +} + + +static string makeNode(const ValidPathInfo & info) +{ + return fmt( + " \n" + " %2%\n" + " %3%\n" + " %4%\n" + " \n", + info.path.to_string(), + info.narSize, + symbolicName(std::string(info.path.name())), + (info.path.isDerivation() ? "derivation" : "output-path")); +} + + +void printGraphML(ref store, StorePathSet && roots) +{ + StorePathSet workList(std::move(roots)); + StorePathSet doneSet; + std::pair ret; + + cout << "\n" + << "\n" + << "" + << "" + << "" + << "\n"; + + while (!workList.empty()) { + auto path = std::move(workList.extract(workList.begin()).value()); + + ret = doneSet.insert(path.clone()); + if (ret.second == false) continue; + + auto info = store->queryPathInfo(path); + cout << makeNode(*info); + + for (auto & p : info->references) { + if (p != path) { + workList.insert(p.clone()); + cout << makeEdge(path.to_string(), p.to_string()); + } + } + + } + + cout << "\n"; + cout << "\n"; +} + + +} diff --git a/src/nix-store/graphml.hh b/src/nix-store/graphml.hh new file mode 100644 index 00000000000..78be8a3676e --- /dev/null +++ b/src/nix-store/graphml.hh @@ -0,0 +1,9 @@ +#pragma once + +#include "store-api.hh" + +namespace nix { + +void printGraphML(ref store, StorePathSet && roots); + +} diff --git a/src/nix-store/local.mk b/src/nix-store/local.mk deleted file mode 100644 index 84ff15b241f..00000000000 --- a/src/nix-store/local.mk +++ /dev/null @@ -1,11 +0,0 @@ -programs += nix-store - -nix-store_DIR := $(d) - -nix-store_SOURCES := $(wildcard $(d)/*.cc) - -nix-store_LIBS = libmain libstore libutil libformat - -nix-store_LDFLAGS = -lbz2 -pthread $(SODIUM_LIBS) - -nix-store_CXXFLAGS = -DCURL=\"$(curl)\" diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index c1e6afef0e5..6d21395261d 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -8,8 +8,8 @@ #include "shared.hh" #include "util.hh" #include "worker-protocol.hh" -#include "xmlgraph.hh" -#include "compression.hh" +#include "graphml.hh" +#include "../nix/legacy.hh" #include #include @@ -47,38 +47,37 @@ ref ensureLocalStore() } -static Path useDeriver(Path path) +static StorePath useDeriver(const StorePath & path) { - if (isDerivation(path)) return path; - Path drvPath = store->queryPathInfo(path)->deriver; - if (drvPath == "") - throw Error(format("deriver of path ‘%1%’ is not known") % path); - return drvPath; + if (path.isDerivation()) return path.clone(); + auto info = store->queryPathInfo(path); + if (!info->deriver) + throw Error("deriver of path '%s' is not known", store->printStorePath(path)); + return info->deriver->clone(); } /* Realise the given path. For a derivation that means build it; for other paths it means ensure their validity. */ -static PathSet realisePath(Path path, bool build = true) +static PathSet realisePath(StorePathWithOutputs path, bool build = true) { - DrvPathWithOutputs p = parseDrvPathWithOutputs(path); - auto store2 = std::dynamic_pointer_cast(store); - if (isDerivation(p.first)) { + if (path.path.isDerivation()) { if (build) store->buildPaths({path}); - Derivation drv = store->derivationFromPath(p.first); + Derivation drv = store->derivationFromPath(path.path); rootNr++; - if (p.second.empty()) - for (auto & i : drv.outputs) p.second.insert(i.first); + if (path.outputs.empty()) + for (auto & i : drv.outputs) path.outputs.insert(i.first); PathSet outputs; - for (auto & j : p.second) { + for (auto & j : path.outputs) { DerivationOutputs::iterator i = drv.outputs.find(j); if (i == drv.outputs.end()) - throw Error(format("derivation ‘%1%’ does not have an output named ‘%2%’") % p.first % j); - Path outPath = i->second.path; + throw Error("derivation '%s' does not have an output named '%s'", + store2->printStorePath(path.path), j); + auto outPath = store2->printStorePath(i->second.path); if (store2) { if (gcRoot == "") printGCWarning(); @@ -86,7 +85,7 @@ static PathSet realisePath(Path path, bool build = true) Path rootName = gcRoot; if (rootNr > 1) rootName += "-" + std::to_string(rootNr); if (i->first != "out") rootName += "-" + i->first; - outPath = store2->addPermRoot(outPath, rootName, indirectRoot); + outPath = store2->addPermRoot(store->parseStorePath(outPath), rootName, indirectRoot); } } outputs.insert(outPath); @@ -95,8 +94,9 @@ static PathSet realisePath(Path path, bool build = true) } else { - if (build) store->ensurePath(path); - else if (!store->isValidPath(path)) throw Error(format("path ‘%1%’ does not exist and cannot be created") % path); + if (build) store->ensurePath(path.path); + else if (!store->isValidPath(path.path)) + throw Error("path '%s' does not exist and cannot be created", store->printStorePath(path.path)); if (store2) { if (gcRoot == "") printGCWarning(); @@ -104,10 +104,10 @@ static PathSet realisePath(Path path, bool build = true) Path rootName = gcRoot; rootNr++; if (rootNr > 1) rootName += "-" + std::to_string(rootNr); - path = store2->addPermRoot(path, rootName, indirectRoot); + return {store2->addPermRoot(path.path, rootName, indirectRoot)}; } } - return {path}; + return {store->printStorePath(path.path)}; } } @@ -123,43 +123,39 @@ static void opRealise(Strings opFlags, Strings opArgs) if (i == "--dry-run") dryRun = true; else if (i == "--repair") buildMode = bmRepair; else if (i == "--check") buildMode = bmCheck; - else if (i == "--hash") buildMode = bmHash; else if (i == "--ignore-unknown") ignoreUnknown = true; - else throw UsageError(format("unknown flag ‘%1%’") % i); + else throw UsageError("unknown flag '%1%'", i); - Paths paths; - for (auto & i : opArgs) { - DrvPathWithOutputs p = parseDrvPathWithOutputs(i); - paths.push_back(makeDrvPathWithOutputs(store->followLinksToStorePath(p.first), p.second)); - } + std::vector paths; + for (auto & i : opArgs) + paths.push_back(store->followLinksToStorePathWithOutputs(i)); unsigned long long downloadSize, narSize; - PathSet willBuild, willSubstitute, unknown; - store->queryMissing(PathSet(paths.begin(), paths.end()), - willBuild, willSubstitute, unknown, downloadSize, narSize); + StorePathSet willBuild, willSubstitute, unknown; + store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); if (ignoreUnknown) { - Paths paths2; + std::vector paths2; for (auto & i : paths) - if (unknown.find(i) == unknown.end()) paths2.push_back(i); - paths = paths2; - unknown = PathSet(); + if (!unknown.count(i.path)) paths2.push_back(i); + paths = std::move(paths2); + unknown = StorePathSet(); } - if (settings.get("print-missing", true)) + if (settings.printMissing) printMissing(ref(store), willBuild, willSubstitute, unknown, downloadSize, narSize); if (dryRun) return; /* Build all paths at the same time to exploit parallelism. */ - store->buildPaths(PathSet(paths.begin(), paths.end()), buildMode); + store->buildPaths(paths, buildMode); if (!ignoreUnknown) for (auto & i : paths) { - PathSet paths = realisePath(i, false); + auto paths2 = realisePath(i, false); if (!noOutput) - for (auto & j : paths) - cout << format("%1%\n") % j; + for (auto & j : paths2) + cout << fmt("%1%\n", j); } } @@ -170,7 +166,7 @@ static void opAdd(Strings opFlags, Strings opArgs) if (!opFlags.empty()) throw UsageError("unknown flag"); for (auto & i : opArgs) - cout << format("%1%\n") % store->addToStore(baseNameOf(i), i); + cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i))); } @@ -178,11 +174,11 @@ static void opAdd(Strings opFlags, Strings opArgs) store. */ static void opAddFixed(Strings opFlags, Strings opArgs) { - bool recursive = false; + auto recursive = FileIngestionMethod::Flat; for (auto & i : opFlags) - if (i == "--recursive") recursive = true; - else throw UsageError(format("unknown flag ‘%1%’") % i); + if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + else throw UsageError("unknown flag '%1%'", i); if (opArgs.empty()) throw UsageError("first argument must be hash algorithm"); @@ -191,79 +187,72 @@ static void opAddFixed(Strings opFlags, Strings opArgs) opArgs.pop_front(); for (auto & i : opArgs) - cout << format("%1%\n") % store->addToStore(baseNameOf(i), i, recursive, hashAlgo); + cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, recursive, hashAlgo))); } /* Hack to support caching in `nix-prefetch-url'. */ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { - bool recursive = false; + auto recursive = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") recursive = true; - else throw UsageError(format("unknown flag ‘%1%’") % i); + if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + else throw UsageError("unknown flag '%1%'", i); if (opArgs.size() != 3) - throw UsageError(format("‘--print-fixed-path’ requires three arguments")); + throw UsageError("'--print-fixed-path' requires three arguments"); Strings::iterator i = opArgs.begin(); HashType hashAlgo = parseHashType(*i++); string hash = *i++; string name = *i++; - cout << format("%1%\n") % - store->makeFixedOutputPath(recursive, parseHash16or32(hashAlgo, hash), name); + cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash(hash, hashAlgo), name))); } -static PathSet maybeUseOutputs(const Path & storePath, bool useOutput, bool forceRealise) +static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput, bool forceRealise) { if (forceRealise) realisePath(storePath); - if (useOutput && isDerivation(storePath)) { - Derivation drv = store->derivationFromPath(storePath); - PathSet outputs; + if (useOutput && storePath.isDerivation()) { + auto drv = store->derivationFromPath(storePath); + StorePathSet outputs; for (auto & i : drv.outputs) - outputs.insert(i.second.path); + outputs.insert(i.second.path.clone()); return outputs; } - else return {storePath}; + else return singleton(storePath.clone()); } /* Some code to print a tree representation of a derivation dependency graph. Topological sorting is used to keep the tree relatively flat. */ - -const string treeConn = "+---"; -const string treeLine = "| "; -const string treeNull = " "; - - -static void printTree(const Path & path, - const string & firstPad, const string & tailPad, PathSet & done) +static void printTree(const StorePath & path, + const string & firstPad, const string & tailPad, StorePathSet & done) { - if (done.find(path) != done.end()) { - cout << format("%1%%2% [...]\n") % firstPad % path; + if (!done.insert(path.clone()).second) { + cout << fmt("%s%s [...]\n", firstPad, store->printStorePath(path)); return; } - done.insert(path); - cout << format("%1%%2%\n") % firstPad % path; + cout << fmt("%s%s\n", firstPad, store->printStorePath(path)); - auto references = store->queryPathInfo(path)->references; + auto info = store->queryPathInfo(path); /* Topologically sort under the relation A < B iff A \in closure(B). That is, if derivation A is an (possibly indirect) input of B, then A is printed first. This has the effect of flattening the tree, preventing deeply nested structures. */ - Paths sorted = store->topoSortPaths(references); + auto sorted = store->topoSortPaths(info->references); reverse(sorted.begin(), sorted.end()); - for (auto i = sorted.begin(); i != sorted.end(); ++i) { - auto j = i; ++j; - printTree(*i, tailPad + treeConn, - j == sorted.end() ? tailPad + treeNull : tailPad + treeLine, + for (const auto &[n, i] : enumerate(sorted)) { + bool last = n + 1 == sorted.size(); + printTree(i, + tailPad + (last ? treeLast : treeConn), + tailPad + (last ? treeNull : treeLine), done); } } @@ -275,7 +264,7 @@ static void opQuery(Strings opFlags, Strings opArgs) enum QueryType { qDefault, qOutputs, qRequisites, qReferences, qReferrers , qReferrersClosure, qDeriver, qBinding, qHash, qSize - , qTree, qGraph, qXml, qResolve, qRoots }; + , qTree, qGraph, qGraphML, qResolve, qRoots }; QueryType query = qDefault; bool useOutput = false; bool includeOutputs = false; @@ -301,15 +290,15 @@ static void opQuery(Strings opFlags, Strings opArgs) else if (i == "--size") query = qSize; else if (i == "--tree") query = qTree; else if (i == "--graph") query = qGraph; - else if (i == "--xml") query = qXml; + else if (i == "--graphml") query = qGraphML; else if (i == "--resolve") query = qResolve; else if (i == "--roots") query = qRoots; else if (i == "--use-output" || i == "-u") useOutput = true; else if (i == "--force-realise" || i == "--force-realize" || i == "-f") forceRealise = true; else if (i == "--include-outputs") includeOutputs = true; - else throw UsageError(format("unknown flag ‘%1%’") % i); + else throw UsageError("unknown flag '%1%'", i); if (prev != qDefault && prev != query) - throw UsageError(format("query type ‘%1%’ conflicts with earlier flag") % i); + throw UsageError("query type '%1%' conflicts with earlier flag", i); } if (query == qDefault) query = qOutputs; @@ -320,11 +309,11 @@ static void opQuery(Strings opFlags, Strings opArgs) case qOutputs: { for (auto & i : opArgs) { - i = store->followLinksToStorePath(i); - if (forceRealise) realisePath(i); - Derivation drv = store->derivationFromPath(i); + auto i2 = store->followLinksToStorePath(i); + if (forceRealise) realisePath(i2); + Derivation drv = store->derivationFromPath(i2); for (auto & j : drv.outputs) - cout << format("%1%\n") % j.second.path; + cout << fmt("%1%\n", store->printStorePath(j.second.path)); } break; } @@ -333,105 +322,110 @@ static void opQuery(Strings opFlags, Strings opArgs) case qReferences: case qReferrers: case qReferrersClosure: { - PathSet paths; + StorePathSet paths; for (auto & i : opArgs) { - PathSet ps = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise); + auto ps = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise); for (auto & j : ps) { if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs); else if (query == qReferences) { for (auto & p : store->queryPathInfo(j)->references) - paths.insert(p); + paths.insert(p.clone()); + } + else if (query == qReferrers) { + StorePathSet tmp; + store->queryReferrers(j, tmp); + for (auto & i : tmp) + paths.insert(i.clone()); } - else if (query == qReferrers) store->queryReferrers(j, paths); else if (query == qReferrersClosure) store->computeFSClosure(j, paths, true); } } - Paths sorted = store->topoSortPaths(paths); - for (Paths::reverse_iterator i = sorted.rbegin(); + auto sorted = store->topoSortPaths(paths); + for (StorePaths::reverse_iterator i = sorted.rbegin(); i != sorted.rend(); ++i) - cout << format("%s\n") % *i; + cout << fmt("%s\n", store->printStorePath(*i)); break; } case qDeriver: for (auto & i : opArgs) { - Path deriver = store->queryPathInfo(store->followLinksToStorePath(i))->deriver; - cout << format("%1%\n") % - (deriver == "" ? "unknown-deriver" : deriver); + auto info = store->queryPathInfo(store->followLinksToStorePath(i)); + cout << fmt("%s\n", info->deriver ? store->printStorePath(*info->deriver) : "unknown-deriver"); } break; case qBinding: for (auto & i : opArgs) { - Path path = useDeriver(store->followLinksToStorePath(i)); + auto path = useDeriver(store->followLinksToStorePath(i)); Derivation drv = store->derivationFromPath(path); StringPairs::iterator j = drv.env.find(bindingName); if (j == drv.env.end()) - throw Error(format("derivation ‘%1%’ has no environment binding named ‘%2%’") - % path % bindingName); - cout << format("%1%\n") % j->second; + throw Error("derivation '%s' has no environment binding named '%s'", + store->printStorePath(path), bindingName); + cout << fmt("%s\n", j->second); } break; case qHash: case qSize: for (auto & i : opArgs) { - PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise); - for (auto & j : paths) { + for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) { auto info = store->queryPathInfo(j); if (query == qHash) { assert(info->narHash.type == htSHA256); - cout << format("sha256:%1%\n") % printHash32(info->narHash); + cout << fmt("%s\n", info->narHash.to_string(Base32, true)); } else if (query == qSize) - cout << format("%1%\n") % info->narSize; + cout << fmt("%d\n", info->narSize); } } break; case qTree: { - PathSet done; + StorePathSet done; for (auto & i : opArgs) printTree(store->followLinksToStorePath(i), "", "", done); break; } case qGraph: { - PathSet roots; - for (auto & i : opArgs) { - PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise); - roots.insert(paths.begin(), paths.end()); - } - printDotGraph(ref(store), roots); + StorePathSet roots; + for (auto & i : opArgs) + for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) + roots.insert(j.clone()); + printDotGraph(ref(store), std::move(roots)); break; } - case qXml: { - PathSet roots; - for (auto & i : opArgs) { - PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise); - roots.insert(paths.begin(), paths.end()); - } - printXmlGraph(ref(store), roots); + case qGraphML: { + StorePathSet roots; + for (auto & i : opArgs) + for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) + roots.insert(j.clone()); + printGraphML(ref(store), std::move(roots)); break; } case qResolve: { for (auto & i : opArgs) - cout << format("%1%\n") % store->followLinksToStorePath(i); + cout << fmt("%s\n", store->printStorePath(store->followLinksToStorePath(i))); break; } case qRoots: { - PathSet referrers; - for (auto & i : opArgs) { - store->computeFSClosure( - maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise), - referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); - } - Roots roots = store->findRoots(); - for (auto & i : roots) - if (referrers.find(i.second) != referrers.end()) - cout << format("%1%\n") % i.first; + StorePathSet args; + for (auto & i : opArgs) + for (auto & p : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) + args.insert(p.clone()); + + StorePathSet referrers; + store->computeFSClosure( + args, referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); + + Roots roots = store->findRoots(false); + for (auto & [target, links] : roots) + if (referrers.find(target) != referrers.end()) + for (auto & link : links) + cout << fmt("%1% -> %2%\n", link, store->printStorePath(target)); break; } @@ -441,27 +435,18 @@ static void opQuery(Strings opFlags, Strings opArgs) } -static string shellEscape(const string & s) -{ - string r; - for (auto & i : s) - if (i == '\'') r += "'\\''"; else r += i; - return r; -} - - static void opPrintEnv(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); - if (opArgs.size() != 1) throw UsageError("‘--print-env’ requires one derivation store path"); + if (opArgs.size() != 1) throw UsageError("'--print-env' requires one derivation store path"); Path drvPath = opArgs.front(); - Derivation drv = store->derivationFromPath(drvPath); + Derivation drv = store->derivationFromPath(store->parseStorePath(drvPath)); /* Print each environment variable in the derivation in a format - that can be sourced by the shell. */ + * that can be sourced by the shell. */ for (auto & i : drv.env) - cout << format("export %1%; %1%='%2%'\n") % i.first % shellEscape(i.second); + cout << format("export %1%; %1%=%2%\n") % i.first % shellEscape(i.second); /* Also output the arguments. This doesn't preserve whitespace in arguments. */ @@ -482,58 +467,12 @@ static void opReadLog(Strings opFlags, Strings opArgs) RunPager pager; - // FIXME: move getting logs into Store. - auto store2 = std::dynamic_pointer_cast(store); - if (!store2) throw Error(format("store ‘%s’ does not support reading logs") % store->getUri()); - for (auto & i : opArgs) { - Path path = useDeriver(store->followLinksToStorePath(i)); - - string baseName = baseNameOf(path); - bool found = false; - - for (int j = 0; j < 2; j++) { - - Path logPath = - j == 0 - ? (format("%1%/%2%/%3%/%4%") % store2->logDir % drvsLogDir % string(baseName, 0, 2) % string(baseName, 2)).str() - : (format("%1%/%2%/%3%") % store2->logDir % drvsLogDir % baseName).str(); - Path logBz2Path = logPath + ".bz2"; - - if (pathExists(logPath)) { - /* !!! Make this run in O(1) memory. */ - string log = readFile(logPath); - writeFull(STDOUT_FILENO, log); - found = true; - break; - } - - else if (pathExists(logBz2Path)) { - std::cout << *decompress("bzip2", readFile(logBz2Path)); - found = true; - break; - } - } - - if (!found) { - for (auto & i : settings.logServers) { - string prefix = i; - if (!prefix.empty() && prefix.back() != '/') prefix += '/'; - string url = prefix + baseName; - try { - string log = runProgram(CURL, true, {"--fail", "--location", "--silent", "--", url}); - std::cout << "(using build log from " << url << ")" << std::endl; - std::cout << log; - found = true; - break; - } catch (ExecError & e) { - /* Ignore errors from curl. FIXME: actually, might be - nice to print a warning on HTTP status != 404. */ - } - } - } - - if (!found) throw Error(format("build log of derivation ‘%1%’ is not available") % path); + auto path = store->followLinksToStorePath(i); + auto log = store->getBuildLog(path); + if (!log) + throw Error("build log of derivation '%s' is not available", store->printStorePath(path)); + std::cout << *log; } } @@ -541,11 +480,13 @@ static void opReadLog(Strings opFlags, Strings opArgs) static void opDumpDB(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); - if (!opArgs.empty()) - throw UsageError("no arguments expected"); - PathSet validPaths = store->queryAllValidPaths(); - for (auto & i : validPaths) - cout << store->makeValidityRegistration({i}, true, true); + if (!opArgs.empty()) { + for (auto & i : opArgs) + cout << store->makeValidityRegistration(singleton(store->followLinksToStorePath(i)), true, true); + } else { + for (auto & i : store->queryAllValidPaths()) + cout << store->makeValidityRegistration(singleton(i), true, true); + } } @@ -554,18 +495,18 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) ValidPathInfos infos; while (1) { - ValidPathInfo info = decodeValidPathInfo(cin, hashGiven); - if (info.path == "") break; - if (!store->isValidPath(info.path) || reregister) { + auto info = decodeValidPathInfo(*store, cin, hashGiven); + if (!info) break; + if (!store->isValidPath(info->path) || reregister) { /* !!! races */ if (canonicalise) - canonicalisePathMetaData(info.path, -1); + canonicalisePathMetaData(store->printStorePath(info->path), -1); if (!hashGiven) { - HashResult hash = hashPath(htSHA256, info.path); - info.narHash = hash.first; - info.narSize = hash.second; + HashResult hash = hashPath(htSHA256, store->printStorePath(info->path)); + info->narHash = hash.first; + info->narSize = hash.second; } - infos.push_back(info); + infos.push_back(std::move(*info)); } } @@ -590,7 +531,7 @@ static void opRegisterValidity(Strings opFlags, Strings opArgs) for (auto & i : opFlags) if (i == "--reregister") reregister = true; else if (i == "--hash-given") hashGiven = true; - else throw UsageError(format("unknown flag ‘%1%’") % i); + else throw UsageError("unknown flag '%1%'", i); if (!opArgs.empty()) throw UsageError("no arguments expected"); @@ -604,15 +545,15 @@ static void opCheckValidity(Strings opFlags, Strings opArgs) for (auto & i : opFlags) if (i == "--print-invalid") printInvalid = true; - else throw UsageError(format("unknown flag ‘%1%’") % i); + else throw UsageError("unknown flag '%1%'", i); for (auto & i : opArgs) { - Path path = store->followLinksToStorePath(i); + auto path = store->followLinksToStorePath(i); if (!store->isValidPath(path)) { if (printInvalid) - cout << format("%1%\n") % path; + cout << fmt("%s\n", store->printStorePath(path)); else - throw Error(format("path ‘%1%’ is not valid") % path); + throw Error("path '%s' is not valid", store->printStorePath(path)); } } } @@ -631,19 +572,23 @@ static void opGC(Strings opFlags, Strings opArgs) if (*i == "--print-roots") printRoots = true; else if (*i == "--print-live") options.action = GCOptions::gcReturnLive; else if (*i == "--print-dead") options.action = GCOptions::gcReturnDead; - else if (*i == "--delete") options.action = GCOptions::gcDeleteDead; else if (*i == "--max-freed") { long long maxFreed = getIntArg(*i, i, opFlags.end(), true); options.maxFreed = maxFreed >= 0 ? maxFreed : 0; } - else throw UsageError(format("bad sub-operation ‘%1%’ in GC") % *i); + else throw UsageError("bad sub-operation '%1%' in GC", *i); if (!opArgs.empty()) throw UsageError("no arguments expected"); if (printRoots) { - Roots roots = store->findRoots(); - for (auto & i : roots) - cout << i.first << " -> " << i.second << std::endl; + Roots roots = store->findRoots(false); + std::set> roots2; + // Transpose and sort the roots. + for (auto & [target, links] : roots) + for (auto & link : links) + roots2.emplace(link, target.clone()); + for (auto & [link, target] : roots2) + std::cout << link << " -> " << store->printStorePath(target) << "\n"; } else { @@ -667,7 +612,7 @@ static void opDelete(Strings opFlags, Strings opArgs) for (auto & i : opFlags) if (i == "--ignore-liveness") options.ignoreLiveness = true; - else throw UsageError(format("unknown flag ‘%1%’") % i); + else throw UsageError("unknown flag '%1%'", i); for (auto & i : opArgs) options.pathsToDelete.insert(store->followLinksToStorePath(i)); @@ -678,8 +623,7 @@ static void opDelete(Strings opFlags, Strings opArgs) } -/* Dump a path as a Nix archive. The archive is written to standard - output. */ +/* Dump a path as a Nix archive. The archive is written to stdout */ static void opDump(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); @@ -688,11 +632,11 @@ static void opDump(Strings opFlags, Strings opArgs) FdSink sink(STDOUT_FILENO); string path = *opArgs.begin(); dumpPath(path, sink); + sink.flush(); } -/* Restore a value from a Nix archive. The archive is read from - standard input. */ +/* Restore a value from a Nix archive. The archive is read from stdin. */ static void opRestore(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); @@ -706,25 +650,31 @@ static void opRestore(Strings opFlags, Strings opArgs) static void opExport(Strings opFlags, Strings opArgs) { for (auto & i : opFlags) - throw UsageError(format("unknown flag ‘%1%’") % i); + throw UsageError("unknown flag '%1%'", i); + + StorePathSet paths; + + for (auto & i : opArgs) + paths.insert(store->followLinksToStorePath(i)); FdSink sink(STDOUT_FILENO); - store->exportPaths(opArgs, sink); + store->exportPaths(paths, sink); + sink.flush(); } static void opImport(Strings opFlags, Strings opArgs) { for (auto & i : opFlags) - throw UsageError(format("unknown flag ‘%1%’") % i); + throw UsageError("unknown flag '%1%'", i); if (!opArgs.empty()) throw UsageError("no arguments expected"); FdSource source(STDIN_FILENO); - Paths paths = store->importPaths(source, 0); + auto paths = store->importPaths(source, nullptr, NoCheckSigs); for (auto & i : paths) - cout << format("%1%\n") % i << std::flush; + cout << fmt("%s\n", store->printStorePath(i)) << std::flush; } @@ -746,15 +696,18 @@ static void opVerify(Strings opFlags, Strings opArgs) throw UsageError("no arguments expected"); bool checkContents = false; - bool repair = false; + RepairFlag repair = NoRepair; for (auto & i : opFlags) if (i == "--check-contents") checkContents = true; - else if (i == "--repair") repair = true; - else throw UsageError(format("unknown flag ‘%1%’") % i); + else if (i == "--repair") repair = Repair; + else throw UsageError("unknown flag '%1%'", i); if (store->verifyStore(checkContents, repair)) { - printError("warning: not all errors were fixed"); + logWarning({ + .name = "Store consistency", + .description = "not all errors were fixed" + }); throw Exit(1); } } @@ -769,16 +722,21 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) int status = 0; for (auto & i : opArgs) { - Path path = store->followLinksToStorePath(i); - printMsg(lvlTalkative, format("checking path ‘%1%’...") % path); + auto path = store->followLinksToStorePath(i); + printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path)); auto info = store->queryPathInfo(path); HashSink sink(info->narHash.type); store->narFromPath(path, sink); auto current = sink.finish(); if (current.first != info->narHash) { - printError( - format("path ‘%1%’ was modified! expected hash ‘%2%’, got ‘%3%’") - % path % printHash(info->narHash) % printHash(current.first)); + logError({ + .name = "Hash mismatch", + .hint = hintfmt( + "path '%s' was modified! expected hash '%s', got '%s'", + store->printStorePath(path), + info->narHash.to_string(Base32, true), + current.first.to_string(Base32, true)) + }); status = 1; } } @@ -794,10 +752,8 @@ static void opRepairPath(Strings opFlags, Strings opArgs) if (!opFlags.empty()) throw UsageError("no flags expected"); - for (auto & i : opArgs) { - Path path = store->followLinksToStorePath(i); - ensureLocalStore()->repairPath(path); - } + for (auto & i : opArgs) + ensureLocalStore()->repairPath(store->followLinksToStorePath(i)); } /* Optimise the disk space usage of the Nix store by hard-linking @@ -816,7 +772,7 @@ static void opServe(Strings opFlags, Strings opArgs) bool writeAllowed = false; for (auto & i : opFlags) if (i == "--write") writeAllowed = true; - else throw UsageError(format("unknown flag ‘%1%’") % i); + else throw UsageError("unknown flag '%1%'", i); if (!opArgs.empty()) throw UsageError("no arguments expected"); @@ -839,11 +795,11 @@ static void opServe(Strings opFlags, Strings opArgs) settings.maxSilentTime = readInt(in); settings.buildTimeout = readInt(in); if (GET_PROTOCOL_MINOR(clientVersion) >= 2) - settings.maxLogSize = readInt(in); + settings.maxLogSize = readNum(in); if (GET_PROTOCOL_MINOR(clientVersion) >= 3) { - settings.set("build-repeat", std::to_string(readInt(in))); - settings.set("enforce-determinism", readInt(in) != 0 ? "true" : "false"); - settings.set("run-diff-hook", "true"); + settings.buildRepeat = readInt(in); + settings.enforceDeterminism = readInt(in); + settings.runDiffHook = true; } settings.printRepeatedBuilds = false; }; @@ -861,7 +817,7 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdQueryValidPaths: { bool lock = readInt(in); bool substitute = readInt(in); - PathSet paths = readStorePaths(*store, in); + auto paths = readStorePaths(*store, in); if (lock && writeAllowed) for (auto & path : paths) store->addTempRoot(path); @@ -871,37 +827,44 @@ static void opServe(Strings opFlags, Strings opArgs) flag. */ if (substitute && writeAllowed) { /* Filter out .drv files (we don't want to build anything). */ - PathSet paths2; + std::vector paths2; for (auto & path : paths) - if (!isDerivation(path)) paths2.insert(path); + if (!path.isDerivation()) + paths2.emplace_back(path.clone()); unsigned long long downloadSize, narSize; - PathSet willBuild, willSubstitute, unknown; - store->queryMissing(PathSet(paths2.begin(), paths2.end()), + StorePathSet willBuild, willSubstitute, unknown; + store->queryMissing(paths2, willBuild, willSubstitute, unknown, downloadSize, narSize); /* FIXME: should use ensurePath(), but it only does one path at a time. */ if (!willSubstitute.empty()) try { - store->buildPaths(willSubstitute); + std::vector subs; + for (auto & p : willSubstitute) subs.emplace_back(p.clone()); + store->buildPaths(subs); } catch (Error & e) { - printError(format("warning: %1%") % e.msg()); + logWarning(e.info()); } } - out << store->queryValidPaths(paths); + writeStorePaths(*store, out, store->queryValidPaths(paths)); break; } case cmdQueryPathInfos: { - PathSet paths = readStorePaths(*store, in); + auto paths = readStorePaths(*store, in); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { try { auto info = store->queryPathInfo(i); - out << info->path << info->deriver << info->references; + out << store->printStorePath(info->path) + << (info->deriver ? store->printStorePath(*info->deriver) : ""); + writeStorePaths(*store, out, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; + if (GET_PROTOCOL_MINOR(clientVersion) >= 4) + out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << info->ca << info->sigs; } catch (InvalidPath &) { } } @@ -910,28 +873,29 @@ static void opServe(Strings opFlags, Strings opArgs) } case cmdDumpStorePath: - dumpPath(readStorePath(*store, in), out); + store->narFromPath(store->parseStorePath(readString(in)), out); break; case cmdImportPaths: { if (!writeAllowed) throw Error("importing paths is not allowed"); - store->importPaths(in, 0, true); // FIXME: should we skip sig checking? + store->importPaths(in, nullptr, NoCheckSigs); // FIXME: should we skip sig checking? out << 1; // indicate success break; } case cmdExportPaths: { readInt(in); // obsolete - Paths sorted = store->topoSortPaths(readStorePaths(*store, in)); - reverse(sorted.begin(), sorted.end()); - store->exportPaths(sorted, out); + store->exportPaths(readStorePaths(*store, in), out); break; } - case cmdBuildPaths: { /* Used by build-remote.pl. */ + case cmdBuildPaths: { if (!writeAllowed) throw Error("building paths is not allowed"); - PathSet paths = readStorePaths(*store, in); + + std::vector paths; + for (auto & s : readStrings(in)) + paths.emplace_back(store->parsePathWithOutputs(s)); getBuildSettings(); @@ -950,7 +914,7 @@ static void opServe(Strings opFlags, Strings opArgs) if (!writeAllowed) throw Error("building paths is not allowed"); - Path drvPath = readStorePath(*store, in); // informational only + auto drvPath = store->parseStorePath(readString(in)); // informational only BasicDerivation drv; readDerivation(in, *store, drv); @@ -969,15 +933,44 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdQueryClosure: { bool includeOutputs = readInt(in); - PathSet closure; - store->computeFSClosure(readStorePaths(*store, in), + StorePathSet closure; + store->computeFSClosure(readStorePaths(*store, in), closure, false, includeOutputs); - out << closure; + writeStorePaths(*store, out, closure); + break; + } + + case cmdAddToStoreNar: { + if (!writeAllowed) throw Error("importing paths is not allowed"); + + auto path = readString(in); + ValidPathInfo info(store->parseStorePath(path)); + auto deriver = readString(in); + if (deriver != "") + info.deriver = store->parseStorePath(deriver); + info.narHash = Hash(readString(in), htSHA256); + info.references = readStorePaths(*store, in); + in >> info.registrationTime >> info.narSize >> info.ultimate; + info.sigs = readStrings(in); + in >> info.ca; + + if (info.narSize == 0) + throw Error("narInfo is too old and missing the narSize field"); + + SizedSource sizedSource(in, info.narSize); + + store->addToStore(info, sizedSource, NoRepair, NoCheckSigs); + + // consume all the data that has been sent before continuing. + sizedSource.drainAll(); + + out << 1; // indicate success + break; } default: - throw Error(format("unknown serve command %1%") % cmd); + throw Error("unknown serve command %1%", cmd); } out.flush(); @@ -988,7 +981,7 @@ static void opServe(Strings opFlags, Strings opArgs) static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs) { for (auto & i : opFlags) - throw UsageError(format("unknown flag ‘%1%’") % i); + throw UsageError("unknown flag '%1%'", i); if (opArgs.size() != 3) throw UsageError("three arguments expected"); auto i = opArgs.begin(); @@ -1023,11 +1016,9 @@ static void opVersion(Strings opFlags, Strings opArgs) /* Scan the arguments; find the operation, set global flags, put all other flags in a list, and put all other arguments in another list. */ -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - + { Strings opFlags, opArgs; Operation op = 0; @@ -1106,11 +1097,19 @@ int main(int argc, char * * argv) return true; }); + initPlugins(); + if (!op) throw UsageError("no operation specified"); if (op != opDump && op != opRestore) /* !!! hack */ store = openStore(); op(opFlags, opArgs); - }); + + logger->stop(); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-store", _main); diff --git a/src/nix-store/xmlgraph.cc b/src/nix-store/xmlgraph.cc deleted file mode 100644 index 0f7be7f7a02..00000000000 --- a/src/nix-store/xmlgraph.cc +++ /dev/null @@ -1,66 +0,0 @@ -#include "xmlgraph.hh" -#include "util.hh" -#include "store-api.hh" - -#include - - -using std::cout; - -namespace nix { - - -static inline const string & xmlQuote(const string & s) -{ - // Luckily, store paths shouldn't contain any character that needs to be - // quoted. - return s; -} - - -static string makeEdge(const string & src, const string & dst) -{ - format f = format(" \n") - % xmlQuote(src) % xmlQuote(dst); - return f.str(); -} - - -static string makeNode(const string & id) -{ - format f = format(" \n") % xmlQuote(id); - return f.str(); -} - - -void printXmlGraph(ref store, const PathSet & roots) -{ - PathSet workList(roots); - PathSet doneSet; - - cout << "\n" - << "\n"; - - while (!workList.empty()) { - Path path = *(workList.begin()); - workList.erase(path); - - if (doneSet.find(path) != doneSet.end()) continue; - doneSet.insert(path); - - cout << makeNode(path); - - for (auto & p : store->queryPathInfo(path)->references) { - if (p != path) { - workList.insert(p); - cout << makeEdge(p, path); - } - } - - } - - cout << "\n"; -} - - -} diff --git a/src/nix-store/xmlgraph.hh b/src/nix-store/xmlgraph.hh deleted file mode 100644 index a6e7d4e2805..00000000000 --- a/src/nix-store/xmlgraph.hh +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -#include "types.hh" - -namespace nix { - -class Store; - -void printXmlGraph(ref store, const PathSet & roots); - -} diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc new file mode 100644 index 00000000000..f43f774c1c8 --- /dev/null +++ b/src/nix/add-to-store.cc @@ -0,0 +1,62 @@ +#include "command.hh" +#include "common-args.hh" +#include "store-api.hh" +#include "archive.hh" + +using namespace nix; + +struct CmdAddToStore : MixDryRun, StoreCommand +{ + Path path; + std::optional namePart; + + CmdAddToStore() + { + expectArg("path", &path); + + addFlag({ + .longName = "name", + .shortName = 'n', + .description = "name component of the store path", + .labels = {"name"}, + .handler = {&namePart}, + }); + } + + std::string description() override + { + return "add a path to the Nix store"; + } + + Examples examples() override + { + return { + }; + } + + Category category() override { return catUtility; } + + void run(ref store) override + { + if (!namePart) namePart = baseNameOf(path); + + StringSink sink; + dumpPath(path, sink); + + auto narHash = hashString(htSHA256, *sink.s); + + ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart)); + info.narHash = narHash; + info.narSize = sink.s->size(); + info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); + + if (!dryRun) { + auto source = StringSource { *sink.s }; + store->addToStore(info, source); + } + + logger->stdout("%s", store->printStorePath(info.path)); + } +}; + +static auto r1 = registerCommand("add-to-store"); diff --git a/src/nix/build.cc b/src/nix/build.cc index 812464d7582..850e09ce843 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -1,20 +1,29 @@ #include "command.hh" #include "common-args.hh" -#include "installables.hh" #include "shared.hh" #include "store-api.hh" using namespace nix; -struct CmdBuild : StoreCommand, MixDryRun, MixInstallables +struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile { + Path outLink = "result"; + CmdBuild() { - } + addFlag({ + .longName = "out-link", + .shortName = 'o', + .description = "path of the symlink to the build result", + .labels = {"path"}, + .handler = {&outLink}, + }); - std::string name() override - { - return "build"; + addFlag({ + .longName = "no-link", + .description = "do not create a symlink to the build result", + .handler = {&outLink, Path("")}, + }); } std::string description() override @@ -22,25 +31,44 @@ struct CmdBuild : StoreCommand, MixDryRun, MixInstallables return "build a derivation or fetch a store path"; } + Examples examples() override + { + return { + Example{ + "To build and run GNU Hello from NixOS 17.03:", + "nix build -f channel:nixos-17.03 hello; ./result/bin/hello" + }, + Example{ + "To build the build.x86_64-linux attribute from release.nix:", + "nix build -f release.nix build.x86_64-linux" + }, + Example{ + "To make a profile point at GNU Hello:", + "nix build --profile /tmp/profile nixpkgs.hello" + }, + }; + } + void run(ref store) override { - auto elems = evalInstallables(store); + auto buildables = build(store, dryRun ? DryRun : Build, installables); - PathSet pathsToBuild; + if (dryRun) return; - for (auto & elem : elems) { - if (elem.isDrv) - pathsToBuild.insert(elem.drvPath); - else - pathsToBuild.insert(elem.outPaths.begin(), elem.outPaths.end()); + if (outLink != "") { + for (size_t i = 0; i < buildables.size(); ++i) { + for (auto & output : buildables[i].outputs) + if (auto store2 = store.dynamic_pointer_cast()) { + std::string symlink = outLink; + if (i) symlink += fmt("-%d", i); + if (output.first != "out") symlink += fmt("-%s", output.first); + store2->addPermRoot(output.second, absPath(symlink), true); + } + } } - printMissing(store, pathsToBuild); - - if (dryRun) return; - - store->buildPaths(pathsToBuild); + updateProfile(buildables); } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("build"); diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 2405a8cb44e..c82819af824 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -13,9 +13,9 @@ struct MixCat : virtual Args { auto st = accessor->stat(path); if (st.type == FSAccessor::Type::tMissing) - throw Error(format("path ‘%1%’ does not exist") % path); + throw Error("path '%1%' does not exist", path); if (st.type != FSAccessor::Type::tRegular) - throw Error(format("path ‘%1%’ is not a regular file") % path); + throw Error("path '%1%' is not a regular file", path); std::cout << accessor->readFile(path); } @@ -28,16 +28,13 @@ struct CmdCatStore : StoreCommand, MixCat expectArg("path", &path); } - std::string name() override - { - return "cat-store"; - } - std::string description() override { - return "print the contents of a store file on stdout"; + return "print the contents of a file in the Nix store on stdout"; } + Category category() override { return catUtility; } + void run(ref store) override { cat(store->getFSAccessor()); @@ -54,21 +51,18 @@ struct CmdCatNar : StoreCommand, MixCat expectArg("path", &path); } - std::string name() override - { - return "cat-nar"; - } - std::string description() override { - return "print the contents of a file inside a NAR file"; + return "print the contents of a file inside a NAR file on stdout"; } + Category category() override { return catUtility; } + void run(ref store) override { cat(makeNarAccessor(make_ref(readFile(narPath)))); } }; -static RegisterCommand r1(make_ref()); -static RegisterCommand r2(make_ref()); +static auto r1 = registerCommand("cat-store"); +static auto r2 = registerCommand("cat-nar"); diff --git a/src/nix/command.cc b/src/nix/command.cc index 5a8288da912..71b02771918 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -1,118 +1,198 @@ #include "command.hh" #include "store-api.hh" +#include "derivations.hh" +#include "nixexpr.hh" +#include "profiles.hh" + +extern char * * environ; namespace nix { -Commands * RegisterCommand::commands = 0; +Commands * RegisterCommand::commands = nullptr; -void Command::printHelp(const string & programName, std::ostream & out) +StoreCommand::StoreCommand() { - Args::printHelp(programName, out); - - auto exs = examples(); - if (!exs.empty()) { - out << "\n"; - out << "Examples:\n"; - for (auto & ex : exs) - out << "\n" - << " " << ex.description << "\n" // FIXME: wrap - << " $ " << ex.command << "\n"; - } } -MultiCommand::MultiCommand(const Commands & _commands) - : commands(_commands) +ref StoreCommand::getStore() { - expectedArgs.push_back(ExpectedArg{"command", 1, [=](Strings ss) { - assert(!command); - auto i = commands.find(ss.front()); - if (i == commands.end()) - throw UsageError(format("‘%1%’ is not a recognised command") % ss.front()); - command = i->second; - }}); + if (!_store) + _store = createStore(); + return ref(_store); } -void MultiCommand::printHelp(const string & programName, std::ostream & out) +ref StoreCommand::createStore() { - if (command) { - command->printHelp(programName + " " + command->name(), out); - return; - } + return openStore(); +} + +void StoreCommand::run() +{ + run(getStore()); +} + +StorePathsCommand::StorePathsCommand(bool recursive) + : recursive(recursive) +{ + if (recursive) + addFlag({ + .longName = "no-recursive", + .description = "apply operation to specified paths only", + .handler = {&this->recursive, false}, + }); + else + addFlag({ + .longName = "recursive", + .shortName = 'r', + .description = "apply operation to closure of the specified paths", + .handler = {&this->recursive, true}, + }); - out << "Usage: " << programName << " ... ...\n"; + mkFlag(0, "all", "apply operation to the entire store", &all); +} + +void StorePathsCommand::run(ref store) +{ + StorePaths storePaths; - out << "\n"; - out << "Common flags:\n"; - printFlags(out); + if (all) { + if (installables.size()) + throw UsageError("'--all' does not expect arguments"); + for (auto & p : store->queryAllValidPaths()) + storePaths.push_back(p.clone()); + } - out << "\n"; - out << "Available commands:\n"; + else { + for (auto & p : toStorePaths(store, realiseMode, installables)) + storePaths.push_back(p.clone()); - Table2 table; - for (auto & command : commands) - table.push_back(std::make_pair(command.second->name(), command.second->description())); - printTable(out, table); + if (recursive) { + StorePathSet closure; + store->computeFSClosure(storePathsToSet(storePaths), closure, false, false); + storePaths.clear(); + for (auto & p : closure) + storePaths.push_back(p.clone()); + } + } - out << "\n"; - out << "For full documentation, run ‘man " << programName << "’ or ‘man " << programName << "-’.\n"; + run(store, std::move(storePaths)); } -bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end) +void StorePathCommand::run(ref store) { - if (Args::processFlag(pos, end)) return true; - if (command && command->processFlag(pos, end)) return true; - return false; + auto storePaths = toStorePaths(store, NoBuild, installables); + + if (storePaths.size() != 1) + throw UsageError("this command requires exactly one store path"); + + run(store, *storePaths.begin()); } -bool MultiCommand::processArgs(const Strings & args, bool finish) +Strings editorFor(const Pos & pos) { - if (command) - return command->processArgs(args, finish); - else - return Args::processArgs(args, finish); + auto editor = getEnv("EDITOR").value_or("cat"); + auto args = tokenizeString(editor); + if (pos.line > 0 && ( + editor.find("emacs") != std::string::npos || + editor.find("nano") != std::string::npos || + editor.find("vim") != std::string::npos)) + args.push_back(fmt("+%d", pos.line)); + args.push_back(pos.file); + return args; } -StoreCommand::StoreCommand() +MixProfile::MixProfile() { - storeUri = getEnv("NIX_REMOTE"); + addFlag({ + .longName = "profile", + .description = "profile to update", + .labels = {"path"}, + .handler = {&profile}, + }); +} - mkFlag(0, "store", "store-uri", "URI of the Nix store to use", &storeUri); +void MixProfile::updateProfile(const StorePath & storePath) +{ + if (!profile) return; + auto store = getStore().dynamic_pointer_cast(); + if (!store) throw Error("'--profile' is not supported for this Nix store"); + auto profile2 = absPath(*profile); + switchLink(profile2, + createGeneration( + ref(store), + profile2, store->printStorePath(storePath))); } -void StoreCommand::run() +void MixProfile::updateProfile(const Buildables & buildables) { - run(openStore(storeUri)); + if (!profile) return; + + std::optional result; + + for (auto & buildable : buildables) { + for (auto & output : buildable.outputs) { + if (result) + throw Error("'--profile' requires that the arguments produce a single store path, but there are multiple"); + result = output.second.clone(); + } + } + + if (!result) + throw Error("'--profile' requires that the arguments produce a single store path, but there are none"); + + updateProfile(*result); } -StorePathsCommand::StorePathsCommand() +MixDefaultProfile::MixDefaultProfile() { - expectArgs("paths", &storePaths); - mkFlag('r', "recursive", "apply operation to closure of the specified paths", &recursive); - mkFlag(0, "all", "apply operation to the entire store", &all); + profile = getDefaultProfile(); } -void StorePathsCommand::run(ref store) +MixEnvironment::MixEnvironment() : ignoreEnvironment(false) { - if (all) { - if (storePaths.size()) - throw UsageError("‘--all’ does not expect arguments"); - for (auto & p : store->queryAllValidPaths()) - storePaths.push_back(p); - } + addFlag({ + .longName = "ignore-environment", + .shortName = 'i', + .description = "clear the entire environment (except those specified with --keep)", + .handler = {&ignoreEnvironment, true}, + }); + + addFlag({ + .longName = "keep", + .shortName = 'k', + .description = "keep specified environment variable", + .labels = {"name"}, + .handler = {[&](std::string s) { keep.insert(s); }}, + }); + + addFlag({ + .longName = "unset", + .shortName = 'u', + .description = "unset specified environment variable", + .labels = {"name"}, + .handler = {[&](std::string s) { unset.insert(s); }}, + }); +} - else { - for (auto & storePath : storePaths) - storePath = store->followLinksToStorePath(storePath); +void MixEnvironment::setEnviron() { + if (ignoreEnvironment) { + if (!unset.empty()) + throw UsageError("--unset does not make sense with --ignore-environment"); - if (recursive) { - PathSet closure; - store->computeFSClosure(PathSet(storePaths.begin(), storePaths.end()), - closure, false, false); - storePaths = Paths(closure.begin(), closure.end()); + for (const auto & var : keep) { + auto val = getenv(var.c_str()); + if (val) stringsEnv.emplace_back(fmt("%s=%s", var.c_str(), val)); } - } - run(store, storePaths); + vectorEnv = stringsToCharPtrs(stringsEnv); + environ = vectorEnv.data(); + } else { + if (!keep.empty()) + throw UsageError("--keep does not make sense without --ignore-environment"); + + for (const auto & var : unset) + unsetenv(var.c_str()); + } } } diff --git a/src/nix/command.hh b/src/nix/command.hh index a29cdcf7f50..959d5f19dbc 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -1,79 +1,125 @@ #pragma once +#include "installables.hh" #include "args.hh" +#include "common-eval-args.hh" +#include "path.hh" +#include "eval.hh" namespace nix { -/* A command is an argument parser that can be executed by calling its - run() method. */ -struct Command : virtual Args +extern std::string programPath; + +static constexpr Command::Category catSecondary = 100; +static constexpr Command::Category catUtility = 101; +static constexpr Command::Category catNixInstallation = 102; + +/* A command that requires a Nix store. */ +struct StoreCommand : virtual Command { - virtual std::string name() = 0; - virtual void prepare() { }; - virtual void run() = 0; + StoreCommand(); + void run() override; + ref getStore(); + virtual ref createStore(); + virtual void run(ref) = 0; - struct Example - { - std::string description; - std::string command; - }; +private: + std::shared_ptr _store; +}; - typedef std::list Examples; +struct SourceExprCommand : virtual StoreCommand, MixEvalArgs +{ + Path file; + + SourceExprCommand(); + + /* Return a value representing the Nix expression from which we + are installing. This is either the file specified by ‘--file’, + or an attribute set constructed from $NIX_PATH, e.g. ‘{ nixpkgs + = import ...; bla = import ...; }’. */ + Value * getSourceExpr(EvalState & state); - virtual Examples examples() { return Examples(); } + ref getEvalState(); + +private: - void printHelp(const string & programName, std::ostream & out) override; + std::shared_ptr evalState; + + RootValue vSourceExpr; }; -class Store; +enum RealiseMode { Build, NoBuild, DryRun }; -/* A command that require a Nix store. */ -struct StoreCommand : virtual Command +/* A command that operates on a list of "installables", which can be + store paths, attribute paths, Nix expressions, etc. */ +struct InstallablesCommand : virtual Args, SourceExprCommand { - std::string storeUri; - StoreCommand(); - void run() override; - virtual void run(ref) = 0; + std::vector> installables; + + InstallablesCommand() + { + expectArgs("installables", &_installables); + } + + void prepare() override; + + virtual bool useDefaultInstallables() { return true; } + +private: + + std::vector _installables; +}; + +/* A command that operates on exactly one "installable" */ +struct InstallableCommand : virtual Args, SourceExprCommand +{ + std::shared_ptr installable; + + InstallableCommand() + { + expectArg("installable", &_installable); + } + + void prepare() override; + +private: + + std::string _installable; }; /* A command that operates on zero or more store paths. */ -struct StorePathsCommand : public StoreCommand +struct StorePathsCommand : public InstallablesCommand { private: - Paths storePaths; bool recursive = false; bool all = false; +protected: + + RealiseMode realiseMode = NoBuild; + public: - StorePathsCommand(); + StorePathsCommand(bool recursive = false); using StoreCommand::run; - virtual void run(ref store, Paths storePaths) = 0; + virtual void run(ref store, std::vector storePaths) = 0; void run(ref store) override; -}; -typedef std::map> Commands; + bool useDefaultInstallables() override { return !all; } +}; -/* An argument parser that supports multiple subcommands, - i.e. ‘ ’. */ -class MultiCommand : virtual Args +/* A command that operates on exactly one store path. */ +struct StorePathCommand : public InstallablesCommand { -public: - Commands commands; - - std::shared_ptr command; - - MultiCommand(const Commands & commands); - - void printHelp(const string & programName, std::ostream & out) override; + using StoreCommand::run; - bool processFlag(Strings::iterator & pos, Strings::iterator end) override; + virtual void run(ref store, const StorePath & storePath) = 0; - bool processArgs(const Strings & args, bool finish) override; + void run(ref store) override; }; /* A helper class for registering commands globally. */ @@ -81,11 +127,71 @@ struct RegisterCommand { static Commands * commands; - RegisterCommand(ref command) + RegisterCommand(const std::string & name, + std::function()> command) { if (!commands) commands = new Commands; - commands->emplace(command->name(), command); + commands->emplace(name, command); } }; +template +static RegisterCommand registerCommand(const std::string & name) +{ + return RegisterCommand(name, [](){ return make_ref(); }); +} + +std::shared_ptr parseInstallable( + SourceExprCommand & cmd, ref store, const std::string & installable, + bool useDefaultInstallables); + +Buildables build(ref store, RealiseMode mode, + std::vector> installables); + +std::set toStorePaths(ref store, RealiseMode mode, + std::vector> installables); + +StorePath toStorePath(ref store, RealiseMode mode, + std::shared_ptr installable); + +std::set toDerivations(ref store, + std::vector> installables, + bool useDeriver = false); + +/* Helper function to generate args that invoke $EDITOR on + filename:lineno. */ +Strings editorFor(const Pos & pos); + +struct MixProfile : virtual StoreCommand +{ + std::optional profile; + + MixProfile(); + + /* If 'profile' is set, make it point at 'storePath'. */ + void updateProfile(const StorePath & storePath); + + /* If 'profile' is set, make it point at the store path produced + by 'buildables'. */ + void updateProfile(const Buildables & buildables); +}; + +struct MixDefaultProfile : MixProfile +{ + MixDefaultProfile(); +}; + +struct MixEnvironment : virtual Args { + + StringSet keep, unset; + Strings stringsEnv; + std::vector vectorEnv; + bool ignoreEnvironment; + + MixEnvironment(); + + /* Modify global environ based on ignoreEnvironment, keep, and unset. It's expected that exec will be called before this class goes out of scope, otherwise environ will become invalid. */ + void setEnviron(); +}; + } diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 976b0d3e0b8..c7c38709d19 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -12,15 +12,39 @@ struct CmdCopy : StorePathsCommand { std::string srcUri, dstUri; + CheckSigsFlag checkSigs = CheckSigs; + + SubstituteFlag substitute = NoSubstitute; + CmdCopy() + : StorePathsCommand(true) { - mkFlag(0, "from", "store-uri", "URI of the source Nix store", &srcUri); - mkFlag(0, "to", "store-uri", "URI of the destination Nix store", &dstUri); - } + addFlag({ + .longName = "from", + .description = "URI of the source Nix store", + .labels = {"store-uri"}, + .handler = {&srcUri}, + }); - std::string name() override - { - return "copy"; + addFlag({ + .longName = "to", + .description = "URI of the destination Nix store", + .labels = {"store-uri"}, + .handler = {&dstUri}, + }); + + addFlag({ + .longName = "no-check-sigs", + .description = "do not require that paths are signed by trusted keys", + .handler = {&checkSigs, NoCheckSigs}, + }); + + addFlag({ + .longName = "substitute-on-destination", + .shortName = 's', + .description = "whether to try substitutes on the destination store (only supported by SSH)", + .handler = {&substitute, Substitute}, + }); } std::string description() override @@ -32,22 +56,47 @@ struct CmdCopy : StorePathsCommand { return { Example{ - "To copy Firefox to the local store to a binary cache in file:///tmp/cache:", - "nix copy --to file:///tmp/cache -r $(type -p firefox)" + "To copy Firefox from the local store to a binary cache in file:///tmp/cache:", + "nix copy --to file:///tmp/cache $(type -p firefox)" + }, + Example{ + "To copy the entire current NixOS system closure to another machine via SSH:", + "nix copy --to ssh://server /run/current-system" + }, + Example{ + "To copy a closure from another machine via SSH:", + "nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2" + }, +#ifdef ENABLE_S3 + Example{ + "To copy Hello to an S3 binary cache:", + "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs.hello" + }, + Example{ + "To copy Hello to an S3-compatible binary cache:", + "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs.hello" }, +#endif }; } - void run(ref store, Paths storePaths) override + Category category() override { return catSecondary; } + + ref createStore() override + { + return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri); + } + + void run(ref srcStore, StorePaths storePaths) override { if (srcUri.empty() && dstUri.empty()) - throw UsageError("you must pass ‘--from’ and/or ‘--to’"); + throw UsageError("you must pass '--from' and/or '--to'"); - ref srcStore = srcUri.empty() ? store : openStore(srcUri); - ref dstStore = dstUri.empty() ? store : openStore(dstUri); + ref dstStore = dstUri.empty() ? openStore() : openStore(dstUri); - copyPaths(srcStore, dstStore, storePaths); + copyPaths(srcStore, dstStore, storePathsToSet(storePaths), + NoRepair, checkSigs, substitute); } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("copy"); diff --git a/src/nix/develop.cc b/src/nix/develop.cc new file mode 100644 index 00000000000..3045d7dc3a5 --- /dev/null +++ b/src/nix/develop.cc @@ -0,0 +1,344 @@ +#include "eval.hh" +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "derivations.hh" +#include "affinity.hh" +#include "progress-bar.hh" + +#include + +using namespace nix; + +struct Var +{ + bool exported = true; + bool associative = false; + std::string value; // quoted string or array +}; + +struct BuildEnvironment +{ + std::map env; + std::string bashFunctions; +}; + +BuildEnvironment readEnvironment(const Path & path) +{ + BuildEnvironment res; + + std::set exported; + + debug("reading environment file '%s'", path); + + auto file = readFile(path); + + auto pos = file.cbegin(); + + static std::string varNameRegex = + R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re"; + + static std::regex declareRegex( + "^declare -x (" + varNameRegex + ")" + + R"re((?:="((?:[^"\\]|\\.)*)")?\n)re"); + + static std::string simpleStringRegex = + R"re((?:[a-zA-Z0-9_/:\.\-\+=]*))re"; + + static std::string quotedStringRegex = + R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re"; + + static std::string indexedArrayRegex = + R"re((?:\(( *\[[0-9]+]="(?:[^"\\]|\\.)*")**\)))re"; + + static std::regex varRegex( + "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + indexedArrayRegex + ")\n"); + + /* Note: we distinguish between an indexed and associative array + using the space before the closing parenthesis. Will + undoubtedly regret this some day. */ + static std::regex assocArrayRegex( + "^(" + varNameRegex + ")=" + R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")* *\)))re" + "\n"); + + static std::regex functionRegex( + "^" + varNameRegex + " \\(\\) *\n"); + + while (pos != file.end()) { + + std::smatch match; + + if (std::regex_search(pos, file.cend(), match, declareRegex)) { + pos = match[0].second; + exported.insert(match[1]); + } + + else if (std::regex_search(pos, file.cend(), match, varRegex)) { + pos = match[0].second; + res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .value = match[2] }}); + } + + else if (std::regex_search(pos, file.cend(), match, assocArrayRegex)) { + pos = match[0].second; + res.env.insert({match[1], Var { .associative = true, .value = match[2] }}); + } + + else if (std::regex_search(pos, file.cend(), match, functionRegex)) { + res.bashFunctions = std::string(pos, file.cend()); + break; + } + + else throw Error("shell environment '%s' has unexpected line '%s'", + path, file.substr(pos - file.cbegin(), 60)); + } + + return res; +} + +const static std::string getEnvSh = + #include "get-env.sh.gen.hh" + ; + +/* Given an existing derivation, return the shell environment as + initialised by stdenv's setup script. We do this by building a + modified derivation with the same dependencies and nearly the same + initial environment variables, that just writes the resulting + environment to a file and exits. */ +StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) +{ + auto drv = store->derivationFromPath(drvPath); + + auto builder = baseNameOf(drv.builder); + if (builder != "bash") + throw Error("'nix develop' only works on derivations that use 'bash' as their builder"); + + auto getEnvShPath = store->addTextToStore("get-env.sh", getEnvSh, {}); + + drv.args = {store->printStorePath(getEnvShPath)}; + + /* Remove derivation checks. */ + drv.env.erase("allowedReferences"); + drv.env.erase("allowedRequisites"); + drv.env.erase("disallowedReferences"); + drv.env.erase("disallowedRequisites"); + + /* Rehash and write the derivation. FIXME: would be nice to use + 'buildDerivation', but that's privileged. */ + auto drvName = std::string(drvPath.name()); + assert(hasSuffix(drvName, ".drv")); + drvName.resize(drvName.size() - 4); + drvName += "-env"; + for (auto & output : drv.outputs) + drv.env.erase(output.first); + drv.env["out"] = ""; + drv.env["outputs"] = "out"; + drv.inputSrcs.insert(std::move(getEnvShPath)); + Hash h = hashDerivationModulo(*store, drv, true); + auto shellOutPath = store->makeOutputPath("out", h, drvName); + drv.outputs.insert_or_assign("out", DerivationOutput(shellOutPath.clone(), "", "")); + drv.env["out"] = store->printStorePath(shellOutPath); + auto shellDrvPath2 = writeDerivation(store, drv, drvName); + + /* Build the derivation. */ + store->buildPaths({shellDrvPath2}); + + assert(store->isValidPath(shellOutPath)); + + return shellOutPath; +} + +struct Common : InstallableCommand, MixProfile +{ + std::set ignoreVars{ + "BASHOPTS", + "EUID", + "HOME", // FIXME: don't ignore in pure mode? + "NIX_BUILD_TOP", + "NIX_ENFORCE_PURITY", + "NIX_LOG_FD", + "PPID", + "PWD", + "SHELLOPTS", + "SHLVL", + "SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt + "TEMP", + "TEMPDIR", + "TERM", + "TMP", + "TMPDIR", + "TZ", + "UID", + }; + + void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out) + { + out << "unset shellHook\n"; + + out << "nix_saved_PATH=\"$PATH\"\n"; + + for (auto & i : buildEnvironment.env) { + if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) { + if (i.second.associative) + out << fmt("declare -A %s=(%s)\n", i.first, i.second.value); + else { + out << fmt("%s=%s\n", i.first, i.second.value); + if (i.second.exported) + out << fmt("export %s\n", i.first); + } + } + } + + out << "PATH=\"$PATH:$nix_saved_PATH\"\n"; + + out << buildEnvironment.bashFunctions << "\n"; + + // FIXME: set outputs + + out << "export NIX_BUILD_TOP=\"$(mktemp -d --tmpdir nix-shell.XXXXXX)\"\n"; + for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) + out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i); + + out << "eval \"$shellHook\"\n"; + } + + StorePath getShellOutPath(ref store) + { + auto path = installable->getStorePath(); + if (path && hasSuffix(path->to_string(), "-env")) + return path->clone(); + else { + auto drvs = toDerivations(store, {installable}); + + if (drvs.size() != 1) + throw Error("'%s' needs to evaluate to a single derivation, but it evaluated to %d derivations", + installable->what(), drvs.size()); + + auto & drvPath = *drvs.begin(); + + return getDerivationEnvironment(store, drvPath); + } + } + + std::pair getBuildEnvironment(ref store) + { + auto shellOutPath = getShellOutPath(store); + + auto strPath = store->printStorePath(shellOutPath); + + updateProfile(shellOutPath); + + return {readEnvironment(strPath), strPath}; + } +}; + +struct CmdDevelop : Common, MixEnvironment +{ + std::vector command; + + CmdDevelop() + { + addFlag({ + .longName = "command", + .shortName = 'c', + .description = "command and arguments to be executed insted of an interactive shell", + .labels = {"command", "args"}, + .handler = {[&](std::vector ss) { + if (ss.empty()) throw UsageError("--command requires at least one argument"); + command = ss; + }} + }); + } + + std::string description() override + { + return "run a bash shell that provides the build environment of a derivation"; + } + + Examples examples() override + { + return { + Example{ + "To get the build environment of GNU hello:", + "nix develop nixpkgs.hello" + }, + Example{ + "To store the build environment in a profile:", + "nix develop --profile /tmp/my-shell nixpkgs.hello" + }, + Example{ + "To use a build environment previously recorded in a profile:", + "nix develop /tmp/my-shell" + }, + }; + } + + void run(ref store) override + { + auto [buildEnvironment, gcroot] = getBuildEnvironment(store); + + auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); + + std::ostringstream ss; + makeRcScript(buildEnvironment, ss); + + ss << fmt("rm -f '%s'\n", rcFilePath); + + if (!command.empty()) { + std::vector args; + for (auto s : command) + args.push_back(shellEscape(s)); + ss << fmt("exec %s\n", concatStringsSep(" ", args)); + } + + writeFull(rcFileFd.get(), ss.str()); + + stopProgressBar(); + + auto shell = getEnv("SHELL").value_or("bash"); + + setEnviron(); + // prevent garbage collection until shell exits + setenv("NIX_GCROOT", gcroot.data(), 1); + + auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; + + restoreAffinity(); + restoreSignals(); + + execvp(shell.c_str(), stringsToCharPtrs(args).data()); + + throw SysError("executing shell '%s'", shell); + } +}; + +struct CmdPrintDevEnv : Common +{ + std::string description() override + { + return "print shell code that can be sourced by bash to reproduce the build environment of a derivation"; + } + + Examples examples() override + { + return { + Example{ + "To apply the build environment of GNU hello to the current shell:", + ". <(nix print-dev-env nixpkgs.hello)" + }, + }; + } + + Category category() override { return catUtility; } + + void run(ref store) override + { + auto buildEnvironment = getBuildEnvironment(store).first; + + stopProgressBar(); + + makeRcScript(buildEnvironment, std::cout); + } +}; + +static auto r1 = registerCommand("print-dev-env"); +static auto r2 = registerCommand("develop"); diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc new file mode 100644 index 00000000000..82e92cdd0a9 --- /dev/null +++ b/src/nix/doctor.cc @@ -0,0 +1,136 @@ +#include + +#include "command.hh" +#include "logging.hh" +#include "serve-protocol.hh" +#include "shared.hh" +#include "store-api.hh" +#include "util.hh" +#include "worker-protocol.hh" + +using namespace nix; + +namespace { + +std::string formatProtocol(unsigned int proto) +{ + if (proto) { + auto major = GET_PROTOCOL_MAJOR(proto) >> 8; + auto minor = GET_PROTOCOL_MINOR(proto); + return (format("%1%.%2%") % major % minor).str(); + } + return "unknown"; +} + +bool checkPass(const std::string & msg) { + logger->log(ANSI_GREEN "[PASS] " ANSI_NORMAL + msg); + return true; +} + +bool checkFail(const std::string & msg) { + logger->log(ANSI_RED "[FAIL] " ANSI_NORMAL + msg); + return false; +} + +} + +struct CmdDoctor : StoreCommand +{ + bool success = true; + + std::string description() override + { + return "check your system for potential problems and print a PASS or FAIL for each check"; + } + + Category category() override { return catNixInstallation; } + + void run(ref store) override + { + logger->log("Running checks against store uri: " + store->getUri()); + + auto type = getStoreType(); + + if (type < tOther) { + success &= checkNixInPath(); + success &= checkProfileRoots(store); + } + success &= checkStoreProtocol(store->getProtocol()); + + if (!success) + throw Exit(2); + } + + bool checkNixInPath() + { + PathSet dirs; + + for (auto & dir : tokenizeString(getEnv("PATH").value_or(""), ":")) + if (pathExists(dir + "/nix-env")) + dirs.insert(dirOf(canonPath(dir + "/nix-env", true))); + + if (dirs.size() != 1) { + std::stringstream ss; + ss << "Multiple versions of nix found in PATH:\n"; + for (auto & dir : dirs) + ss << " " << dir << "\n"; + return checkFail(ss.str()); + } + + return checkPass("PATH contains only one nix version."); + } + + bool checkProfileRoots(ref store) + { + PathSet dirs; + + for (auto & dir : tokenizeString(getEnv("PATH").value_or(""), ":")) { + Path profileDir = dirOf(dir); + try { + Path userEnv = canonPath(profileDir, true); + + if (store->isStorePath(userEnv) && hasSuffix(userEnv, "user-environment")) { + while (profileDir.find("/profiles/") == std::string::npos && isLink(profileDir)) + profileDir = absPath(readLink(profileDir), dirOf(profileDir)); + + if (profileDir.find("/profiles/") == std::string::npos) + dirs.insert(dir); + } + } catch (SysError &) {} + } + + if (!dirs.empty()) { + std::stringstream ss; + ss << "Found profiles outside of " << settings.nixStateDir << "/profiles.\n" + << "The generation this profile points to might not have a gcroot and could be\n" + << "garbage collected, resulting in broken symlinks.\n\n"; + for (auto & dir : dirs) + ss << " " << dir << "\n"; + ss << "\n"; + return checkFail(ss.str()); + } + + return checkPass("All profiles are gcroots."); + } + + bool checkStoreProtocol(unsigned int storeProto) + { + unsigned int clientProto = GET_PROTOCOL_MAJOR(SERVE_PROTOCOL_VERSION) == GET_PROTOCOL_MAJOR(storeProto) + ? SERVE_PROTOCOL_VERSION + : PROTOCOL_VERSION; + + if (clientProto != storeProto) { + std::stringstream ss; + ss << "Warning: protocol version of this client does not match the store.\n" + << "While this is not necessarily a problem it's recommended to keep the client in\n" + << "sync with the daemon.\n\n" + << "Client protocol: " << formatProtocol(clientProto) << "\n" + << "Store protocol: " << formatProtocol(storeProto) << "\n\n"; + return checkFail(ss.str()); + } + + return checkPass("Client protocol matches store protocol."); + } +}; + +static auto r1 = registerCommand("doctor"); diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc new file mode 100644 index 00000000000..e1de71bf830 --- /dev/null +++ b/src/nix/dump-path.cc @@ -0,0 +1,33 @@ +#include "command.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdDumpPath : StorePathCommand +{ + std::string description() override + { + return "dump a store path to stdout (in NAR format)"; + } + + Examples examples() override + { + return { + Example{ + "To get a NAR from the binary cache https://cache.nixos.org/:", + "nix dump-path --store https://cache.nixos.org/ /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25" + }, + }; + } + + Category category() override { return catUtility; } + + void run(ref store, const StorePath & storePath) override + { + FdSink sink(STDOUT_FILENO); + store->narFromPath(storePath, sink); + sink.flush(); + } +}; + +static auto r1 = registerCommand("dump-path"); diff --git a/src/nix/edit.cc b/src/nix/edit.cc new file mode 100644 index 00000000000..067d3a9738b --- /dev/null +++ b/src/nix/edit.cc @@ -0,0 +1,56 @@ +#include "command.hh" +#include "shared.hh" +#include "eval.hh" +#include "attr-path.hh" +#include "progress-bar.hh" + +#include + +using namespace nix; + +struct CmdEdit : InstallableCommand +{ + std::string description() override + { + return "open the Nix expression of a Nix package in $EDITOR"; + } + + Examples examples() override + { + return { + Example{ + "To open the Nix expression of the GNU Hello package:", + "nix edit nixpkgs.hello" + }, + }; + } + + Category category() override { return catSecondary; } + + void run(ref store) override + { + auto state = getEvalState(); + + auto [v, pos] = installable->toValue(*state); + + try { + pos = findDerivationFilename(*state, *v, installable->what()); + } catch (NoPositionInfo &) { + } + + if (pos == noPos) + throw Error("cannot find position information for '%s", installable->what()); + + stopProgressBar(); + + auto args = editorFor(pos); + + execvp(args.front().c_str(), stringsToCharPtrs(args).data()); + + std::string command; + for (const auto &arg : args) command += " '" + arg + "'"; + throw SysError("cannot run command%s", command); + } +}; + +static auto r1 = registerCommand("edit"); diff --git a/src/nix/eval.cc b/src/nix/eval.cc new file mode 100644 index 00000000000..26e98ac2a0f --- /dev/null +++ b/src/nix/eval.cc @@ -0,0 +1,73 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "eval.hh" +#include "json.hh" +#include "value-to-json.hh" +#include "progress-bar.hh" + +using namespace nix; + +struct CmdEval : MixJSON, InstallableCommand +{ + bool raw = false; + + CmdEval() + { + mkFlag(0, "raw", "print strings unquoted", &raw); + } + + std::string description() override + { + return "evaluate a Nix expression"; + } + + Examples examples() override + { + return { + Example{ + "To evaluate a Nix expression given on the command line:", + "nix eval '(1 + 2)'" + }, + Example{ + "To evaluate a Nix expression from a file or URI:", + "nix eval -f channel:nixos-17.09 hello.name" + }, + Example{ + "To get the current version of Nixpkgs:", + "nix eval --raw nixpkgs.lib.version" + }, + Example{ + "To print the store path of the Hello package:", + "nix eval --raw nixpkgs.hello" + }, + }; + } + + Category category() override { return catSecondary; } + + void run(ref store) override + { + if (raw && json) + throw UsageError("--raw and --json are mutually exclusive"); + + auto state = getEvalState(); + + auto v = installable->toValue(*state).first; + PathSet context; + + if (raw) { + stopProgressBar(); + std::cout << state->coerceToString(noPos, *v, context); + } else if (json) { + JSONPlaceholder jsonOut(std::cout); + printValueAsJSON(*state, true, *v, jsonOut, context); + } else { + state->forceValueDeep(*v); + logger->stdout("%s", *v); + } + } +}; + +static auto r1 = registerCommand("eval"); diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh new file mode 100644 index 00000000000..a25ec43a9dc --- /dev/null +++ b/src/nix/get-env.sh @@ -0,0 +1,9 @@ +set -e +if [ -e .attrs.sh ]; then source .attrs.sh; fi +export IN_NIX_SHELL=impure +export dontAddDisableDepTrack=1 +if [[ -n $stdenv ]]; then + source $stdenv/setup +fi +export > $out +set >> $out diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 5dd891e8add..0dd4998d99e 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -2,88 +2,114 @@ #include "hash.hh" #include "legacy.hh" #include "shared.hh" +#include "references.hh" +#include "archive.hh" using namespace nix; struct CmdHash : Command { - enum Mode { mFile, mPath }; - Mode mode; - bool base32 = false; + FileIngestionMethod mode; + Base base = SRI; bool truncate = false; - HashType ht = htSHA512; - Strings paths; + HashType ht = htSHA256; + std::vector paths; + std::optional modulus; - CmdHash(Mode mode) : mode(mode) + CmdHash(FileIngestionMethod mode) : mode(mode) { - mkFlag(0, "base32", "print hash in base-32", &base32); - mkFlag(0, "base16", "print hash in base-16", &base32, false); - mkHashTypeFlag("type", &ht); + mkFlag(0, "sri", "print hash in SRI format", &base, SRI); + mkFlag(0, "base64", "print hash in base-64", &base, Base64); + mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32); + mkFlag(0, "base16", "print hash in base-16", &base, Base16); + addFlag(Flag::mkHashTypeFlag("type", &ht)); + #if 0 + mkFlag() + .longName("modulo") + .description("compute hash modulo specified string") + .labels({"modulus"}) + .dest(&modulus); + #endif expectArgs("paths", &paths); } - std::string name() override - { - return mode == mFile ? "hash-file" : "hash-path"; - } - std::string description() override { - return mode == mFile - ? "print cryptographic hash of a regular file" - : "print cryptographic hash of the NAR serialisation of a path"; + const char* d; + switch (mode) { + case FileIngestionMethod::Flat: + d = "print cryptographic hash of a regular file"; + case FileIngestionMethod::Recursive: + d = "print cryptographic hash of the NAR serialisation of a path"; + }; + return d; } + Category category() override { return catUtility; } + void run() override { for (auto path : paths) { - Hash h = mode == mFile ? hashFile(ht, path) : hashPath(ht, path).first; + + std::unique_ptr hashSink; + if (modulus) + hashSink = std::make_unique(ht, *modulus); + else + hashSink = std::make_unique(ht); + + switch (mode) { + case FileIngestionMethod::Flat: + readFile(path, *hashSink); + break; + case FileIngestionMethod::Recursive: + dumpPath(path, *hashSink); + break; + } + + Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); - std::cout << format("%1%\n") % - (base32 ? printHash32(h) : printHash(h)); + logger->stdout(h.to_string(base, base == SRI)); } } }; -static RegisterCommand r1(make_ref(CmdHash::mFile)); -static RegisterCommand r2(make_ref(CmdHash::mPath)); +static RegisterCommand r1("hash-file", [](){ return make_ref(FileIngestionMethod::Flat); }); +static RegisterCommand r2("hash-path", [](){ return make_ref(FileIngestionMethod::Recursive); }); struct CmdToBase : Command { - bool toBase32; - HashType ht = htSHA512; - Strings args; + Base base; + HashType ht = htUnknown; + std::vector args; - CmdToBase(bool toBase32) : toBase32(toBase32) + CmdToBase(Base base) : base(base) { - mkHashTypeFlag("type", &ht); + addFlag(Flag::mkHashTypeFlag("type", &ht)); expectArgs("strings", &args); } - std::string name() override - { - return toBase32 ? "to-base32" : "to-base16"; - } - std::string description() override { - return toBase32 - ? "convert a hash to base-32 representation" - : "convert a hash to base-16 representation"; + return fmt("convert a hash to %s representation", + base == Base16 ? "base-16" : + base == Base32 ? "base-32" : + base == Base64 ? "base-64" : + "SRI"); } + Category category() override { return catUtility; } + void run() override { - for (auto s : args) { - Hash h = parseHash16or32(ht, s); - std::cout << format("%1%\n") % - (toBase32 ? printHash32(h) : printHash(h)); - } + for (auto s : args) + logger->stdout(Hash(s, ht).to_string(base, base == SRI)); } }; -static RegisterCommand r3(make_ref(false)); -static RegisterCommand r4(make_ref(true)); +static RegisterCommand r3("to-base16", [](){ return make_ref(Base16); }); +static RegisterCommand r4("to-base32", [](){ return make_ref(Base32); }); +static RegisterCommand r5("to-base64", [](){ return make_ref(Base64); }); +static RegisterCommand r6("to-sri", [](){ return make_ref(SRI); }); /* Legacy nix-hash command. */ static int compatNixHash(int argc, char * * argv) @@ -93,7 +119,7 @@ static int compatNixHash(int argc, char * * argv) bool base32 = false; bool truncate = false; enum { opHash, opTo32, opTo16 } op = opHash; - Strings ss; + std::vector ss; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--help") @@ -107,7 +133,7 @@ static int compatNixHash(int argc, char * * argv) string s = getArg(*arg, arg, end); ht = parseHashType(s); if (ht == htUnknown) - throw UsageError(format("unknown hash type ‘%1%’") % s); + throw UsageError("unknown hash type '%1%'", s); } else if (*arg == "--to-base16") op = opTo16; else if (*arg == "--to-base32") op = opTo32; @@ -119,16 +145,16 @@ static int compatNixHash(int argc, char * * argv) }); if (op == opHash) { - CmdHash cmd(flat ? CmdHash::mFile : CmdHash::mPath); + CmdHash cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); cmd.ht = ht; - cmd.base32 = base32; + cmd.base = base32 ? Base32 : Base16; cmd.truncate = truncate; cmd.paths = ss; cmd.run(); } else { - CmdToBase cmd(op == opTo32); + CmdToBase cmd(op == opTo32 ? Base32 : Base16); cmd.args = ss; cmd.ht = ht; cmd.run(); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 8341bbc5a3a..937d692063b 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -1,22 +1,39 @@ +#include "command.hh" #include "attr-path.hh" -#include "common-opts.hh" +#include "common-eval-args.hh" #include "derivations.hh" #include "eval-inline.hh" #include "eval.hh" #include "get-drvs.hh" -#include "installables.hh" #include "store-api.hh" +#include "shared.hh" + +#include namespace nix { -Value * MixInstallables::buildSourceExpr(EvalState & state) + +SourceExprCommand::SourceExprCommand() +{ + addFlag({ + .longName = "file", + .shortName = 'f', + .description = "evaluate FILE rather than the default", + .labels = {"file"}, + .handler = {&file} + }); +} + +Value * SourceExprCommand::getSourceExpr(EvalState & state) { - Value * vRoot = state.allocValue(); + if (vSourceExpr) return *vSourceExpr; - if (file != "") { - Expr * e = state.parseExprFromFile(resolveExprPath(lookupFileArg(state, file))); - state.eval(e, *vRoot); - } + auto sToplevel = state.symbols.create("_toplevel"); + + vSourceExpr = allocRootValue(state.allocValue()); + + if (file != "") + state.evalFile(lookupFileArg(state, file), **vSourceExpr); else { @@ -24,84 +41,306 @@ Value * MixInstallables::buildSourceExpr(EvalState & state) auto searchPath = state.getSearchPath(); - state.mkAttrs(*vRoot, searchPath.size()); + state.mkAttrs(**vSourceExpr, 1024); + + mkBool(*state.allocAttr(**vSourceExpr, sToplevel), true); std::unordered_set seen; - for (auto & i : searchPath) { - if (i.first == "") continue; - if (seen.count(i.first)) continue; - seen.insert(i.first); - if (!pathExists(i.second)) continue; - mkApp(*state.allocAttr(*vRoot, state.symbols.create(i.first)), - state.getBuiltin("import"), - mkString(*state.allocValue(), i.second)); - } + auto addEntry = [&](const std::string & name) { + if (name == "") return; + if (!seen.insert(name).second) return; + Value * v1 = state.allocValue(); + mkPrimOpApp(*v1, state.getBuiltin("findFile"), state.getBuiltin("nixPath")); + Value * v2 = state.allocValue(); + mkApp(*v2, *v1, mkString(*state.allocValue(), name)); + mkApp(*state.allocAttr(**vSourceExpr, state.symbols.create(name)), + state.getBuiltin("import"), *v2); + }; + + for (auto & i : searchPath) + /* Hack to handle channels. */ + if (i.first.empty() && pathExists(i.second + "/manifest.nix")) { + for (auto & j : readDirectory(i.second)) + if (j.name != "manifest.nix" + && pathExists(fmt("%s/%s/default.nix", i.second, j.name))) + addEntry(j.name); + } else + addEntry(i.first); - vRoot->attrs->sort(); + (*vSourceExpr)->attrs->sort(); } - return vRoot; + return *vSourceExpr; } -UserEnvElems MixInstallables::evalInstallables(ref store) +ref SourceExprCommand::getEvalState() { - UserEnvElems res; + if (!evalState) + evalState = std::make_shared(searchPath, getStore()); + return ref(evalState); +} - for (auto & installable : installables) { +Buildable Installable::toBuildable() +{ + auto buildables = toBuildables(); + if (buildables.size() != 1) + throw Error("installable '%s' evaluates to %d derivations, where only one is expected", what(), buildables.size()); + return std::move(buildables[0]); +} - if (std::string(installable, 0, 1) == "/") { +struct InstallableStorePath : Installable +{ + ref store; + StorePath storePath; - if (store->isStorePath(installable)) { + InstallableStorePath(ref store, const Path & storePath) + : store(store), storePath(store->parseStorePath(storePath)) { } - if (isDerivation(installable)) { - UserEnvElem elem; - // FIXME: handle empty case, drop version - elem.attrPath = {storePathToName(installable)}; - elem.isDrv = true; - elem.drvPath = installable; - res.push_back(elem); - } + std::string what() override { return store->printStorePath(storePath); } - else { - UserEnvElem elem; - // FIXME: handle empty case, drop version - elem.attrPath = {storePathToName(installable)}; - elem.isDrv = false; - elem.outPaths = {installable}; - res.push_back(elem); - } - } + Buildables toBuildables() override + { + std::map outputs; + outputs.insert_or_assign("out", storePath.clone()); + Buildable b{ + .drvPath = storePath.isDerivation() ? storePath.clone() : std::optional(), + .outputs = std::move(outputs) + }; + Buildables bs; + bs.push_back(std::move(b)); + return bs; + } + + std::optional getStorePath() override + { + return storePath.clone(); + } +}; + +struct InstallableValue : Installable +{ + SourceExprCommand & cmd; + + InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { } + + Buildables toBuildables() override + { + auto state = cmd.getEvalState(); + + auto v = toValue(*state).first; + + Bindings & autoArgs = *cmd.getAutoArgs(*state); + + DrvInfos drvs; + getDerivations(*state, *v, "", autoArgs, drvs, false); + + Buildables res; + + StorePathSet drvPaths; + + for (auto & drv : drvs) { + Buildable b{.drvPath = state->store->parseStorePath(drv.queryDrvPath())}; + drvPaths.insert(b.drvPath->clone()); + + auto outputName = drv.queryOutputName(); + if (outputName == "") + throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(*b.drvPath)); + + b.outputs.emplace(outputName, state->store->parseStorePath(drv.queryOutPath())); + + res.push_back(std::move(b)); + } + + // Hack to recognize .all: if all drvs have the same drvPath, + // merge the buildables. + if (drvPaths.size() == 1) { + Buildable b{.drvPath = drvPaths.begin()->clone()}; + for (auto & b2 : res) + for (auto & output : b2.outputs) + b.outputs.insert_or_assign(output.first, output.second.clone()); + Buildables bs; + bs.push_back(std::move(b)); + return bs; + } else + return res; + } +}; + +struct InstallableExpr : InstallableValue +{ + std::string text; + + InstallableExpr(SourceExprCommand & cmd, const std::string & text) + : InstallableValue(cmd), text(text) { } + + std::string what() override { return text; } + + std::pair toValue(EvalState & state) override + { + auto v = state.allocValue(); + state.eval(state.parseExprFromString(text, absPath(".")), *v); + return {v, noPos}; + } +}; + +struct InstallableAttrPath : InstallableValue +{ + std::string attrPath; + + InstallableAttrPath(SourceExprCommand & cmd, const std::string & attrPath) + : InstallableValue(cmd), attrPath(attrPath) + { } - else - throw UsageError(format("don't know what to do with ‘%1%’") % installable); + std::string what() override { return attrPath; } + + std::pair toValue(EvalState & state) override + { + auto source = cmd.getSourceExpr(state); + + Bindings & autoArgs = *cmd.getAutoArgs(state); + + auto v = findAlongAttrPath(state, attrPath, autoArgs, *source).first; + state.forceValue(*v); + + return {v, noPos}; + } +}; + +// FIXME: extend +std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)"; +static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex)); + +static std::vector> parseInstallables( + SourceExprCommand & cmd, ref store, std::vector ss, bool useDefaultInstallables) +{ + std::vector> result; + + if (ss.empty() && useDefaultInstallables) { + if (cmd.file == "") + cmd.file = "."; + ss = {""}; + } + + for (auto & s : ss) { + + if (s.compare(0, 1, "(") == 0) + result.push_back(std::make_shared(cmd, s)); + + else if (s.find("/") != std::string::npos) { + + auto path = store->toStorePath(store->followLinksToStore(s)); + + if (store->isStorePath(path)) + result.push_back(std::make_shared(store, path)); } - else { + else if (s == "" || std::regex_match(s, attrPathRegex)) + result.push_back(std::make_shared(cmd, s)); - EvalState state({}, store); + else + throw UsageError("don't know what to do with argument '%s'", s); + } - auto vRoot = buildSourceExpr(state); + return result; +} - std::map autoArgs_; - Bindings & autoArgs(*evalAutoArgs(state, autoArgs_)); +std::shared_ptr parseInstallable( + SourceExprCommand & cmd, ref store, const std::string & installable, + bool useDefaultInstallables) +{ + auto installables = parseInstallables(cmd, store, {installable}, false); + assert(installables.size() == 1); + return installables.front(); +} + +Buildables build(ref store, RealiseMode mode, + std::vector> installables) +{ + if (mode != Build) + settings.readOnlyMode = true; - Value & v(*findAlongAttrPath(state, installable, autoArgs, *vRoot)); - state.forceValue(v); + Buildables buildables; - DrvInfos drvs; - getDerivations(state, v, "", autoArgs, drvs, false); + std::vector pathsToBuild; - for (auto & i : drvs) { - UserEnvElem elem; - elem.isDrv = true; - elem.drvPath = i.queryDrvPath(); - res.push_back(elem); - } + for (auto & i : installables) { + for (auto & b : i->toBuildables()) { + if (b.drvPath) { + StringSet outputNames; + for (auto & output : b.outputs) + outputNames.insert(output.first); + pathsToBuild.push_back({*b.drvPath, outputNames}); + } else + for (auto & output : b.outputs) + pathsToBuild.push_back({output.second.clone()}); + buildables.push_back(std::move(b)); } } - return res; + if (mode == DryRun) + printMissing(store, pathsToBuild, lvlError); + else if (mode == Build) + store->buildPaths(pathsToBuild); + + return buildables; +} + +StorePathSet toStorePaths(ref store, RealiseMode mode, + std::vector> installables) +{ + StorePathSet outPaths; + + for (auto & b : build(store, mode, installables)) + for (auto & output : b.outputs) + outPaths.insert(output.second.clone()); + + return outPaths; +} + +StorePath toStorePath(ref store, RealiseMode mode, + std::shared_ptr installable) +{ + auto paths = toStorePaths(store, mode, {installable}); + + if (paths.size() != 1) + throw Error("argument '%s' should evaluate to one store path", installable->what()); + + return paths.begin()->clone(); +} + +StorePathSet toDerivations(ref store, + std::vector> installables, bool useDeriver) +{ + StorePathSet drvPaths; + + for (auto & i : installables) + for (auto & b : i->toBuildables()) { + if (!b.drvPath) { + if (!useDeriver) + throw Error("argument '%s' did not evaluate to a derivation", i->what()); + for (auto & output : b.outputs) { + auto derivers = store->queryValidDerivers(output.second); + if (derivers.empty()) + throw Error("'%s' does not have a known deriver", i->what()); + // FIXME: use all derivers? + drvPaths.insert(derivers.begin()->clone()); + } + } else + drvPaths.insert(b.drvPath->clone()); + } + + return drvPaths; +} + +void InstallablesCommand::prepare() +{ + installables = parseInstallables(*this, getStore(), _installables, useDefaultInstallables()); +} + +void InstallableCommand::prepare() +{ + installable = parseInstallable(*this, getStore(), _installable, false); } } diff --git a/src/nix/installables.hh b/src/nix/installables.hh index a58f7dc59bb..503984220bc 100644 --- a/src/nix/installables.hh +++ b/src/nix/installables.hh @@ -1,48 +1,45 @@ #pragma once -#include "args.hh" +#include "util.hh" +#include "path.hh" +#include "eval.hh" + +#include namespace nix { -struct UserEnvElem +struct Buildable { - Strings attrPath; - - // FIXME: should use boost::variant or so. - bool isDrv; - - // Derivation case: - Path drvPath; - StringSet outputNames; - - // Non-derivation case: - PathSet outPaths; + std::optional drvPath; + std::map outputs; }; -typedef std::vector UserEnvElems; - -struct Value; -class EvalState; +typedef std::vector Buildables; -struct MixInstallables : virtual Args +struct Installable { - Strings installables; - Path file; + virtual ~Installable() { } - MixInstallables() + virtual std::string what() = 0; + + virtual Buildables toBuildables() { - mkFlag('f', "file", "file", "evaluate FILE rather than the default", &file); - expectArgs("installables", &installables); + throw Error("argument '%s' cannot be built", what()); } - UserEnvElems evalInstallables(ref store); + Buildable toBuildable(); - /* Return a value representing the Nix expression from which we - are installing. This is either the file specified by ‘--file’, - or an attribute set constructed from $NIX_PATH, e.g. ‘{ nixpkgs - = import ...; bla = import ...; }’. */ - Value * buildSourceExpr(EvalState & state); + virtual std::pair toValue(EvalState & state) + { + throw Error("argument '%s' cannot be evaluated", what()); + } + /* Return a value only if this installable is a store path or a + symlink to it. */ + virtual std::optional getStorePath() + { + return {}; + } }; } diff --git a/src/nix/local.mk b/src/nix/local.mk index f6e7073b6e7..43b7754e34b 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -2,8 +2,30 @@ programs += nix nix_DIR := $(d) -nix_SOURCES := $(wildcard $(d)/*.cc) +nix_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard src/build-remote/*.cc) \ + $(wildcard src/nix-build/*.cc) \ + $(wildcard src/nix-channel/*.cc) \ + $(wildcard src/nix-collect-garbage/*.cc) \ + $(wildcard src/nix-copy-closure/*.cc) \ + $(wildcard src/nix-daemon/*.cc) \ + $(wildcard src/nix-env/*.cc) \ + $(wildcard src/nix-instantiate/*.cc) \ + $(wildcard src/nix-prefetch-url/*.cc) \ + $(wildcard src/nix-store/*.cc) \ -nix_LIBS = libexpr libmain libstore libutil libformat +nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain -$(eval $(call install-symlink, nix, $(bindir)/nix-hash)) +nix_LIBS = libexpr libmain libfetchers libstore libutil libnixrust + +nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system + +$(foreach name, \ + nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ + $(eval $(call install-symlink, nix, $(bindir)/$(name)))) +$(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) + +src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh + +src/nix/develop.cc: src/nix/get-env.sh.gen.hh diff --git a/src/nix/log.cc b/src/nix/log.cc new file mode 100644 index 00000000000..3fe22f6c27a --- /dev/null +++ b/src/nix/log.cc @@ -0,0 +1,64 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "progress-bar.hh" + +using namespace nix; + +struct CmdLog : InstallableCommand +{ + std::string description() override + { + return "show the build log of the specified packages or paths, if available"; + } + + Examples examples() override + { + return { + Example{ + "To get the build log of GNU Hello:", + "nix log nixpkgs.hello" + }, + Example{ + "To get the build log of a specific path:", + "nix log /nix/store/lmngj4wcm9rkv3w4dfhzhcyij3195hiq-thunderbird-52.2.1" + }, + Example{ + "To get a build log from a specific binary cache:", + "nix log --store https://cache.nixos.org nixpkgs.hello" + }, + }; + } + + Category category() override { return catSecondary; } + + void run(ref store) override + { + settings.readOnlyMode = true; + + auto subs = getDefaultSubstituters(); + + subs.push_front(store); + + auto b = installable->toBuildable(); + + RunPager pager; + for (auto & sub : subs) { + auto log = b.drvPath ? sub->getBuildLog(*b.drvPath) : nullptr; + for (auto & output : b.outputs) { + if (log) break; + log = sub->getBuildLog(output.second); + } + if (!log) continue; + stopProgressBar(); + printInfo("got build log for '%s' from '%s'", installable->what(), sub->getUri()); + std::cout << *log; + return; + } + + throw Error("build log of '%s' is not available", installable->what()); + } +}; + +static auto r1 = registerCommand("log"); diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 3476dfb0528..d2157f2d449 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -2,10 +2,12 @@ #include "store-api.hh" #include "fs-accessor.hh" #include "nar-accessor.hh" +#include "common-args.hh" +#include "json.hh" using namespace nix; -struct MixLs : virtual Args +struct MixLs : virtual Args, MixJSON { std::string path; @@ -20,7 +22,7 @@ struct MixLs : virtual Args mkFlag('d', "directory", "show directories rather than their contents", &showDirectory); } - void list(ref accessor) + void listText(ref accessor) { std::function doPath; @@ -32,16 +34,14 @@ struct MixLs : virtual Args (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : st.type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : "dr-xr-xr-x"; - std::cout << - (format("%s %20d %s") % tp % st.fileSize % relPath); + auto line = fmt("%s %20d %s", tp, st.fileSize, relPath); if (st.type == FSAccessor::Type::tSymlink) - std::cout << " -> " << accessor->readLink(curPath) - ; - std::cout << "\n"; + line += " -> " + accessor->readLink(curPath); + logger->stdout(line); if (recursive && st.type == FSAccessor::Type::tDirectory) doPath(st, curPath, relPath, false); } else { - std::cout << relPath << "\n"; + logger->stdout(relPath); if (recursive) { auto st = accessor->stat(curPath); if (st.type == FSAccessor::Type::tDirectory) @@ -50,7 +50,7 @@ struct MixLs : virtual Args } }; - doPath = [&](const FSAccessor::Stat & st , const Path & curPath, + doPath = [&](const FSAccessor::Stat & st, const Path & curPath, const std::string & relPath, bool showDirectory) { if (st.type == FSAccessor::Type::tDirectory && !showDirectory) { @@ -63,11 +63,22 @@ struct MixLs : virtual Args auto st = accessor->stat(path); if (st.type == FSAccessor::Type::tMissing) - throw Error(format("path ‘%1%’ does not exist") % path); + throw Error("path '%1%' does not exist", path); doPath(st, path, - st.type == FSAccessor::Type::tDirectory ? "." : baseNameOf(path), + st.type == FSAccessor::Type::tDirectory ? "." : std::string(baseNameOf(path)), showDirectory); } + + void list(ref accessor) + { + if (path == "/") path = ""; + + if (json) { + JSONPlaceholder jsonRoot(std::cout); + listNar(jsonRoot, accessor, path, recursive); + } else + listText(accessor); + } }; struct CmdLsStore : StoreCommand, MixLs @@ -77,16 +88,23 @@ struct CmdLsStore : StoreCommand, MixLs expectArg("path", &path); } - std::string name() override + Examples examples() override { - return "ls-store"; + return { + Example{ + "To list the contents of a store path in a binary cache:", + "nix ls-store --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10" + }, + }; } std::string description() override { - return "show information about a store path"; + return "show information about a path in the Nix store"; } + Category category() override { return catUtility; } + void run(ref store) override { list(store->getFSAccessor()); @@ -103,21 +121,28 @@ struct CmdLsNar : Command, MixLs expectArg("path", &path); } - std::string name() override + Examples examples() override { - return "ls-nar"; + return { + Example{ + "To list a specific file in a NAR:", + "nix ls-nar -l hello.nar /bin/hello" + }, + }; } std::string description() override { - return "show information about the contents of a NAR file"; + return "show information about a path inside a NAR file"; } + Category category() override { return catUtility; } + void run() override { list(makeNarAccessor(make_ref(readFile(narPath)))); } }; -static RegisterCommand r1(make_ref()); -static RegisterCommand r2(make_ref()); +static auto r1 = registerCommand("ls-store"); +static auto r2 = registerCommand("ls-nar"); diff --git a/src/nix/main.cc b/src/nix/main.cc index 440ced97dfc..203901168b9 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -8,47 +8,203 @@ #include "shared.hh" #include "store-api.hh" #include "progress-bar.hh" +#include "filetransfer.hh" +#include "finally.hh" +#include "loggers.hh" + +#include +#include +#include +#include +#include + +extern std::string chrootHelperName; + +void chrootHelper(int argc, char * * argv); namespace nix { +/* Check if we have a non-loopback/link-local network interface. */ +static bool haveInternet() +{ + struct ifaddrs * addrs; + + if (getifaddrs(&addrs)) + return true; + + Finally free([&]() { freeifaddrs(addrs); }); + + for (auto i = addrs; i; i = i->ifa_next) { + if (!i->ifa_addr) continue; + if (i->ifa_addr->sa_family == AF_INET) { + if (ntohl(((sockaddr_in *) i->ifa_addr)->sin_addr.s_addr) != INADDR_LOOPBACK) { + return true; + } + } else if (i->ifa_addr->sa_family == AF_INET6) { + if (!IN6_IS_ADDR_LOOPBACK(&((sockaddr_in6 *) i->ifa_addr)->sin6_addr) && + !IN6_IS_ADDR_LINKLOCAL(&((sockaddr_in6 *) i->ifa_addr)->sin6_addr)) + return true; + } + } + + return false; +} + +std::string programPath; + struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { + bool printBuildLogs = false; + bool useNet = true; + bool refresh = false; + NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") { - mkFlag('h', "help", "show usage information", [=]() { - printHelp(programName, std::cout); - std::cout << "\nNote: this program is EXPERIMENTAL and subject to change.\n"; - throw Exit(); + categories.clear(); + categories[Command::catDefault] = "Main commands"; + categories[catSecondary] = "Infrequently used commands"; + categories[catUtility] = "Utility/scripting commands"; + categories[catNixInstallation] = "Commands for upgrading or troubleshooting your Nix installation"; + + addFlag({ + .longName = "help", + .description = "show usage information", + .handler = {[&]() { showHelpAndExit(); }}, + }); + + addFlag({ + .longName = "help-config", + .description = "show configuration options", + .handler = {[&]() { + std::cout << "The following configuration options are available:\n\n"; + Table2 tbl; + std::map settings; + globalConfig.getSettings(settings); + for (const auto & s : settings) + tbl.emplace_back(s.first, s.second.description); + printTable(std::cout, tbl); + throw Exit(); + }}, + }); + + addFlag({ + .longName = "print-build-logs", + .shortName = 'L', + .description = "print full build logs on stderr", + .handler = {[&]() {setLogFormat(LogFormat::barWithLogs); }}, + }); + + addFlag({ + .longName = "version", + .description = "show version information", + .handler = {[&]() { printVersion(programName); }}, + }); + + addFlag({ + .longName = "no-net", + .description = "disable substituters and consider all previously downloaded files up-to-date", + .handler = {[&]() { useNet = false; }}, + }); + + addFlag({ + .longName = "refresh", + .description = "consider all previously downloaded files out-of-date", + .handler = {[&]() { refresh = true; }}, }); - mkFlag(0, "version", "show version information", std::bind(printVersion, programName)); + deprecatedAliases.insert({"dev-shell", "develop"}); + } + + void printFlags(std::ostream & out) override + { + Args::printFlags(out); + std::cout << + "\n" + "In addition, most configuration settings can be overriden using '--" ANSI_ITALIC "name value" ANSI_NORMAL "'.\n" + "Boolean settings can be overriden using '--" ANSI_ITALIC "name" ANSI_NORMAL "' or '--no-" ANSI_ITALIC "name" ANSI_NORMAL "'. See 'nix\n" + "--help-config' for a list of configuration settings.\n"; + } + + void printHelp(const string & programName, std::ostream & out) override + { + MultiCommand::printHelp(programName, out); + +#if 0 + out << "\nFor full documentation, run 'man " << programName << "' or 'man " << programName << "-" ANSI_ITALIC "COMMAND" ANSI_NORMAL "'.\n"; +#endif + + std::cout << "\nNote: this program is " ANSI_RED "EXPERIMENTAL" ANSI_NORMAL " and subject to change.\n"; + } + + void showHelpAndExit() + { + printHelp(programName, std::cout); + throw Exit(); } }; void mainWrapped(int argc, char * * argv) { - settings.verboseBuild = false; + /* The chroot helper needs to be run before any threads have been + started. */ + if (argc > 0 && argv[0] == chrootHelperName) { + chrootHelper(argc, argv); + return; + } initNix(); initGC(); - string programName = baseNameOf(argv[0]); + programPath = argv[0]; + auto programName = std::string(baseNameOf(programPath)); { auto legacy = (*RegisterLegacyCommand::commands)[programName]; if (legacy) return legacy(argc, argv); } + verbosity = lvlWarn; + settings.verboseBuild = false; + + setLogFormat("bar"); + + Finally f([] { logger->stop(); }); + NixArgs args; args.parseCmdline(argvToStrings(argc, argv)); - assert(args.command); + initPlugins(); + + if (!args.command) args.showHelpAndExit(); + + if (args.command->first != "repl" + && args.command->first != "doctor" + && args.command->first != "upgrade-nix") + settings.requireExperimentalFeature("nix-command"); + + if (args.useNet && !haveInternet()) { + warn("you don't have Internet access; disabling some network-dependent features"); + args.useNet = false; + } + + if (!args.useNet) { + // FIXME: should check for command line overrides only. + if (!settings.useSubstitutes.overriden) + settings.useSubstitutes = false; + if (!settings.tarballTtl.overriden) + settings.tarballTtl = std::numeric_limits::max(); + if (!fileTransferSettings.tries.overriden) + fileTransferSettings.tries = 0; + if (!fileTransferSettings.connectTimeout.overriden) + fileTransferSettings.connectTimeout = 1; + } - StartProgressBar bar; + if (args.refresh) + settings.tarballTtl = 0; - args.command->prepare(); - args.command->run(); + args.command->second->prepare(); + args.command->second->run(); } } diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc new file mode 100644 index 00000000000..719ea4fd16d --- /dev/null +++ b/src/nix/make-content-addressable.cc @@ -0,0 +1,106 @@ +#include "command.hh" +#include "store-api.hh" +#include "references.hh" +#include "common-args.hh" +#include "json.hh" + +using namespace nix; + +struct CmdMakeContentAddressable : StorePathsCommand, MixJSON +{ + CmdMakeContentAddressable() + { + realiseMode = Build; + } + + std::string description() override + { + return "rewrite a path or closure to content-addressable form"; + } + + Examples examples() override + { + return { + Example{ + "To create a content-addressable representation of GNU Hello (but not its dependencies):", + "nix make-content-addressable nixpkgs.hello" + }, + Example{ + "To compute a content-addressable representation of the current NixOS system closure:", + "nix make-content-addressable -r /run/current-system" + }, + }; + } + + Category category() override { return catUtility; } + + void run(ref store, StorePaths storePaths) override + { + auto paths = store->topoSortPaths(storePathsToSet(storePaths)); + + std::reverse(paths.begin(), paths.end()); + + std::map remappings; + + auto jsonRoot = json ? std::make_unique(std::cout) : nullptr; + auto jsonRewrites = json ? std::make_unique(jsonRoot->object("rewrites")) : nullptr; + + for (auto & path : paths) { + auto pathS = store->printStorePath(path); + auto oldInfo = store->queryPathInfo(path); + auto oldHashPart = storePathToHash(pathS); + + StringSink sink; + store->narFromPath(path, sink); + + StringMap rewrites; + + StorePathSet references; + bool hasSelfReference = false; + for (auto & ref : oldInfo->references) { + if (ref == path) + hasSelfReference = true; + else { + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second.clone() : ref.clone(); + // FIXME: warn about unremapped paths? + if (replacement != ref) + rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); + references.insert(std::move(replacement)); + } + } + + *sink.s = rewriteStrings(*sink.s, rewrites); + + HashModuloSink hashModuloSink(htSHA256, oldHashPart); + hashModuloSink((unsigned char *) sink.s->data(), sink.s->size()); + + auto narHash = hashModuloSink.finish().first; + + ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference)); + info.references = std::move(references); + if (hasSelfReference) info.references.insert(info.path.clone()); + info.narHash = narHash; + info.narSize = sink.s->size(); + info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); + + if (!json) + printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); + + auto source = sinkToSource([&](Sink & nextSink) { + RewritingSink rsink2(oldHashPart, storePathToHash(store->printStorePath(info.path)), nextSink); + rsink2((unsigned char *) sink.s->data(), sink.s->size()); + rsink2.flush(); + }); + + store->addToStore(info, *source); + + if (json) + jsonRewrites->attr(store->printStorePath(path), store->printStorePath(info.path)); + + remappings.insert_or_assign(std::move(path), std::move(info.path)); + } + } +}; + +static auto r1 = registerCommand("make-content-addressable"); diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc new file mode 100644 index 00000000000..b4595187929 --- /dev/null +++ b/src/nix/optimise-store.cc @@ -0,0 +1,34 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" + +#include + +using namespace nix; + +struct CmdOptimiseStore : StoreCommand +{ + std::string description() override + { + return "replace identical files in the store by hard links"; + } + + Examples examples() override + { + return { + Example{ + "To optimise the Nix store:", + "nix optimise-store" + }, + }; + } + + Category category() override { return catUtility; } + + void run(ref store) override + { + store->optimiseStore(); + } +}; + +static auto r1 = registerCommand("optimise-store"); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index a9b33e1877d..88d7fffd445 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -2,30 +2,26 @@ #include "shared.hh" #include "store-api.hh" #include "json.hh" +#include "common-args.hh" -#include #include +#include using namespace nix; -struct CmdPathInfo : StorePathsCommand +struct CmdPathInfo : StorePathsCommand, MixJSON { bool showSize = false; bool showClosureSize = false; + bool humanReadable = false; bool showSigs = false; - bool json = false; CmdPathInfo() { mkFlag('s', "size", "print size of the NAR dump of each path", &showSize); mkFlag('S', "closure-size", "print sum size of the NAR dumps of the closure of each path", &showClosureSize); + mkFlag('h', "human-readable", "with -s and -S, print sizes like 1K 234M 5.67G etc.", &humanReadable); mkFlag(0, "sigs", "show signatures", &showSigs); - mkFlag(0, "json", "produce JSON output", &json); - } - - std::string name() override - { - return "path-info"; } std::string description() override @@ -33,6 +29,8 @@ struct CmdPathInfo : StorePathsCommand return "query information about store paths"; } + Category category() override { return catSecondary; } + Examples examples() override { return { @@ -40,6 +38,10 @@ struct CmdPathInfo : StorePathsCommand "To show the closure sizes of every path in the current NixOS system closure, sorted by size:", "nix path-info -rS /run/current-system | sort -nk2" }, + Example{ + "To show a package's closure size and all its dependencies with human readable sizes:", + "nix path-info -rsSh nixpkgs.rust" + }, Example{ "To check the existence of a path in a binary cache:", "nix path-info -r /nix/store/7qvk5c91...-geeqie-1.1 --store https://cache.nixos.org/" @@ -59,76 +61,55 @@ struct CmdPathInfo : StorePathsCommand }; } - void run(ref store, Paths storePaths) override + void printSize(unsigned long long value) + { + if (!humanReadable) { + std::cout << fmt("\t%11d", value); + return; + } + + static const std::array idents{{ + ' ', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y' + }}; + size_t power = 0; + double res = value; + while (res > 1024 && power < idents.size()) { + ++power; + res /= 1024; + } + std::cout << fmt("\t%6.1f%c", res, idents.at(power)); + } + + void run(ref store, StorePaths storePaths) override { size_t pathLen = 0; for (auto & storePath : storePaths) - pathLen = std::max(pathLen, storePath.size()); - - auto getClosureSize = [&](const Path & storePath) -> unsigned long long { - unsigned long long totalSize = 0; - PathSet closure; - store->computeFSClosure(storePath, closure, false, false); - for (auto & p : closure) - totalSize += store->queryPathInfo(p)->narSize; - return totalSize; - }; + pathLen = std::max(pathLen, store->printStorePath(storePath).size()); if (json) { - JSONList jsonRoot(std::cout, true); - - for (auto storePath : storePaths) { - auto info = store->queryPathInfo(storePath); - storePath = info->path; - - auto jsonPath = jsonRoot.object(); - jsonPath - .attr("path", storePath) - .attr("narHash", info->narHash.to_string()) - .attr("narSize", info->narSize); - - if (showClosureSize) - jsonPath.attr("closureSize", getClosureSize(storePath)); - - if (info->deriver != "") - jsonPath.attr("deriver", info->deriver); - - { - auto jsonRefs = jsonPath.list("references"); - for (auto & ref : info->references) - jsonRefs.elem(ref); - } - - if (info->registrationTime) - jsonPath.attr("registrationTime", info->registrationTime); - - if (info->ultimate) - jsonPath.attr("ultimate", info->ultimate); - - if (info->ca != "") - jsonPath.attr("ca", info->ca); - - if (!info->sigs.empty()) { - auto jsonSigs = jsonPath.list("signatures"); - for (auto & sig : info->sigs) - jsonSigs.elem(sig); - } - } + JSONPlaceholder jsonRoot(std::cout); + store->pathInfoToJSON(jsonRoot, + // FIXME: preserve order? + storePathsToSet(storePaths), + true, showClosureSize, SRI, AllowInvalid); } else { - for (auto storePath : storePaths) { + for (auto & storePath : storePaths) { auto info = store->queryPathInfo(storePath); - storePath = info->path; // FIXME: screws up padding + auto storePathS = store->printStorePath(storePath); + + std::cout << storePathS; - std::cout << storePath << std::string(std::max(0, (int) pathLen - (int) storePath.size()), ' '); + if (showSize || showClosureSize || showSigs) + std::cout << std::string(std::max(0, (int) pathLen - (int) storePathS.size()), ' '); if (showSize) - std::cout << '\t' << std::setw(11) << info->narSize; + printSize(info->narSize); if (showClosureSize) - std::cout << '\t' << std::setw(11) << getClosureSize(storePath); + printSize(store->getClosureSize(info->path).first); if (showSigs) { std::cout << '\t'; @@ -143,8 +124,7 @@ struct CmdPathInfo : StorePathsCommand } } - } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("path-info"); diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc new file mode 100644 index 00000000000..127397a292c --- /dev/null +++ b/src/nix/ping-store.cc @@ -0,0 +1,32 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdPingStore : StoreCommand +{ + std::string description() override + { + return "test whether a store can be opened"; + } + + Examples examples() override + { + return { + Example{ + "To test whether connecting to a remote Nix store via SSH works:", + "nix ping-store --store ssh://mac1" + }, + }; + } + + Category category() override { return catUtility; } + + void run(ref store) override + { + store->connect(); + } +}; + +static auto r1 = registerCommand("ping-store"); diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc deleted file mode 100644 index 69811b28280..00000000000 --- a/src/nix/progress-bar.cc +++ /dev/null @@ -1,157 +0,0 @@ -#include "progress-bar.hh" -#include "util.hh" -#include "sync.hh" - -#include - -namespace nix { - -class ProgressBar : public Logger -{ -private: - - struct ActInfo - { - Activity * activity; - Verbosity lvl; - std::string s; - }; - - struct Progress - { - uint64_t expected = 0, progress = 0; - }; - - struct State - { - std::list activities; - std::map::iterator> its; - std::map progress; - }; - - Sync state_; - -public: - - ~ProgressBar() - { - auto state(state_.lock()); - assert(state->activities.empty()); - writeToStderr("\r\e[K"); - } - - void log(Verbosity lvl, const FormatOrString & fs) override - { - auto state(state_.lock()); - log(*state, lvl, fs.s); - } - - void log(State & state, Verbosity lvl, const std::string & s) - { - writeToStderr("\r\e[K" + s + "\n"); - update(state); - } - - void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override - { - if (lvl > verbosity) return; - auto state(state_.lock()); - state->activities.emplace_back(ActInfo{&activity, lvl, fs.s}); - state->its.emplace(&activity, std::prev(state->activities.end())); - update(*state); - } - - void stopActivity(Activity & activity) override - { - auto state(state_.lock()); - auto i = state->its.find(&activity); - if (i == state->its.end()) return; - state->activities.erase(i->second); - state->its.erase(i); - update(*state); - } - - void setExpected(const std::string & label, uint64_t value) override - { - auto state(state_.lock()); - state->progress[label].expected = value; - } - - void setProgress(const std::string & label, uint64_t value) override - { - auto state(state_.lock()); - state->progress[label].progress = value; - } - - void incExpected(const std::string & label, uint64_t value) override - { - auto state(state_.lock()); - state->progress[label].expected += value; - } - - void incProgress(const std::string & label, uint64_t value) override - { - auto state(state_.lock()); - state->progress[label].progress += value; - } - - void update() - { - auto state(state_.lock()); - } - - void update(State & state) - { - std::string line = "\r"; - - std::string status = getStatus(state); - if (!status.empty()) { - line += '['; - line += status; - line += "]"; - } - - if (!state.activities.empty()) { - if (!status.empty()) line += " "; - line += state.activities.rbegin()->s; - } - - line += "\e[K"; - writeToStderr(line); - } - - std::string getStatus(State & state) - { - std::string res; - for (auto & p : state.progress) - if (p.second.expected || p.second.progress) { - if (!res.empty()) res += ", "; - res += std::to_string(p.second.progress); - if (p.second.expected) { - res += "/"; - res += std::to_string(p.second.expected); - } - res += " "; res += p.first; - } - return res; - } -}; - -StartProgressBar::StartProgressBar() -{ - if (isatty(STDERR_FILENO)) { - prev = logger; - logger = new ProgressBar(); - } -} - -StartProgressBar::~StartProgressBar() -{ - if (prev) { - auto bar = logger; - logger = prev; - delete bar; - } -} - -} diff --git a/src/nix/progress-bar.hh b/src/nix/progress-bar.hh deleted file mode 100644 index d2e44f7c4fd..00000000000 --- a/src/nix/progress-bar.hh +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include "logging.hh" - -namespace nix { - -class StartProgressBar -{ - Logger * prev = 0; -public: - StartProgressBar(); - ~StartProgressBar(); -}; - -} diff --git a/src/nix/repl.cc b/src/nix/repl.cc new file mode 100644 index 00000000000..4bcaaeebf8d --- /dev/null +++ b/src/nix/repl.cc @@ -0,0 +1,798 @@ +#include +#include +#include +#include + +#include + +#ifdef READLINE +#include +#include +#else +// editline < 1.15.2 don't wrap their API for C++ usage +// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). +// This results in linker errors due to to name-mangling of editline C symbols. +// For compatibility with these versions, we wrap the API here +// (wrapping multiple times on newer versions is no problem). +extern "C" { +#include +} +#endif + +#include "shared.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "attr-path.hh" +#include "store-api.hh" +#include "common-eval-args.hh" +#include "get-drvs.hh" +#include "derivations.hh" +#include "affinity.hh" +#include "globals.hh" +#include "command.hh" +#include "finally.hh" + +#define GC_INCLUDE_NEW +#include + +namespace nix { + +#define ESC_RED "\033[31m" +#define ESC_GRE "\033[32m" +#define ESC_YEL "\033[33m" +#define ESC_BLU "\033[34;1m" +#define ESC_MAG "\033[35m" +#define ESC_CYA "\033[36m" +#define ESC_END "\033[0m" + +struct NixRepl : gc +{ + string curDir; + std::unique_ptr state; + Bindings * autoArgs; + + Strings loadedFiles; + + const static int envSize = 32768; + StaticEnv staticEnv; + Env * env; + int displ; + StringSet varNames; + + const Path historyFile; + + NixRepl(const Strings & searchPath, nix::ref store); + ~NixRepl(); + void mainLoop(const std::vector & files); + StringSet completePrefix(string prefix); + bool getLine(string & input, const std::string &prompt); + Path getDerivationPath(Value & v); + bool processLine(string line); + void loadFile(const Path & path); + void initEnv(); + void reloadFiles(); + void addAttrsToScope(Value & attrs); + void addVarToScope(const Symbol & name, Value & v); + Expr * parseString(string s); + void evalString(string s, Value & v); + + typedef set ValuesSeen; + std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth); + std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen); +}; + + +string removeWhitespace(string s) +{ + s = chomp(s); + size_t n = s.find_first_not_of(" \n\r\t"); + if (n != string::npos) s = string(s, n); + return s; +} + + +NixRepl::NixRepl(const Strings & searchPath, nix::ref store) + : state(std::make_unique(searchPath, store)) + , staticEnv(false, &state->staticBaseEnv) + , historyFile(getDataDir() + "/nix/repl-history") +{ + curDir = absPath("."); +} + + +NixRepl::~NixRepl() +{ + write_history(historyFile.c_str()); +} + +static NixRepl * curRepl; // ugly + +static char * completionCallback(char * s, int *match) { + auto possible = curRepl->completePrefix(s); + if (possible.size() == 1) { + *match = 1; + auto *res = strdup(possible.begin()->c_str() + strlen(s)); + if (!res) throw Error("allocation failure"); + return res; + } else if (possible.size() > 1) { + auto checkAllHaveSameAt = [&](size_t pos) { + auto &first = *possible.begin(); + for (auto &p : possible) { + if (p.size() <= pos || p[pos] != first[pos]) + return false; + } + return true; + }; + size_t start = strlen(s); + size_t len = 0; + while (checkAllHaveSameAt(start + len)) ++len; + if (len > 0) { + *match = 1; + auto *res = strdup(std::string(*possible.begin(), start, len).c_str()); + if (!res) throw Error("allocation failure"); + return res; + } + } + + *match = 0; + return nullptr; +} + +static int listPossibleCallback(char *s, char ***avp) { + auto possible = curRepl->completePrefix(s); + + if (possible.size() > (INT_MAX / sizeof(char*))) + throw Error("too many completions"); + + int ac = 0; + char **vp = nullptr; + + auto check = [&](auto *p) { + if (!p) { + if (vp) { + while (--ac >= 0) + free(vp[ac]); + free(vp); + } + throw Error("allocation failure"); + } + return p; + }; + + vp = check((char **)malloc(possible.size() * sizeof(char*))); + + for (auto & p : possible) + vp[ac++] = check(strdup(p.c_str())); + + *avp = vp; + + return ac; +} + +namespace { + // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. + volatile sig_atomic_t g_signal_received = 0; + + void sigintHandler(int signo) { + g_signal_received = signo; + } +} + +void NixRepl::mainLoop(const std::vector & files) +{ + string error = ANSI_RED "error:" ANSI_NORMAL " "; + std::cout << "Welcome to Nix version " << nixVersion << ". Type :? for help." << std::endl << std::endl; + + for (auto & i : files) + loadedFiles.push_back(i); + + reloadFiles(); + if (!loadedFiles.empty()) std::cout << std::endl; + + // Allow nix-repl specific settings in .inputrc + rl_readline_name = "nix-repl"; + createDirs(dirOf(historyFile)); +#ifndef READLINE + el_hist_size = 1000; +#endif + read_history(historyFile.c_str()); + curRepl = this; +#ifndef READLINE + rl_set_complete_func(completionCallback); + rl_set_list_possib_func(listPossibleCallback); +#endif + + std::string input; + + while (true) { + // When continuing input from previous lines, don't print a prompt, just align to the same + // number of chars as the prompt. + if (!getLine(input, input.empty() ? "nix-repl> " : " ")) + break; + + try { + if (!removeWhitespace(input).empty() && !processLine(input)) return; + } catch (ParseError & e) { + if (e.msg().find("unexpected $end") != std::string::npos) { + // For parse errors on incomplete input, we continue waiting for the next line of + // input without clearing the input so far. + continue; + } else { + printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg()); + } + } catch (Error & e) { + printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg()); + } catch (Interrupted & e) { + printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg()); + } + + // We handled the current input fully, so we should clear it + // and read brand new input. + input.clear(); + std::cout << std::endl; + } +} + + +bool NixRepl::getLine(string & input, const std::string &prompt) +{ + struct sigaction act, old; + sigset_t savedSignalMask, set; + + auto setupSignals = [&]() { + act.sa_handler = sigintHandler; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGINT, &act, &old)) + throw SysError("installing handler for SIGINT"); + + sigemptyset(&set); + sigaddset(&set, SIGINT); + if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) + throw SysError("unblocking SIGINT"); + }; + auto restoreSignals = [&]() { + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); + + if (sigaction(SIGINT, &old, 0)) + throw SysError("restoring handler for SIGINT"); + }; + + setupSignals(); + char * s = readline(prompt.c_str()); + Finally doFree([&]() { free(s); }); + restoreSignals(); + + if (g_signal_received) { + g_signal_received = 0; + input.clear(); + return true; + } + + if (!s) + return false; + input += s; + input += '\n'; + return true; +} + + +StringSet NixRepl::completePrefix(string prefix) +{ + StringSet completions; + + size_t start = prefix.find_last_of(" \n\r\t(){}[]"); + std::string prev, cur; + if (start == std::string::npos) { + prev = ""; + cur = prefix; + } else { + prev = std::string(prefix, 0, start + 1); + cur = std::string(prefix, start + 1); + } + + size_t slash, dot; + + if ((slash = cur.rfind('/')) != string::npos) { + try { + auto dir = std::string(cur, 0, slash); + auto prefix2 = std::string(cur, slash + 1); + for (auto & entry : readDirectory(dir == "" ? "/" : dir)) { + if (entry.name[0] != '.' && hasPrefix(entry.name, prefix2)) + completions.insert(prev + dir + "/" + entry.name); + } + } catch (Error &) { + } + } else if ((dot = cur.rfind('.')) == string::npos) { + /* This is a variable name; look it up in the current scope. */ + StringSet::iterator i = varNames.lower_bound(cur); + while (i != varNames.end()) { + if (string(*i, 0, cur.size()) != cur) break; + completions.insert(prev + *i); + i++; + } + } else { + try { + /* This is an expression that should evaluate to an + attribute set. Evaluate it to get the names of the + attributes. */ + string expr(cur, 0, dot); + string cur2 = string(cur, dot + 1); + + Expr * e = parseString(expr); + Value v; + e->eval(*state, *env, v); + state->forceAttrs(v); + + for (auto & i : *v.attrs) { + string name = i.name; + if (string(name, 0, cur2.size()) != cur2) continue; + completions.insert(prev + expr + "." + name); + } + + } catch (ParseError & e) { + // Quietly ignore parse errors. + } catch (EvalError & e) { + // Quietly ignore evaluation errors. + } catch (UndefinedVarError & e) { + // Quietly ignore undefined variable errors. + } + } + + return completions; +} + + +static int runProgram(const string & program, const Strings & args) +{ + Strings args2(args); + args2.push_front(program); + + Pid pid; + pid = fork(); + if (pid == -1) throw SysError("forking"); + if (pid == 0) { + restoreAffinity(); + execvp(program.c_str(), stringsToCharPtrs(args2).data()); + _exit(1); + } + + return pid.wait(); +} + + +bool isVarName(const string & s) +{ + if (s.size() == 0) return false; + char c = s[0]; + if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false; + for (auto & i : s) + if (!((i >= 'a' && i <= 'z') || + (i >= 'A' && i <= 'Z') || + (i >= '0' && i <= '9') || + i == '_' || i == '-' || i == '\'')) + return false; + return true; +} + + +Path NixRepl::getDerivationPath(Value & v) { + auto drvInfo = getDerivation(*state, v, false); + if (!drvInfo) + throw Error("expression does not evaluate to a derivation, so I can't build it"); + Path drvPath = drvInfo->queryDrvPath(); + if (drvPath == "" || !state->store->isValidPath(state->store->parseStorePath(drvPath))) + throw Error("expression did not evaluate to a valid derivation"); + return drvPath; +} + + +bool NixRepl::processLine(string line) +{ + if (line == "") return true; + + string command, arg; + + if (line[0] == ':') { + size_t p = line.find_first_of(" \n\r\t"); + command = string(line, 0, p); + if (p != string::npos) arg = removeWhitespace(string(line, p)); + } else { + arg = line; + } + + if (command == ":?" || command == ":help") { + std::cout + << "The following commands are available:\n" + << "\n" + << " Evaluate and print expression\n" + << " = Bind expression to variable\n" + << " :a Add attributes from resulting set to scope\n" + << " :b Build derivation\n" + << " :e Open the derivation in $EDITOR\n" + << " :i Build derivation, then install result into current profile\n" + << " :l Load Nix expression and add it to scope\n" + << " :p Evaluate and print expression recursively\n" + << " :q Exit nix-repl\n" + << " :r Reload all files\n" + << " :s Build dependencies of derivation, then start nix-shell\n" + << " :t Describe result of evaluation\n" + << " :u Build derivation, then start nix-shell\n"; + } + + else if (command == ":a" || command == ":add") { + Value v; + evalString(arg, v); + addAttrsToScope(v); + } + + else if (command == ":l" || command == ":load") { + state->resetFileCache(); + loadFile(arg); + } + + else if (command == ":r" || command == ":reload") { + state->resetFileCache(); + reloadFiles(); + } + + else if (command == ":e" || command == ":edit") { + Value v; + evalString(arg, v); + + Pos pos; + + if (v.type == tPath || v.type == tString) { + PathSet context; + auto filename = state->coerceToString(noPos, v, context); + pos.file = state->symbols.create(filename); + } else if (v.type == tLambda) { + pos = v.lambda.fun->pos; + } else { + // assume it's a derivation + pos = findDerivationFilename(*state, v, arg); + } + + // Open in EDITOR + auto args = editorFor(pos); + auto editor = args.front(); + args.pop_front(); + runProgram(editor, args); + + // Reload right after exiting the editor + state->resetFileCache(); + reloadFiles(); + } + + else if (command == ":t") { + Value v; + evalString(arg, v); + std::cout << showType(v) << std::endl; + + } else if (command == ":u") { + Value v, f, result; + evalString(arg, v); + evalString("drv: (import {}).runCommand \"shell\" { buildInputs = [ drv ]; } \"\"", f); + state->callFunction(f, v, result, Pos()); + + Path drvPath = getDerivationPath(result); + runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath}); + } + + else if (command == ":b" || command == ":i" || command == ":s") { + Value v; + evalString(arg, v); + Path drvPath = getDerivationPath(v); + + if (command == ":b") { + /* We could do the build in this process using buildPaths(), + but doing it in a child makes it easier to recover from + problems / SIGINT. */ + if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) { + auto drv = readDerivation(*state->store, drvPath); + std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; + for (auto & i : drv.outputs) + std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.path)); + } + } else if (command == ":i") { + runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath}); + } else { + runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath}); + } + } + + else if (command == ":p" || command == ":print") { + Value v; + evalString(arg, v); + printValue(std::cout, v, 1000000000) << std::endl; + } + + else if (command == ":q" || command == ":quit") + return false; + + else if (command != "") + throw Error("unknown command '%1%'", command); + + else { + size_t p = line.find('='); + string name; + if (p != string::npos && + p < line.size() && + line[p + 1] != '=' && + isVarName(name = removeWhitespace(string(line, 0, p)))) + { + Expr * e = parseString(string(line, p + 1)); + Value & v(*state->allocValue()); + v.type = tThunk; + v.thunk.env = env; + v.thunk.expr = e; + addVarToScope(state->symbols.create(name), v); + } else { + Value v; + evalString(line, v); + printValue(std::cout, v, 1) << std::endl; + } + } + + return true; +} + + +void NixRepl::loadFile(const Path & path) +{ + loadedFiles.remove(path); + loadedFiles.push_back(path); + Value v, v2; + state->evalFile(lookupFileArg(*state, path), v); + state->autoCallFunction(*autoArgs, v, v2); + addAttrsToScope(v2); +} + + +void NixRepl::initEnv() +{ + env = &state->allocEnv(envSize); + env->up = &state->baseEnv; + displ = 0; + staticEnv.vars.clear(); + + varNames.clear(); + for (auto & i : state->staticBaseEnv.vars) + varNames.insert(i.first); +} + + +void NixRepl::reloadFiles() +{ + initEnv(); + + Strings old = loadedFiles; + loadedFiles.clear(); + + bool first = true; + for (auto & i : old) { + if (!first) std::cout << std::endl; + first = false; + std::cout << format("Loading '%1%'...") % i << std::endl; + loadFile(i); + } +} + + +void NixRepl::addAttrsToScope(Value & attrs) +{ + state->forceAttrs(attrs); + for (auto & i : *attrs.attrs) + addVarToScope(i.name, *i.value); + std::cout << format("Added %1% variables.") % attrs.attrs->size() << std::endl; +} + + +void NixRepl::addVarToScope(const Symbol & name, Value & v) +{ + if (displ >= envSize) + throw Error("environment full; cannot add more variables"); + staticEnv.vars[name] = displ; + env->values[displ++] = &v; + varNames.insert((string) name); +} + + +Expr * NixRepl::parseString(string s) +{ + Expr * e = state->parseExprFromString(s, curDir, staticEnv); + return e; +} + + +void NixRepl::evalString(string s, Value & v) +{ + Expr * e = parseString(s); + e->eval(*state, *env, v); + state->forceValue(v); +} + + +std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth) +{ + ValuesSeen seen; + return printValue(str, v, maxDepth, seen); +} + + +std::ostream & printStringValue(std::ostream & str, const char * string) { + str << "\""; + for (const char * i = string; *i; i++) + if (*i == '\"' || *i == '\\') str << "\\" << *i; + else if (*i == '\n') str << "\\n"; + else if (*i == '\r') str << "\\r"; + else if (*i == '\t') str << "\\t"; + else str << *i; + str << "\""; + return str; +} + + +// FIXME: lot of cut&paste from Nix's eval.cc. +std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen) +{ + str.flush(); + checkInterrupt(); + + state->forceValue(v); + + switch (v.type) { + + case tInt: + str << ESC_CYA << v.integer << ESC_END; + break; + + case tBool: + str << ESC_CYA << (v.boolean ? "true" : "false") << ESC_END; + break; + + case tString: + str << ESC_YEL; + printStringValue(str, v.string.s); + str << ESC_END; + break; + + case tPath: + str << ESC_GRE << v.path << ESC_END; // !!! escaping? + break; + + case tNull: + str << ESC_CYA "null" ESC_END; + break; + + case tAttrs: { + seen.insert(&v); + + bool isDrv = state->isDerivation(v); + + if (isDrv) { + str << "«derivation "; + Bindings::iterator i = v.attrs->find(state->sDrvPath); + PathSet context; + Path drvPath = i != v.attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : "???"; + str << drvPath << "»"; + } + + else if (maxDepth > 0) { + str << "{ "; + + typedef std::map Sorted; + Sorted sorted; + for (auto & i : *v.attrs) + sorted[i.name] = i.value; + + for (auto & i : sorted) { + if (isVarName(i.first)) + str << i.first; + else + printStringValue(str, i.first.c_str()); + str << " = "; + if (seen.find(i.second) != seen.end()) + str << "«repeated»"; + else + try { + printValue(str, *i.second, maxDepth - 1, seen); + } catch (AssertionError & e) { + str << ESC_RED "«error: " << e.msg() << "»" ESC_END; + } + str << "; "; + } + + str << "}"; + } else + str << "{ ... }"; + + break; + } + + case tList1: + case tList2: + case tListN: + seen.insert(&v); + + str << "[ "; + if (maxDepth > 0) + for (unsigned int n = 0; n < v.listSize(); ++n) { + if (seen.find(v.listElems()[n]) != seen.end()) + str << "«repeated»"; + else + try { + printValue(str, *v.listElems()[n], maxDepth - 1, seen); + } catch (AssertionError & e) { + str << ESC_RED "«error: " << e.msg() << "»" ESC_END; + } + str << " "; + } + else + str << "... "; + str << "]"; + break; + + case tLambda: { + std::ostringstream s; + s << v.lambda.fun->pos; + str << ESC_BLU "«lambda @ " << filterANSIEscapes(s.str()) << "»" ESC_END; + break; + } + + case tPrimOp: + str << ESC_MAG "«primop»" ESC_END; + break; + + case tPrimOpApp: + str << ESC_BLU "«primop-app»" ESC_END; + break; + + case tFloat: + str << v.fpoint; + break; + + default: + str << ESC_RED "«unknown»" ESC_END; + break; + } + + return str; +} + +struct CmdRepl : StoreCommand, MixEvalArgs +{ + std::vector files; + + CmdRepl() + { + expectArgs("files", &files); + } + + std::string description() override + { + return "start an interactive environment for evaluating Nix expressions"; + } + + Examples examples() override + { + return { + Example{ + "Display all special commands within the REPL:", + "nix repl\n nix-repl> :?" + } + }; + } + + void run(ref store) override + { + auto repl = std::make_unique(searchPath, openStore()); + repl->autoArgs = getAutoArgs(*repl->state); + repl->mainLoop(files); + } +}; + +static auto r1 = registerCommand("repl"); + +} diff --git a/src/nix/run.cc b/src/nix/run.cc index a30031ad07b..c9b69aec7a6 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -1,27 +1,78 @@ #include "command.hh" #include "common-args.hh" -#include "installables.hh" #include "shared.hh" #include "store-api.hh" #include "derivations.hh" #include "local-store.hh" #include "finally.hh" +#include "fs-accessor.hh" +#include "progress-bar.hh" +#include "affinity.hh" +#include "eval.hh" #if __linux__ #include #endif +#include + using namespace nix; -struct CmdRun : StoreCommand, MixInstallables +std::string chrootHelperName = "__run_in_chroot"; + +struct RunCommon : virtual Command { - CmdRun() + void runProgram(ref store, + const std::string & program, + const Strings & args) { + stopProgressBar(); + + restoreSignals(); + + restoreAffinity(); + + /* If this is a diverted store (i.e. its "logical" location + (typically /nix/store) differs from its "physical" location + (e.g. /home/eelco/nix/store), then run the command in a + chroot. For non-root users, this requires running it in new + mount and user namespaces. Unfortunately, + unshare(CLONE_NEWUSER) doesn't work in a multithreaded + program (which "nix" is), so we exec() a single-threaded + helper program (chrootHelper() below) to do the work. */ + auto store2 = store.dynamic_pointer_cast(); + + if (store2 && store->storeDir != store2->realStoreDir) { + Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, program }; + for (auto & arg : args) helperArgs.push_back(arg); + + execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data()); + + throw SysError("could not execute chroot helper"); + } + + execvp(program.c_str(), stringsToCharPtrs(args).data()); + + throw SysError("unable to execute '%s'", program); } +}; + +struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment +{ + std::vector command = { getEnv("SHELL").value_or("bash") }; - std::string name() override + CmdShell() { - return "run"; + addFlag({ + .longName = "command", + .shortName = 'c', + .description = "command and arguments to be executed; defaults to '$SHELL'", + .labels = {"command", "args"}, + .handler = {[&](std::vector ss) { + if (ss.empty()) throw UsageError("--command requires at least one argument"); + command = ss; + }} + }); } std::string description() override @@ -29,98 +80,140 @@ struct CmdRun : StoreCommand, MixInstallables return "run a shell in which the specified packages are available"; } + Examples examples() override + { + return { + Example{ + "To start a shell providing GNU Hello from NixOS 17.03:", + "nix shell -f channel:nixos-17.03 hello" + }, + Example{ + "To start a shell providing youtube-dl from your 'nixpkgs' channel:", + "nix shell nixpkgs.youtube-dl" + }, + Example{ + "To run GNU Hello:", + "nix shell nixpkgs.hello -c hello --greeting 'Hi everybody!'" + }, + Example{ + "To run GNU Hello in a chroot store:", + "nix shell --store ~/my-nix nixpkgs.hello -c hello" + }, + }; + } + void run(ref store) override { - auto elems = evalInstallables(store); + auto outPaths = toStorePaths(store, Build, installables); - PathSet pathsToBuild; + auto accessor = store->getFSAccessor(); - for (auto & elem : elems) { - if (elem.isDrv) - pathsToBuild.insert(elem.drvPath); - else - pathsToBuild.insert(elem.outPaths.begin(), elem.outPaths.end()); - } - printMissing(store, pathsToBuild); + std::unordered_set done; + std::queue todo; + for (auto & path : outPaths) todo.push(path.clone()); - store->buildPaths(pathsToBuild); + setEnviron(); - auto store2 = store.dynamic_pointer_cast(); + auto unixPath = tokenizeString(getEnv("PATH").value_or(""), ":"); - if (store2 && store->storeDir != store2->realStoreDir) { -#if __linux__ - uid_t uid = getuid(); - uid_t gid = getgid(); - - if (unshare(CLONE_NEWUSER | CLONE_NEWNS) == -1) - throw SysError("setting up a private mount namespace"); - - /* Bind-mount realStoreDir on /nix/store. If the latter - mount point doesn't already exists, we have to create a - chroot environment containing the mount point and bind - mounts for the children of /. Would be nice if we could - use overlayfs here, but that doesn't work in a user - namespace yet (Ubuntu has a patch for this: - https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1478578). */ - if (!pathExists(store->storeDir)) { - // FIXME: Use overlayfs? - - Path tmpDir = createTempDir(); - - createDirs(tmpDir + store->storeDir); - - if (mount(store2->realStoreDir.c_str(), (tmpDir + store->storeDir).c_str(), "", MS_BIND, 0) == -1) - throw SysError(format("mounting ‘%s’ on ‘%s’") % store2->realStoreDir % store->storeDir); - - for (auto entry : readDirectory("/")) { - Path dst = tmpDir + "/" + entry.name; - if (pathExists(dst)) continue; - if (mkdir(dst.c_str(), 0700) == -1) - throw SysError(format("creating directory ‘%s’") % dst); - if (mount(("/" + entry.name).c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError(format("mounting ‘%s’ on ‘%s’") % ("/" + entry.name) % dst); - } - - char * cwd = getcwd(0, 0); - if (!cwd) throw SysError("getting current directory"); - Finally freeCwd([&]() { free(cwd); }); - - if (chroot(tmpDir.c_str()) == -1) - throw SysError(format("chrooting into ‘%s’") % tmpDir); - - if (chdir(cwd) == -1) - throw SysError(format("chdir to ‘%s’ in chroot") % cwd); - } else - if (mount(store2->realStoreDir.c_str(), store->storeDir.c_str(), "", MS_BIND, 0) == -1) - throw SysError(format("mounting ‘%s’ on ‘%s’") % store2->realStoreDir % store->storeDir); - - writeFile("/proc/self/setgroups", "deny"); - writeFile("/proc/self/uid_map", (format("%d %d %d") % uid % uid % 1).str()); - writeFile("/proc/self/gid_map", (format("%d %d %d") % gid % gid % 1).str()); -#else - throw Error(format("mounting the Nix store on ‘%s’ is not supported on this platform") % store->storeDir); -#endif + while (!todo.empty()) { + auto path = todo.front().clone(); + todo.pop(); + if (!done.insert(path.clone()).second) continue; + + if (true) + unixPath.push_front(store->printStorePath(path) + "/bin"); + + auto propPath = store->printStorePath(path) + "/nix-support/propagated-user-env-packages"; + if (accessor->stat(propPath).type == FSAccessor::tRegular) { + for (auto & p : tokenizeString(readFile(propPath))) + todo.push(store->parseStorePath(p)); + } } - PathSet outPaths; - for (auto & path : pathsToBuild) - if (isDerivation(path)) { - Derivation drv = store->derivationFromPath(path); - for (auto & output : drv.outputs) - outPaths.insert(output.second.path); - } else - outPaths.insert(path); - - auto unixPath = tokenizeString(getEnv("PATH"), ":"); - for (auto & path : outPaths) - if (pathExists(path + "/bin")) - unixPath.push_front(path + "/bin"); setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1); - if (execlp("bash", "bash", nullptr) == -1) - throw SysError("unable to exec ‘bash’"); + Strings args; + for (auto & arg : command) args.push_back(arg); + + runProgram(store, *command.begin(), args); } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("shell"); + +void chrootHelper(int argc, char * * argv) +{ + int p = 1; + std::string storeDir = argv[p++]; + std::string realStoreDir = argv[p++]; + std::string cmd = argv[p++]; + Strings args; + while (p < argc) + args.push_back(argv[p++]); + +#if __linux__ + uid_t uid = getuid(); + uid_t gid = getgid(); + + if (unshare(CLONE_NEWUSER | CLONE_NEWNS) == -1) + /* Try with just CLONE_NEWNS in case user namespaces are + specifically disabled. */ + if (unshare(CLONE_NEWNS) == -1) + throw SysError("setting up a private mount namespace"); + + /* Bind-mount realStoreDir on /nix/store. If the latter mount + point doesn't already exists, we have to create a chroot + environment containing the mount point and bind mounts for the + children of /. Would be nice if we could use overlayfs here, + but that doesn't work in a user namespace yet (Ubuntu has a + patch for this: + https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1478578). */ + if (!pathExists(storeDir)) { + // FIXME: Use overlayfs? + + Path tmpDir = createTempDir(); + + createDirs(tmpDir + storeDir); + + if (mount(realStoreDir.c_str(), (tmpDir + storeDir).c_str(), "", MS_BIND, 0) == -1) + throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); + + for (auto entry : readDirectory("/")) { + auto src = "/" + entry.name; + auto st = lstat(src); + if (!S_ISDIR(st.st_mode)) continue; + Path dst = tmpDir + "/" + entry.name; + if (pathExists(dst)) continue; + if (mkdir(dst.c_str(), 0700) == -1) + throw SysError("creating directory '%s'", dst); + if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1) + throw SysError("mounting '%s' on '%s'", src, dst); + } + + char * cwd = getcwd(0, 0); + if (!cwd) throw SysError("getting current directory"); + Finally freeCwd([&]() { free(cwd); }); + + if (chroot(tmpDir.c_str()) == -1) + throw SysError("chrooting into '%s'", tmpDir); + + if (chdir(cwd) == -1) + throw SysError("chdir to '%s' in chroot", cwd); + } else + if (mount(realStoreDir.c_str(), storeDir.c_str(), "", MS_BIND, 0) == -1) + throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); + + writeFile("/proc/self/setgroups", "deny"); + writeFile("/proc/self/uid_map", fmt("%d %d %d", uid, uid, 1)); + writeFile("/proc/self/gid_map", fmt("%d %d %d", gid, gid, 1)); + + execvp(cmd.c_str(), stringsToCharPtrs(args).data()); + + throw SysError("unable to exec '%s'", cmd); + +#else + throw Error("mounting the Nix store on '%s' is not supported on this platform", storeDir); +#endif +} diff --git a/src/nix/search.cc b/src/nix/search.cc new file mode 100644 index 00000000000..ba72c1e799f --- /dev/null +++ b/src/nix/search.cc @@ -0,0 +1,277 @@ +#include "command.hh" +#include "globals.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "names.hh" +#include "get-drvs.hh" +#include "common-args.hh" +#include "json.hh" +#include "json-to-value.hh" +#include "shared.hh" + +#include +#include + +using namespace nix; + +std::string wrap(std::string prefix, std::string s) +{ + return prefix + s + ANSI_NORMAL; +} + +std::string hilite(const std::string & s, const std::smatch & m, std::string postfix) +{ + return + m.empty() + ? s + : std::string(m.prefix()) + + ANSI_RED + std::string(m.str()) + postfix + + std::string(m.suffix()); +} + +struct CmdSearch : SourceExprCommand, MixJSON +{ + std::vector res; + + bool writeCache = true; + bool useCache = true; + + CmdSearch() + { + expectArgs("regex", &res); + + addFlag({ + .longName = "update-cache", + .shortName = 'u', + .description = "update the package search cache", + .handler = {[&]() { writeCache = true; useCache = false; }} + }); + + addFlag({ + .longName = "no-cache", + .description = "do not use or update the package search cache", + .handler = {[&]() { writeCache = false; useCache = false; }} + }); + } + + std::string description() override + { + return "query available packages"; + } + + Examples examples() override + { + return { + Example{ + "To show all available packages:", + "nix search" + }, + Example{ + "To show any packages containing 'blender' in its name or description:", + "nix search blender" + }, + Example{ + "To search for Firefox or Chromium:", + "nix search 'firefox|chromium'" + }, + Example{ + "To search for git and frontend or gui:", + "nix search git 'frontend|gui'" + } + }; + } + + void run(ref store) override + { + settings.readOnlyMode = true; + + // Empty search string should match all packages + // Use "^" here instead of ".*" due to differences in resulting highlighting + // (see #1893 -- libc++ claims empty search string is not in POSIX grammar) + if (res.empty()) { + res.push_back("^"); + } + + std::vector regexes; + regexes.reserve(res.size()); + + for (auto &re : res) { + regexes.push_back(std::regex(re, std::regex::extended | std::regex::icase)); + } + + auto state = getEvalState(); + + auto jsonOut = json ? std::make_unique(std::cout) : nullptr; + + auto sToplevel = state->symbols.create("_toplevel"); + auto sRecurse = state->symbols.create("recurseForDerivations"); + + bool fromCache = false; + + std::map results; + + std::function doExpr; + + doExpr = [&](Value * v, std::string attrPath, bool toplevel, JSONObject * cache) { + debug("at attribute '%s'", attrPath); + + try { + uint found = 0; + + state->forceValue(*v); + + if (v->type == tLambda && toplevel) { + Value * v2 = state->allocValue(); + state->autoCallFunction(*state->allocBindings(1), *v, *v2); + v = v2; + state->forceValue(*v); + } + + if (state->isDerivation(*v)) { + + DrvInfo drv(*state, attrPath, v->attrs); + std::string description; + std::smatch attrPathMatch; + std::smatch descriptionMatch; + std::smatch nameMatch; + std::string name; + + DrvName parsed(drv.queryName()); + + for (auto ®ex : regexes) { + std::regex_search(attrPath, attrPathMatch, regex); + + name = parsed.name; + std::regex_search(name, nameMatch, regex); + + description = drv.queryMetaString("description"); + std::replace(description.begin(), description.end(), '\n', ' '); + std::regex_search(description, descriptionMatch, regex); + + if (!attrPathMatch.empty() + || !nameMatch.empty() + || !descriptionMatch.empty()) + { + found++; + } + } + + if (found == res.size()) { + if (json) { + + auto jsonElem = jsonOut->object(attrPath); + + jsonElem.attr("pkgName", parsed.name); + jsonElem.attr("version", parsed.version); + jsonElem.attr("description", description); + + } else { + auto name = hilite(parsed.name, nameMatch, "\e[0;2m") + + std::string(parsed.fullName, parsed.name.length()); + results[attrPath] = fmt( + "* %s (%s)\n %s\n", + wrap("\e[0;1m", hilite(attrPath, attrPathMatch, "\e[0;1m")), + wrap("\e[0;2m", hilite(name, nameMatch, "\e[0;2m")), + hilite(description, descriptionMatch, ANSI_NORMAL)); + } + } + + if (cache) { + cache->attr("type", "derivation"); + cache->attr("name", drv.queryName()); + cache->attr("system", drv.querySystem()); + if (description != "") { + auto meta(cache->object("meta")); + meta.attr("description", description); + } + } + } + + else if (v->type == tAttrs) { + + if (!toplevel) { + auto attrs = v->attrs; + Bindings::iterator j = attrs->find(sRecurse); + if (j == attrs->end() || !state->forceBool(*j->value, *j->pos)) { + debug("skip attribute '%s'", attrPath); + return; + } + } + + bool toplevel2 = false; + if (!fromCache) { + Bindings::iterator j = v->attrs->find(sToplevel); + toplevel2 = j != v->attrs->end() && state->forceBool(*j->value, *j->pos); + } + + for (auto & i : *v->attrs) { + auto cache2 = + cache ? std::make_unique(cache->object(i.name)) : nullptr; + doExpr(i.value, + attrPath == "" ? (std::string) i.name : attrPath + "." + (std::string) i.name, + toplevel2 || fromCache, cache2 ? cache2.get() : nullptr); + } + } + + } catch (AssertionError & e) { + } catch (Error & e) { + if (!toplevel) { + e.addPrefix(fmt("While evaluating the attribute '%s':\n", attrPath)); + throw; + } + } + }; + + Path jsonCacheFileName = getCacheDir() + "/nix/package-search.json"; + + if (useCache && pathExists(jsonCacheFileName)) { + + warn("using cached results; pass '-u' to update the cache"); + + Value vRoot; + parseJSON(*state, readFile(jsonCacheFileName), vRoot); + + fromCache = true; + + doExpr(&vRoot, "", true, nullptr); + } + + else { + createDirs(dirOf(jsonCacheFileName)); + + Path tmpFile = fmt("%s.tmp.%d", jsonCacheFileName, getpid()); + + std::ofstream jsonCacheFile; + + try { + // iostream considered harmful + jsonCacheFile.exceptions(std::ofstream::failbit); + jsonCacheFile.open(tmpFile); + + auto cache = writeCache ? std::make_unique(jsonCacheFile, false) : nullptr; + + doExpr(getSourceExpr(*state), "", true, cache.get()); + + } catch (std::exception &) { + /* Fun fact: catching std::ios::failure does not work + due to C++11 ABI shenanigans. + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66145 */ + if (!jsonCacheFile) + throw Error("error writing to %s", tmpFile); + throw; + } + + if (writeCache && rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1) + throw SysError("cannot rename '%s' to '%s'", tmpFile, jsonCacheFileName); + } + + if (!json && results.size() == 0) + throw Error("no results for the given search term(s)!"); + + RunPager pager; + for (auto el : results) std::cout << el.second << "\n"; + + } +}; + +static auto r1 = registerCommand("search"); diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc new file mode 100644 index 00000000000..4fd8886de08 --- /dev/null +++ b/src/nix/show-config.cc @@ -0,0 +1,33 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "json.hh" + +using namespace nix; + +struct CmdShowConfig : Command, MixJSON +{ + std::string description() override + { + return "show the Nix configuration"; + } + + Category category() override { return catUtility; } + + void run() override + { + if (json) { + // FIXME: use appropriate JSON types (bool, ints, etc). + JSONObject jsonObj(std::cout); + globalConfig.toJSON(jsonObj); + } else { + std::map settings; + globalConfig.getSettings(settings); + for (auto & s : settings) + logger->stdout("%s = %s", s.first, s.second.value); + } + } +}; + +static auto r1 = registerCommand("show-config"); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc new file mode 100644 index 00000000000..2d31894c271 --- /dev/null +++ b/src/nix/show-derivation.cc @@ -0,0 +1,117 @@ +// FIXME: integrate this with nix path-info? + +#include "command.hh" +#include "common-args.hh" +#include "store-api.hh" +#include "archive.hh" +#include "json.hh" +#include "derivations.hh" + +using namespace nix; + +struct CmdShowDerivation : InstallablesCommand +{ + bool recursive = false; + + CmdShowDerivation() + { + addFlag({ + .longName = "recursive", + .shortName = 'r', + .description = "include the dependencies of the specified derivations", + .handler = {&recursive, true} + }); + } + + std::string description() override + { + return "show the contents of a store derivation"; + } + + Examples examples() override + { + return { + Example{ + "To show the store derivation that results from evaluating the Hello package:", + "nix show-derivation nixpkgs.hello" + }, + Example{ + "To show the full derivation graph (if available) that produced your NixOS system:", + "nix show-derivation -r /run/current-system" + }, + }; + } + + Category category() override { return catUtility; } + + void run(ref store) override + { + auto drvPaths = toDerivations(store, installables, true); + + if (recursive) { + StorePathSet closure; + store->computeFSClosure(drvPaths, closure); + drvPaths = std::move(closure); + } + + { + + JSONObject jsonRoot(std::cout, true); + + for (auto & drvPath : drvPaths) { + if (!drvPath.isDerivation()) continue; + + auto drvObj(jsonRoot.object(store->printStorePath(drvPath))); + + auto drv = store->readDerivation(drvPath); + + { + auto outputsObj(drvObj.object("outputs")); + for (auto & output : drv.outputs) { + auto outputObj(outputsObj.object(output.first)); + outputObj.attr("path", store->printStorePath(output.second.path)); + if (output.second.hash != "") { + outputObj.attr("hashAlgo", output.second.hashAlgo); + outputObj.attr("hash", output.second.hash); + } + } + } + + { + auto inputsList(drvObj.list("inputSrcs")); + for (auto & input : drv.inputSrcs) + inputsList.elem(store->printStorePath(input)); + } + + { + auto inputDrvsObj(drvObj.object("inputDrvs")); + for (auto & input : drv.inputDrvs) { + auto inputList(inputDrvsObj.list(store->printStorePath(input.first))); + for (auto & outputId : input.second) + inputList.elem(outputId); + } + } + + drvObj.attr("platform", drv.platform); + drvObj.attr("builder", drv.builder); + + { + auto argsList(drvObj.list("args")); + for (auto & arg : drv.args) + argsList.elem(arg); + } + + { + auto envObj(drvObj.object("env")); + for (auto & var : drv.env) + envObj.attr(var.first, var.second); + } + } + + } + + std::cout << "\n"; + } +}; + +static auto r1 = registerCommand("show-derivation"); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index d8d8c0f53df..6c9b9a79286 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -13,13 +13,13 @@ struct CmdCopySigs : StorePathsCommand CmdCopySigs() { - mkFlag('s', "substituter", {"store-uri"}, "use signatures from specified store", 1, - [&](Strings ss) { substituterUris.push_back(ss.front()); }); - } - - std::string name() override - { - return "copy-sigs"; + addFlag({ + .longName = "substituter", + .shortName = 's', + .description = "use signatures from specified store", + .labels = {"store-uri"}, + .handler = {[&](std::string s) { substituterUris.push_back(s); }}, + }); } std::string description() override @@ -27,10 +27,12 @@ struct CmdCopySigs : StorePathsCommand return "copy path signatures from substituters (like binary caches)"; } - void run(ref store, Paths storePaths) override + Category category() override { return catUtility; } + + void run(ref store, StorePaths storePaths) override { if (substituterUris.empty()) - throw UsageError("you must specify at least one substituter using ‘-s’"); + throw UsageError("you must specify at least one substituter using '-s'"); // FIXME: factor out commonality with MixVerify. std::vector> substituters; @@ -42,20 +44,22 @@ struct CmdCopySigs : StorePathsCommand std::string doneLabel = "done"; std::atomic added{0}; - logger->setExpected(doneLabel, storePaths.size()); + //logger->setExpected(doneLabel, storePaths.size()); - auto doPath = [&](const Path & storePath) { - Activity act(*logger, lvlInfo, format("getting signatures for ‘%s’") % storePath); + auto doPath = [&](const Path & storePathS) { + //Activity act(*logger, lvlInfo, format("getting signatures for '%s'") % storePath); checkInterrupt(); + auto storePath = store->parseStorePath(storePathS); + auto info = store->queryPathInfo(storePath); StringSet newSigs; for (auto & store2 : substituters) { try { - auto info2 = store2->queryPathInfo(storePath); + auto info2 = store2->queryPathInfo(info->path); /* Don't import signatures that don't match this binary. */ @@ -76,19 +80,19 @@ struct CmdCopySigs : StorePathsCommand added += newSigs.size(); } - logger->incProgress(doneLabel); + //logger->incProgress(doneLabel); }; for (auto & storePath : storePaths) - pool.enqueue(std::bind(doPath, storePath)); + pool.enqueue(std::bind(doPath, store->printStorePath(storePath))); pool.process(); - printInfo(format("imported %d signatures") % added); + printInfo("imported %d signatures", added); } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("copy-sigs"); struct CmdSignPaths : StorePathsCommand { @@ -96,12 +100,13 @@ struct CmdSignPaths : StorePathsCommand CmdSignPaths() { - mkFlag('k', "key-file", {"file"}, "file containing the secret signing key", &secretKeyFile); - } - - std::string name() override - { - return "sign-paths"; + addFlag({ + .longName = "key-file", + .shortName = 'k', + .description = "file containing the secret signing key", + .labels = {"file"}, + .handler = {&secretKeyFile} + }); } std::string description() override @@ -109,10 +114,12 @@ struct CmdSignPaths : StorePathsCommand return "sign the specified paths"; } - void run(ref store, Paths storePaths) override + Category category() override { return catUtility; } + + void run(ref store, StorePaths storePaths) override { if (secretKeyFile.empty()) - throw UsageError("you must specify a secret key file using ‘-k’"); + throw UsageError("you must specify a secret key file using '-k'"); SecretKey secretKey(readFile(secretKeyFile)); @@ -123,7 +130,7 @@ struct CmdSignPaths : StorePathsCommand auto info2(*info); info2.sigs.clear(); - info2.sign(secretKey); + info2.sign(*store, secretKey); assert(!info2.sigs.empty()); if (!info->sigs.count(*info2.sigs.begin())) { @@ -132,8 +139,8 @@ struct CmdSignPaths : StorePathsCommand } } - printInfo(format("added %d signatures") % added); + printInfo("added %d signatures", added); } }; -static RegisterCommand r3(make_ref()); +static auto r2 = registerCommand("sign-paths"); diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc new file mode 100644 index 00000000000..a880bdae097 --- /dev/null +++ b/src/nix/upgrade-nix.cc @@ -0,0 +1,161 @@ +#include "command.hh" +#include "common-args.hh" +#include "store-api.hh" +#include "filetransfer.hh" +#include "eval.hh" +#include "attr-path.hh" +#include "names.hh" +#include "progress-bar.hh" + +using namespace nix; + +struct CmdUpgradeNix : MixDryRun, StoreCommand +{ + Path profileDir; + std::string storePathsUrl = "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix"; + + CmdUpgradeNix() + { + addFlag({ + .longName = "profile", + .shortName = 'p', + .description = "the Nix profile to upgrade", + .labels = {"profile-dir"}, + .handler = {&profileDir} + }); + + addFlag({ + .longName = "nix-store-paths-url", + .description = "URL of the file that contains the store paths of the latest Nix release", + .labels = {"url"}, + .handler = {&storePathsUrl} + }); + } + + std::string description() override + { + return "upgrade Nix to the latest stable version"; + } + + Examples examples() override + { + return { + Example{ + "To upgrade Nix to the latest stable version:", + "nix upgrade-nix" + }, + Example{ + "To upgrade Nix in a specific profile:", + "nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile" + }, + }; + } + + Category category() override { return catNixInstallation; } + + void run(ref store) override + { + evalSettings.pureEval = true; + + if (profileDir == "") + profileDir = getProfileDir(store); + + printInfo("upgrading Nix in profile '%s'", profileDir); + + auto storePath = getLatestNix(store); + + auto version = DrvName(storePath.name()).version; + + if (dryRun) { + stopProgressBar(); + logWarning({ + .name = "Version update", + .hint = hintfmt("would upgrade to version %s", version) + }); + return; + } + + { + Activity act(*logger, lvlInfo, actUnknown, fmt("downloading '%s'...", store->printStorePath(storePath))); + store->ensurePath(storePath); + } + + { + Activity act(*logger, lvlInfo, actUnknown, fmt("verifying that '%s' works...", store->printStorePath(storePath))); + auto program = store->printStorePath(storePath) + "/bin/nix-env"; + auto s = runProgram(program, false, {"--version"}); + if (s.find("Nix") == std::string::npos) + throw Error("could not verify that '%s' works", program); + } + + stopProgressBar(); + + { + Activity act(*logger, lvlInfo, actUnknown, + fmt("installing '%s' into profile '%s'...", store->printStorePath(storePath), profileDir)); + runProgram(settings.nixBinDir + "/nix-env", false, + {"--profile", profileDir, "-i", store->printStorePath(storePath), "--no-sandbox"}); + } + + printInfo(ANSI_GREEN "upgrade to version %s done" ANSI_NORMAL, version); + } + + /* Return the profile in which Nix is installed. */ + Path getProfileDir(ref store) + { + Path where; + + for (auto & dir : tokenizeString(getEnv("PATH").value_or(""), ":")) + if (pathExists(dir + "/nix-env")) { + where = dir; + break; + } + + if (where == "") + throw Error("couldn't figure out how Nix is installed, so I can't upgrade it"); + + printInfo("found Nix in '%s'", where); + + if (hasPrefix(where, "/run/current-system")) + throw Error("Nix on NixOS must be upgraded via 'nixos-rebuild'"); + + Path profileDir = dirOf(where); + + // Resolve profile to /nix/var/nix/profiles/ link. + while (canonPath(profileDir).find("/profiles/") == std::string::npos && isLink(profileDir)) + profileDir = readLink(profileDir); + + printInfo("found profile '%s'", profileDir); + + Path userEnv = canonPath(profileDir, true); + + if (baseNameOf(where) != "bin" || + !hasSuffix(userEnv, "user-environment")) + throw Error("directory '%s' does not appear to be part of a Nix profile", where); + + if (!store->isValidPath(store->parseStorePath(userEnv))) + throw Error("directory '%s' is not in the Nix store", userEnv); + + return profileDir; + } + + /* Return the store path of the latest stable Nix. */ + StorePath getLatestNix(ref store) + { + Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version"); + + // FIXME: use nixos.org? + auto req = FileTransferRequest(storePathsUrl); + auto res = getFileTransfer()->download(req); + + auto state = std::make_unique(Strings(), store); + auto v = state->allocValue(); + state->eval(state->parseExprFromString(*res.data, "/no-such-path"), *v); + Bindings & bindings(*state->allocBindings(0)); + auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v).first; + + return store->parseStorePath(state->forceString(*v2)); + } +}; + +static auto r1 = registerCommand("upgrade-nix"); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 2f8d02fa060..001401ac231 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -3,6 +3,7 @@ #include "store-api.hh" #include "sync.hh" #include "thread-pool.hh" +#include "references.hh" #include @@ -13,22 +14,22 @@ struct CmdVerify : StorePathsCommand bool noContents = false; bool noTrust = false; Strings substituterUris; - size_t sigsNeeded; + size_t sigsNeeded = 0; CmdVerify() { mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents); mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust); - mkFlag('s', "substituter", {"store-uri"}, "use signatures from specified store", 1, - [&](Strings ss) { substituterUris.push_back(ss.front()); }); + addFlag({ + .longName = "substituter", + .shortName = 's', + .description = "use signatures from specified store", + .labels = {"store-uri"}, + .handler = {[&](std::string s) { substituterUris.push_back(s); }} + }); mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded); } - std::string name() override - { - return "verify"; - } - std::string description() override { return "verify the integrity of store paths"; @@ -48,7 +49,9 @@ struct CmdVerify : StorePathsCommand }; } - void run(ref store, Paths storePaths) override + Category category() override { return catSecondary; } + + void run(ref store, StorePaths storePaths) override { std::vector> substituters; for (auto & s : substituterUris) @@ -56,16 +59,17 @@ struct CmdVerify : StorePathsCommand auto publicKeys = getDefaultPublicKeys(); + Activity act(*logger, actVerifyPaths); + std::atomic done{0}; std::atomic untrusted{0}; std::atomic corrupted{0}; std::atomic failed{0}; + std::atomic active{0}; - std::string doneLabel("paths checked"); - std::string untrustedLabel("untrusted"); - std::string corruptedLabel("corrupted"); - std::string failedLabel("failed"); - logger->setExpected(doneLabel, storePaths.size()); + auto update = [&]() { + act.progress(done, storePaths.size(), active, failed); + }; ThreadPool pool; @@ -73,25 +77,37 @@ struct CmdVerify : StorePathsCommand try { checkInterrupt(); - Activity act(*logger, lvlInfo, format("checking ‘%s’") % storePath); + Activity act2(*logger, lvlInfo, actUnknown, fmt("checking '%s'", storePath)); - auto info = store->queryPathInfo(storePath); + MaintainCount> mcActive(active); + update(); + + auto info = store->queryPathInfo(store->parseStorePath(storePath)); if (!noContents) { - HashSink sink(info->narHash.type); - store->narFromPath(info->path, sink); + std::unique_ptr hashSink; + if (info->ca == "") + hashSink = std::make_unique(info->narHash.type); + else + hashSink = std::make_unique(info->narHash.type, storePathToHash(store->printStorePath(info->path))); + + store->narFromPath(info->path, *hashSink); - auto hash = sink.finish(); + auto hash = hashSink->finish(); if (hash.first != info->narHash) { - logger->incProgress(corruptedLabel); - corrupted = 1; - printError( - format("path ‘%s’ was modified! expected hash ‘%s’, got ‘%s’") - % info->path % printHash(info->narHash) % printHash(hash.first)); + corrupted++; + act2.result(resCorruptedPath, store->printStorePath(info->path)); + logError({ + .name = "Hash error - path modified", + .hint = hintfmt( + "path '%s' was modified! expected hash '%s', got '%s'", + store->printStorePath(info->path), + info->narHash.to_string(Base32, true), + hash.first.to_string(Base32, true)) + }); } - } if (!noTrust) { @@ -104,14 +120,13 @@ struct CmdVerify : StorePathsCommand else { StringSet sigsSeen; - size_t actualSigsNeeded = sigsNeeded ? sigsNeeded : 1; + size_t actualSigsNeeded = std::max(sigsNeeded, (size_t) 1); size_t validSigs = 0; auto doSigs = [&](StringSet sigs) { for (auto sig : sigs) { - if (sigsSeen.count(sig)) continue; - sigsSeen.insert(sig); - if (info->checkSignature(publicKeys, sig)) + if (!sigsSeen.insert(sig).second) continue; + if (validSigs < ValidPathInfo::maxSigs && info->checkSignature(*store, publicKeys, sig)) validSigs++; } }; @@ -128,7 +143,7 @@ struct CmdVerify : StorePathsCommand doSigs(info2->sigs); } catch (InvalidPath &) { } catch (Error & e) { - printError(format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what()); + logError(e.info()); } } @@ -137,31 +152,33 @@ struct CmdVerify : StorePathsCommand } if (!good) { - logger->incProgress(untrustedLabel); untrusted++; - printError(format("path ‘%s’ is untrusted") % info->path); + act2.result(resUntrustedPath, store->printStorePath(info->path)); + logError({ + .name = "Untrusted path", + .hint = hintfmt("path '%s' is untrusted", + store->printStorePath(info->path)) + }); + } } - logger->incProgress(doneLabel); done++; } catch (Error & e) { - printError(format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what()); - logger->incProgress(failedLabel); + logError(e.info()); failed++; } + + update(); }; for (auto & storePath : storePaths) - pool.enqueue(std::bind(doPath, storePath)); + pool.enqueue(std::bind(doPath, store->printStorePath(storePath))); pool.process(); - printInfo(format("%d paths checked, %d untrusted, %d corrupted, %d failed") - % done % untrusted % corrupted % failed); - throw Exit( (corrupted ? 1 : 0) | (untrusted ? 2 : 0) | @@ -169,4 +186,4 @@ struct CmdVerify : StorePathsCommand } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("verify"); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc new file mode 100644 index 00000000000..6057beedb64 --- /dev/null +++ b/src/nix/why-depends.cc @@ -0,0 +1,262 @@ +#include "command.hh" +#include "store-api.hh" +#include "progress-bar.hh" +#include "fs-accessor.hh" +#include "shared.hh" + +#include + +using namespace nix; + +static std::string hilite(const std::string & s, size_t pos, size_t len, + const std::string & colour = ANSI_RED) +{ + return + std::string(s, 0, pos) + + colour + + std::string(s, pos, len) + + ANSI_NORMAL + + std::string(s, pos + len); +} + +static std::string filterPrintable(const std::string & s) +{ + std::string res; + for (char c : s) + res += isprint(c) ? c : '.'; + return res; +} + +struct CmdWhyDepends : SourceExprCommand +{ + std::string _package, _dependency; + bool all = false; + + CmdWhyDepends() + { + expectArg("package", &_package); + expectArg("dependency", &_dependency); + + addFlag({ + .longName = "all", + .shortName = 'a', + .description = "show all edges in the dependency graph leading from 'package' to 'dependency', rather than just a shortest path", + .handler = {&all, true}, + }); + } + + std::string description() override + { + return "show why a package has another package in its closure"; + } + + Examples examples() override + { + return { + Example{ + "To show one path through the dependency graph leading from Hello to Glibc:", + "nix why-depends nixpkgs.hello nixpkgs.glibc" + }, + Example{ + "To show all files and paths in the dependency graph leading from Thunderbird to libX11:", + "nix why-depends --all nixpkgs.thunderbird nixpkgs.xorg.libX11" + }, + Example{ + "To show why Glibc depends on itself:", + "nix why-depends nixpkgs.glibc nixpkgs.glibc" + }, + }; + } + + Category category() override { return catSecondary; } + + void run(ref store) override + { + auto package = parseInstallable(*this, store, _package, false); + auto packagePath = toStorePath(store, Build, package); + auto dependency = parseInstallable(*this, store, _dependency, false); + auto dependencyPath = toStorePath(store, NoBuild, dependency); + auto dependencyPathHash = storePathToHash(store->printStorePath(dependencyPath)); + + StorePathSet closure; + store->computeFSClosure({packagePath}, closure, false, false); + + if (!closure.count(dependencyPath)) { + printError("'%s' does not depend on '%s'", package->what(), dependency->what()); + return; + } + + stopProgressBar(); // FIXME + + auto accessor = store->getFSAccessor(); + + auto const inf = std::numeric_limits::max(); + + struct Node + { + StorePath path; + StorePathSet refs; + StorePathSet rrefs; + size_t dist = inf; + Node * prev = nullptr; + bool queued = false; + bool visited = false; + }; + + std::map graph; + + for (auto & path : closure) + graph.emplace(path.clone(), Node { .path = path.clone(), .refs = cloneStorePathSet(store->queryPathInfo(path)->references) }); + + // Transpose the graph. + for (auto & node : graph) + for (auto & ref : node.second.refs) + graph.find(ref)->second.rrefs.insert(node.first.clone()); + + /* Run Dijkstra's shortest path algorithm to get the distance + of every path in the closure to 'dependency'. */ + graph.emplace(dependencyPath.clone(), Node { .path = dependencyPath.clone(), .dist = 0 }); + + std::priority_queue queue; + + queue.push(&graph.at(dependencyPath)); + + while (!queue.empty()) { + auto & node = *queue.top(); + queue.pop(); + + for (auto & rref : node.rrefs) { + auto & node2 = graph.at(rref); + auto dist = node.dist + 1; + if (dist < node2.dist) { + node2.dist = dist; + node2.prev = &node; + if (!node2.queued) { + node2.queued = true; + queue.push(&node2); + } + } + + } + } + + /* Print the subgraph of nodes that have 'dependency' in their + closure (i.e., that have a non-infinite distance to + 'dependency'). Print every edge on a path between `package` + and `dependency`. */ + std::function printNode; + + struct BailOut { }; + + printNode = [&](Node & node, const string & firstPad, const string & tailPad) { + auto pathS = store->printStorePath(node.path); + + assert(node.dist != inf); + logger->stdout("%s%s%s%s" ANSI_NORMAL, + firstPad, + node.visited ? "\e[38;5;244m" : "", + firstPad != "" ? "→ " : "", + pathS); + + if (node.path == dependencyPath && !all + && packagePath != dependencyPath) + throw BailOut(); + + if (node.visited) return; + node.visited = true; + + /* Sort the references by distance to `dependency` to + ensure that the shortest path is printed first. */ + std::multimap refs; + std::set hashes; + + for (auto & ref : node.refs) { + if (ref == node.path && packagePath != dependencyPath) continue; + auto & node2 = graph.at(ref); + if (node2.dist == inf) continue; + refs.emplace(node2.dist, &node2); + hashes.insert(storePathToHash(store->printStorePath(node2.path))); + } + + /* For each reference, find the files and symlinks that + contain the reference. */ + std::map hits; + + std::function visitPath; + + visitPath = [&](const Path & p) { + auto st = accessor->stat(p); + + auto p2 = p == pathS ? "/" : std::string(p, pathS.size() + 1); + + auto getColour = [&](const std::string & hash) { + return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE; + }; + + if (st.type == FSAccessor::Type::tDirectory) { + auto names = accessor->readDirectory(p); + for (auto & name : names) + visitPath(p + "/" + name); + } + + else if (st.type == FSAccessor::Type::tRegular) { + auto contents = accessor->readFile(p); + + for (auto & hash : hashes) { + auto pos = contents.find(hash); + if (pos != std::string::npos) { + size_t margin = 32; + auto pos2 = pos >= margin ? pos - margin : 0; + hits[hash].emplace_back(fmt("%s: …%s…\n", + p2, + hilite(filterPrintable( + std::string(contents, pos2, pos - pos2 + hash.size() + margin)), + pos - pos2, storePathHashLen, + getColour(hash)))); + } + } + } + + else if (st.type == FSAccessor::Type::tSymlink) { + auto target = accessor->readLink(p); + + for (auto & hash : hashes) { + auto pos = target.find(hash); + if (pos != std::string::npos) + hits[hash].emplace_back(fmt("%s -> %s\n", p2, + hilite(target, pos, storePathHashLen, getColour(hash)))); + } + } + }; + + // FIXME: should use scanForReferences(). + + visitPath(pathS); + + RunPager pager; + for (auto & ref : refs) { + auto hash = storePathToHash(store->printStorePath(ref.second->path)); + + bool last = all ? ref == *refs.rbegin() : true; + + for (auto & hit : hits[hash]) { + bool first = hit == *hits[hash].begin(); + std::cout << tailPad + << (first ? (last ? treeLast : treeConn) : (last ? treeNull : treeLine)) + << hit; + if (!all) break; + } + + printNode(*ref.second, + tailPad + (last ? treeNull : treeLine), + tailPad + (last ? treeNull : treeLine)); + } + }; + + try { + printNode(graph.at(packagePath), "", ""); + } catch (BailOut & ) { } + } +}; + +static auto r1 = registerCommand("why-depends"); diff --git a/src/resolve-system-dependencies/local.mk b/src/resolve-system-dependencies/local.mk index 8792a4a252f..f0e82e02369 100644 --- a/src/resolve-system-dependencies/local.mk +++ b/src/resolve-system-dependencies/local.mk @@ -6,6 +6,8 @@ resolve-system-dependencies_DIR := $(d) resolve-system-dependencies_INSTALL_DIR := $(libexecdir)/nix -resolve-system-dependencies_LIBS := libstore libmain libutil libformat +resolve-system-dependencies_CXXFLAGS += -I src/libutil -I src/libstore -I src/libmain + +resolve-system-dependencies_LIBS := libstore libmain libutil libnixrust resolve-system-dependencies_SOURCES := $(d)/resolve-system-dependencies.cc diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc index ae8ca36ba9d..434ad80a662 100644 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc @@ -17,75 +17,90 @@ using namespace nix; static auto cacheDir = Path{}; -Path resolveCacheFile(Path lib) { +Path resolveCacheFile(Path lib) +{ std::replace(lib.begin(), lib.end(), '/', '%'); return cacheDir + "/" + lib; } -std::set readCacheFile(const Path & file) { +std::set readCacheFile(const Path & file) +{ return tokenizeString>(readFile(file), "\n"); } -void writeCacheFile(const Path & file, std::set & deps) { - std::ofstream fp; - fp.open(file); - for (auto & d : deps) { - fp << d << "\n"; +std::set runResolver(const Path & filename) +{ + AutoCloseFD fd = open(filename.c_str(), O_RDONLY); + if (!fd) + throw SysError("opening '%s'", filename); + + struct stat st; + if (fstat(fd.get(), &st)) + throw SysError("statting '%s'", filename); + + if (!S_ISREG(st.st_mode)) { + logError({ + .name = "Regular MACH file", + .hint = hintfmt("file '%s' is not a regular file", filename) + }); + return {}; } - fp.close(); -} -std::string findDylibName(bool should_swap, ptrdiff_t dylib_command_start) { - struct dylib_command *dylc = (struct dylib_command*)dylib_command_start; - return std::string((char*)(dylib_command_start + DO_SWAP(should_swap, dylc->dylib.name.offset))); -} + if (st.st_size < sizeof(mach_header_64)) { + logError({ + .name = "File too short", + .hint = hintfmt("file '%s' is too short for a MACH binary", filename) + }); + return {}; + } -std::set runResolver(const Path & filename) { - int fd = open(filename.c_str(), O_RDONLY); - struct stat s; - fstat(fd, &s); - void *obj = mmap(NULL, s.st_size, PROT_READ, MAP_SHARED, fd, 0); + char* obj = (char*) mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd.get(), 0); + if (!obj) + throw SysError("mmapping '%s'", filename); ptrdiff_t mach64_offset = 0; - uint32_t magic = ((struct mach_header_64*)obj)->magic; - if(magic == FAT_CIGAM || magic == FAT_MAGIC) { + uint32_t magic = ((mach_header_64*) obj)->magic; + if (magic == FAT_CIGAM || magic == FAT_MAGIC) { bool should_swap = magic == FAT_CIGAM; - uint32_t narches = DO_SWAP(should_swap, ((struct fat_header*)obj)->nfat_arch); - - for(uint32_t iter = 0; iter < narches; iter++) { - ptrdiff_t header_offset = (ptrdiff_t)obj + sizeof(struct fat_header) * (iter + 1); - struct fat_arch* arch = (struct fat_arch*)header_offset; - if(DO_SWAP(should_swap, arch->cputype) == CPU_TYPE_X86_64) { - mach64_offset = (ptrdiff_t)DO_SWAP(should_swap, arch->offset); + uint32_t narches = DO_SWAP(should_swap, ((fat_header *) obj)->nfat_arch); + for (uint32_t i = 0; i < narches; i++) { + fat_arch* arch = (fat_arch*) (obj + sizeof(fat_header) + sizeof(fat_arch) * i); + if (DO_SWAP(should_swap, arch->cputype) == CPU_TYPE_X86_64) { + mach64_offset = (ptrdiff_t) DO_SWAP(should_swap, arch->offset); break; } } if (mach64_offset == 0) { - printError(format("Could not find any mach64 blobs in file ‘%1%’, continuing...") % filename); - return std::set(); + logError({ + .name = "No mach64 blobs", + .hint = hintfmt("Could not find any mach64 blobs in file '%1%', continuing...", filename) + }); + return {}; } } else if (magic == MH_MAGIC_64 || magic == MH_CIGAM_64) { mach64_offset = 0; } else { - printError(format("Object file has unknown magic number ‘%1%’, skipping it...") % magic); - return std::set(); + logError({ + .name = "Magic number", + .hint = hintfmt("Object file has unknown magic number '%1%', skipping it...", magic) + }); + return {}; } - ptrdiff_t mach_header_offset = (ptrdiff_t)obj + mach64_offset; - struct mach_header_64 *m_header = (struct mach_header_64 *)mach_header_offset; + mach_header_64 * m_header = (mach_header_64 *) (obj + mach64_offset); bool should_swap = magic == MH_CIGAM_64; - ptrdiff_t cmd_offset = mach_header_offset + sizeof(struct mach_header_64); + ptrdiff_t cmd_offset = mach64_offset + sizeof(mach_header_64); std::set libs; - for(uint32_t i = 0; i < DO_SWAP(should_swap, m_header->ncmds); i++) { - struct load_command *cmd = (struct load_command*)cmd_offset; + for (uint32_t i = 0; i < DO_SWAP(should_swap, m_header->ncmds); i++) { + load_command * cmd = (load_command *) (obj + cmd_offset); switch(DO_SWAP(should_swap, cmd->cmd)) { case LC_LOAD_UPWARD_DYLIB: case LC_LOAD_DYLIB: case LC_REEXPORT_DYLIB: - libs.insert(findDylibName(should_swap, cmd_offset)); + libs.insert(std::string((char *) cmd + ((dylib_command*) cmd)->dylib.name.offset)); break; } cmd_offset += DO_SWAP(should_swap, cmd->cmdsize); @@ -94,31 +109,27 @@ std::set runResolver(const Path & filename) { return libs; } -bool isSymlink(const Path & path) { +bool isSymlink(const Path & path) +{ struct stat st; - if(lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path ‘%1%’") % path); + if (lstat(path.c_str(), &st) == -1) + throw SysError("getting attributes of path '%1%'", path); return S_ISLNK(st.st_mode); } -Path resolveSymlink(const Path & path) { - char buf[PATH_MAX]; - ssize_t len = readlink(path.c_str(), buf, sizeof(buf) - 1); - if(len != -1) { - buf[len] = 0; - return Path(buf); - } else { - throw SysError(format("readlink('%1%')") % path); - } +Path resolveSymlink(const Path & path) +{ + auto target = readLink(path); + return hasPrefix(target, "/") + ? target + : dirOf(path) + "/" + target; } -std::set resolveTree(const Path & path, PathSet & deps) { +std::set resolveTree(const Path & path, PathSet & deps) +{ std::set results; - if(deps.find(path) != deps.end()) { - return std::set(); - } - deps.insert(path); + if (!deps.insert(path).second) return {}; for (auto & lib : runResolver(path)) { results.insert(lib); for (auto & p : resolveTree(lib, deps)) { @@ -128,32 +139,33 @@ std::set resolveTree(const Path & path, PathSet & deps) { return results; } -std::set getPath(const Path & path) { +std::set getPath(const Path & path) +{ + if (hasPrefix(path, "/dev")) return {}; + Path cacheFile = resolveCacheFile(path); - if(pathExists(cacheFile)) { + if (pathExists(cacheFile)) return readCacheFile(cacheFile); - } - std::set deps; - std::set paths; + std::set deps, paths; paths.insert(path); - Path next_path = Path(path); - while(isSymlink(next_path)) { - next_path = resolveSymlink(next_path); - paths.insert(next_path); + Path nextPath(path); + while (isSymlink(nextPath)) { + nextPath = resolveSymlink(nextPath); + paths.insert(nextPath); } - for(auto & t : resolveTree(next_path, deps)) { + for (auto & t : resolveTree(nextPath, deps)) paths.insert(t); - } - writeCacheFile(cacheFile, paths); + writeFile(cacheFile, concatStringsSep("\n", paths)); return paths; } -int main(int argc, char ** argv) { +int main(int argc, char ** argv) +{ return handleExceptions(argv[0], [&]() { initNix(); @@ -174,21 +186,25 @@ int main(int argc, char ** argv) { auto store = openStore(); - auto drv = store->derivationFromPath(Path(argv[1])); - Strings impurePaths = tokenizeString(get(drv.env, "__impureHostDeps")); - - std::set all_paths; + StringSet impurePaths; - for (auto & path : impurePaths) { - for(auto & p : getPath(path)) { - all_paths.insert(p); - } + if (std::string(argv[1]) == "--test") + impurePaths.insert(argv[2]); + else { + auto drv = store->derivationFromPath(store->parseStorePath(argv[1])); + impurePaths = tokenizeString(get(drv.env, "__impureHostDeps").value_or("")); + impurePaths.insert("/usr/lib/libSystem.dylib"); } + std::set allPaths; + + for (auto & path : impurePaths) + for (auto & p : getPath(path)) + allPaths.insert(p); + std::cout << "extra-chroot-dirs" << std::endl; - for(auto & path : all_paths) { + for (auto & path : allPaths) std::cout << path << std::endl; - } std::cout << std::endl; }); } diff --git a/tests/bad.tar.xz b/tests/bad.tar.xz new file mode 100644 index 00000000000..250a5ad1a79 Binary files /dev/null and b/tests/bad.tar.xz differ diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 4ce428f643e..17b63d978e7 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -6,7 +6,7 @@ clearCache # Create the binary cache. outPath=$(nix-build dependencies.nix --no-out-link) -nix copy --recursive --to file://$cacheDir $outPath +nix copy --to file://$cacheDir $outPath basicTests() { @@ -16,9 +16,9 @@ basicTests() { clearStore clearCacheCache - nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "---" + nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "---" - nix-store --option binary-caches "file://$cacheDir" -r $outPath + nix-store --substituters "file://$cacheDir" --no-require-sigs -r $outPath [ -x $outPath/program ] @@ -28,13 +28,13 @@ basicTests() { clearCacheCache echo "WantMassQuery: 1" >> $cacheDir/nix-cache-info - nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S" - nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S" + nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S" + nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S" x=$(nix-env -f dependencies.nix -qas \* --prebuilt-only) [ -z "$x" ] - nix-store --option binary-caches "file://$cacheDir" -r $outPath + nix-store --substituters "file://$cacheDir" --no-require-sigs -r $outPath nix-store --check-validity $outPath nix-store -qR $outPath | grep input-2 @@ -48,13 +48,10 @@ basicTests # Test HttpBinaryCacheStore. -export _NIX_FORCE_HTTP_BINARY_CACHE_STORE=1 +export _NIX_FORCE_HTTP=1 basicTests -unset _NIX_FORCE_HTTP_BINARY_CACHE_STORE - - # Test whether Nix notices if the NAR doesn't match the hash in the NAR info. clearStore @@ -63,7 +60,7 @@ mv $nar $nar.good mkdir -p $TEST_ROOT/empty nix-store --dump $TEST_ROOT/empty | xz > $nar -nix-build --option binary-caches "file://$cacheDir" dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log grep -q "hash mismatch" $TEST_ROOT/log mv $nar.good $nar @@ -73,40 +70,66 @@ mv $nar.good $nar clearStore clearCacheCache -if nix-store --option binary-caches "file://$cacheDir" --option signed-binary-caches '*' -r $outPath; then +if nix-store --substituters "file://$cacheDir" -r $outPath; then echo "unsigned binary cache incorrectly accepted" exit 1 fi -# Test whether fallback works if we have cached info but the -# corresponding NAR has disappeared. +# Test whether fallback works if a NAR has disappeared. This does not require --fallback. +clearStore + +mv $cacheDir/nar $cacheDir/nar2 + +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result + +mv $cacheDir/nar2 $cacheDir/nar + + +# Test whether fallback works if a NAR is corrupted. This does require --fallback. clearStore -nix-build --option binary-caches "file://$cacheDir" dependencies.nix --dry-run # get info +mv $cacheDir/nar $cacheDir/nar2 +mkdir $cacheDir/nar +for i in $(cd $cacheDir/nar2 && echo *); do touch $cacheDir/nar/$i; done -mkdir $cacheDir/tmp -mv $cacheDir/*.nar* $cacheDir/tmp/ +(! nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result) -NIX_DEBUG_SUBST=1 nix-build --option binary-caches "file://$cacheDir" dependencies.nix -o $TEST_ROOT/result --fallback +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result --fallback -mv $cacheDir/tmp/* $cacheDir/ +rm -rf $cacheDir/nar +mv $cacheDir/nar2 $cacheDir/nar # Test whether building works if the binary cache contains an # incomplete closure. clearStore -rm $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo) +rm -v $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo) + +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log +grep -q "copying path.*input-0" $TEST_ROOT/log +grep -q "copying path.*input-2" $TEST_ROOT/log +grep -q "copying path.*top" $TEST_ROOT/log -nix-build --option binary-caches "file://$cacheDir" dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log -grep -q "fetching path" $TEST_ROOT/log + +# Idem, but without cached .narinfo. +clearStore +clearCacheCache + +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log +grep -q "don't know how to build" $TEST_ROOT/log +grep -q "building.*input-1" $TEST_ROOT/log +grep -q "building.*input-2" $TEST_ROOT/log +grep -q "copying path.*input-0" $TEST_ROOT/log +grep -q "copying path.*top" $TEST_ROOT/log if [ -n "$HAVE_SODIUM" ]; then # Create a signed binary cache. clearCache +clearCacheCache declare -a res=($(nix-store --generate-binary-cache-key test.nixos.org-1 $TEST_ROOT/sk1 $TEST_ROOT/pk1 )) publicKey="$(cat $TEST_ROOT/pk1)" @@ -117,25 +140,25 @@ badKey="$(cat $TEST_ROOT/pk2)" res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3)) otherKey="$(cat $TEST_ROOT/pk3)" -nix copy --recursive --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath +_NIX_FORCE_HTTP= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath # Downloading should fail if we don't provide a key. clearStore clearCacheCache -(! nix-store -r $outPath --option binary-caches "file://$cacheDir" --option signed-binary-caches '*' ) +(! nix-store -r $outPath --substituters "file://$cacheDir") # And it should fail if we provide an incorrect key. clearStore clearCacheCache -(! nix-store -r $outPath --option binary-caches "file://$cacheDir" --option signed-binary-caches '*' --option binary-cache-public-keys "$badKey") +(! nix-store -r $outPath --substituters "file://$cacheDir" --trusted-public-keys "$badKey") # It should succeed if we provide the correct key. -nix-store -r $outPath --option binary-caches "file://$cacheDir" --option signed-binary-caches '*' --option binary-cache-public-keys "$otherKey $publicKey" +nix-store -r $outPath --substituters "file://$cacheDir" --trusted-public-keys "$otherKey $publicKey" # It should fail if we corrupt the .narinfo. @@ -152,10 +175,10 @@ done clearCacheCache -(! nix-store -r $outPath --option binary-caches "file://$cacheDir2" --option signed-binary-caches '*' --option binary-cache-public-keys "$publicKey") +(! nix-store -r $outPath --substituters "file://$cacheDir2" --trusted-public-keys "$publicKey") # If we provide a bad and a good binary cache, it should succeed. -nix-store -r $outPath --option binary-caches "file://$cacheDir2 file://$cacheDir" --option signed-binary-caches '*' --option binary-cache-public-keys "$publicKey" +nix-store -r $outPath --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey" fi # HAVE_LIBSODIUM diff --git a/tests/brotli.sh b/tests/brotli.sh new file mode 100644 index 00000000000..a3c6e55a8fa --- /dev/null +++ b/tests/brotli.sh @@ -0,0 +1,21 @@ +source common.sh + +clearStore +clearCache + +cacheURI="file://$cacheDir?compression=br" + +outPath=$(nix-build dependencies.nix --no-out-link) + +nix copy --to $cacheURI $outPath + +HASH=$(nix hash-path $outPath) + +clearStore +clearCacheCache + +nix copy --from $cacheURI $outPath --no-check-sigs + +HASH2=$(nix hash-path $outPath) + +[[ $HASH = $HASH2 ]] diff --git a/tests/build-dry.sh b/tests/build-dry.sh new file mode 100644 index 00000000000..e72533e7061 --- /dev/null +++ b/tests/build-dry.sh @@ -0,0 +1,52 @@ +source common.sh + +################################################### +# Check that --dry-run isn't confused with read-only mode +# https://github.com/NixOS/nix/issues/1795 + +clearStore +clearCache + +# Ensure this builds successfully first +nix build --no-link -f dependencies.nix + +clearStore +clearCache + +# Try --dry-run using old command first +nix-build --no-out-link dependencies.nix --dry-run 2>&1 | grep "will be built" +# Now new command: +nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built" + +# TODO: XXX: FIXME: #1793 +# Disable this part of the test until the problem is resolved: +if [ -n "$ISSUE_1795_IS_FIXED" ]; then +clearStore +clearCache + +# Try --dry-run using new command first +nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built" +# Now old command: +nix-build --no-out-link dependencies.nix --dry-run 2>&1 | grep "will be built" +fi + +################################################### +# Check --dry-run doesn't create links with --dry-run +# https://github.com/NixOS/nix/issues/1849 +clearStore +clearCache + +RESULT=$TEST_ROOT/result-link +rm -f $RESULT + +nix-build dependencies.nix -o $RESULT --dry-run + +[[ ! -h $RESULT ]] || fail "nix-build --dry-run created output link" + +nix build -f dependencies.nix -o $RESULT --dry-run + +[[ ! -h $RESULT ]] || fail "nix build --dry-run created output link" + +nix build -f dependencies.nix -o $RESULT + +[[ -h $RESULT ]] diff --git a/tests/build-hook.hook.sh b/tests/build-hook.hook.sh deleted file mode 100755 index c7472eab760..00000000000 --- a/tests/build-hook.hook.sh +++ /dev/null @@ -1,23 +0,0 @@ -#! /bin/sh - -#set -x - -while read x y drv rest; do - - echo "HOOK for $drv" >&2 - - outPath=`sed 's/Derive(\[("out",\"\([^\"]*\)\".*/\1/' $drv` - - echo "output path is $outPath" >&2 - - if `echo $outPath | grep -q input-1`; then - echo "# accept" >&2 - read inputs - read outputs - mkdir $outPath - echo "BAR" > $outPath/foo - else - echo "# decline" >&2 - fi - -done diff --git a/tests/build-hook.nix b/tests/build-hook.nix index 666cc6ef804..8c5ca8cd3c7 100644 --- a/tests/build-hook.nix +++ b/tests/build-hook.nix @@ -4,12 +4,13 @@ let input1 = mkDerivation { name = "build-hook-input-1"; - builder = ./dependencies.builder1.sh; + buildCommand = "mkdir $out; echo FOO > $out/foo"; + requiredSystemFeatures = ["foo"]; }; input2 = mkDerivation { name = "build-hook-input-2"; - builder = ./dependencies.builder2.sh; + buildCommand = "mkdir $out; echo BAR > $out/bar"; }; in diff --git a/tests/build-hook.sh b/tests/build-hook.sh deleted file mode 100644 index ef77a3ae528..00000000000 --- a/tests/build-hook.sh +++ /dev/null @@ -1,10 +0,0 @@ -source common.sh - -export NIX_BUILD_HOOK="$(pwd)/build-hook.hook.sh" - -outPath=$(nix-build build-hook.nix --no-out-link) - -echo "output path is $outPath" - -text=$(cat "$outPath"/foobar) -if test "$text" != "BARBAR"; then exit 1; fi diff --git a/tests/build-remote.sh b/tests/build-remote.sh new file mode 100644 index 00000000000..a550f4460de --- /dev/null +++ b/tests/build-remote.sh @@ -0,0 +1,24 @@ +source common.sh + +clearStore + +if ! canUseSandbox; then exit; fi +if [[ ! $SHELL =~ /nix/store ]]; then exit; fi + +chmod -R u+w $TEST_ROOT/store0 || true +chmod -R u+w $TEST_ROOT/store1 || true +rm -rf $TEST_ROOT/store0 $TEST_ROOT/store1 + +nix build -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \ + --sandbox-paths /nix/store --sandbox-build-dir /build-tmp \ + --builders "$TEST_ROOT/store0; $TEST_ROOT/store1 - - 1 1 foo" \ + --system-features foo + +outPath=$TEST_ROOT/result + +cat $outPath/foobar | grep FOOBAR + +# Ensure that input1 was built on store1 due to the required feature. +p=$(readlink -f $outPath/input-2) +(! nix path-info --store $TEST_ROOT/store0 --all | grep builder-build-hook-input-1.sh) +nix path-info --store $TEST_ROOT/store1 --all | grep builder-build-hook-input-1.sh diff --git a/tests/check-refs.sh b/tests/check-refs.sh index 34ee22cfc8f..16bbabc4098 100644 --- a/tests/check-refs.sh +++ b/tests/check-refs.sh @@ -1,5 +1,7 @@ source common.sh +clearStore + RESULT=$TEST_ROOT/result dep=$(nix-build -o $RESULT check-refs.nix -A dep) diff --git a/tests/check-reqs.sh b/tests/check-reqs.sh index 77689215def..e9f65fc2a6d 100644 --- a/tests/check-reqs.sh +++ b/tests/check-reqs.sh @@ -1,5 +1,7 @@ source common.sh +clearStore + RESULT=$TEST_ROOT/result nix-build -o $RESULT check-reqs.nix -A test1 diff --git a/tests/check.nix b/tests/check.nix new file mode 100644 index 00000000000..bca04fdafa1 --- /dev/null +++ b/tests/check.nix @@ -0,0 +1,55 @@ +{checkBuildId ? 0}: + +with import ./config.nix; + +{ + nondeterministic = mkDerivation { + inherit checkBuildId; + name = "nondeterministic"; + buildCommand = + '' + mkdir $out + date +%s.%N > $out/date + echo "CHECK_TMPDIR=$TMPDIR" + echo "checkBuildId=$checkBuildId" + echo "$checkBuildId" > $TMPDIR/checkBuildId + ''; + }; + + deterministic = mkDerivation { + inherit checkBuildId; + name = "deterministic"; + buildCommand = + '' + mkdir $out + echo date > $out/date + echo "CHECK_TMPDIR=$TMPDIR" + echo "checkBuildId=$checkBuildId" + echo "$checkBuildId" > $TMPDIR/checkBuildId + ''; + }; + + failed = mkDerivation { + inherit checkBuildId; + name = "failed"; + buildCommand = + '' + mkdir $out + echo date > $out/date + echo "CHECK_TMPDIR=$TMPDIR" + echo "checkBuildId=$checkBuildId" + echo "$checkBuildId" > $TMPDIR/checkBuildId + false + ''; + }; + + hashmismatch = import { + url = "file://" + toString ./dummy; + sha256 = "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73"; + }; + + fetchurl = import { + url = "file://" + toString ./lang/eval-okay-xml.exp.xml; + sha256 = "0kg4sla7ihm8ijr8cb3117fhl99zrc2bwy1jrngsfmkh8bav4m0v"; + }; +} diff --git a/tests/check.sh b/tests/check.sh new file mode 100644 index 00000000000..5f25d04cb2c --- /dev/null +++ b/tests/check.sh @@ -0,0 +1,90 @@ +source common.sh + +checkBuildTempDirRemoved () +{ + buildDir=$(sed -n 's/CHECK_TMPDIR=//p' $1 | head -1) + checkBuildIdFile=${buildDir}/checkBuildId + [[ ! -f $checkBuildIdFile ]] || ! grep $checkBuildId $checkBuildIdFile +} + +# written to build temp directories to verify created by this instance +checkBuildId=$(date +%s%N) + +clearStore + +nix-build dependencies.nix --no-out-link +nix-build dependencies.nix --no-out-link --check + +# check for dangling temporary build directories +# only retain if build fails and --keep-failed is specified, or... +# ...build is non-deterministic and --check and --keep-failed are both specified +nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \ + --no-out-link 2> $TEST_ROOT/log || status=$? +[ "$status" = "100" ] +checkBuildTempDirRemoved $TEST_ROOT/log + +nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \ + --no-out-link --keep-failed 2> $TEST_ROOT/log || status=$? +[ "$status" = "100" ] +if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi + +nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \ + --no-out-link 2> $TEST_ROOT/log +checkBuildTempDirRemoved $TEST_ROOT/log + +nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \ + --no-out-link --check --keep-failed 2> $TEST_ROOT/log +if grep -q 'may not be deterministic' $TEST_ROOT/log; then false; fi +checkBuildTempDirRemoved $TEST_ROOT/log + +nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ + --no-out-link 2> $TEST_ROOT/log +checkBuildTempDirRemoved $TEST_ROOT/log + +nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ + --no-out-link --check 2> $TEST_ROOT/log || status=$? +grep 'may not be deterministic' $TEST_ROOT/log +[ "$status" = "104" ] +checkBuildTempDirRemoved $TEST_ROOT/log + +nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ + --no-out-link --check --keep-failed 2> $TEST_ROOT/log || status=$? +grep 'may not be deterministic' $TEST_ROOT/log +[ "$status" = "104" ] +if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi + +clearStore + +nix-build dependencies.nix --no-out-link --repeat 3 + +nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/log || status=$? +[ "$status" = "1" ] +grep 'differs from previous round' $TEST_ROOT/log + +path=$(nix-build check.nix -A fetchurl --no-out-link --hashed-mirrors '') + +chmod +w $path +echo foo > $path +chmod -w $path + +nix-build check.nix -A fetchurl --no-out-link --check --hashed-mirrors '' +# Note: "check" doesn't repair anything, it just compares to the hash stored in the database. +[[ $(cat $path) = foo ]] + +nix-build check.nix -A fetchurl --no-out-link --repair --hashed-mirrors '' +[[ $(cat $path) != foo ]] + +nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' || status=$? +[ "$status" = "102" ] + +echo -n > ./dummy +nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' +echo 'Hello World' > ./dummy + +nix-build check.nix -A hashmismatch --no-out-link --check --hashed-mirrors '' || status=$? +[ "$status" = "102" ] + +# Multiple failures with --keep-going +nix-build check.nix -A nondeterministic --no-out-link +nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going --hashed-mirrors '' || status=$? +[ "$status" = "110" ] diff --git a/tests/common.sh.in b/tests/common.sh.in index 4565a490adf..dd7e6182290 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -1,7 +1,5 @@ set -e -datadir="@datadir@" - export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test) export NIX_STORE_DIR if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then @@ -13,18 +11,25 @@ export NIX_LOCALSTATE_DIR=$TEST_ROOT/var export NIX_LOG_DIR=$TEST_ROOT/var/log/nix export NIX_STATE_DIR=$TEST_ROOT/var/nix export NIX_CONF_DIR=$TEST_ROOT/etc -export NIX_MANIFESTS_DIR=$TEST_ROOT/var/nix/manifests +unset NIX_USER_CONF_FILES export _NIX_TEST_SHARED=$TEST_ROOT/shared +if [[ -n $NIX_STORE ]]; then + export _NIX_TEST_NO_SANDBOX=1 +fi +export _NIX_IN_TEST=$TEST_ROOT/shared +export _NIX_TEST_NO_LSOF=1 export NIX_REMOTE=$NIX_REMOTE_ unset NIX_PATH export TEST_HOME=$TEST_ROOT/test-home export HOME=$TEST_HOME +unset XDG_CONFIG_HOME +unset XDG_CONFIG_DIRS +unset XDG_CACHE_HOME mkdir -p $TEST_HOME export PATH=@bindir@:$PATH coreutils=@coreutils@ -export NIX_BUILD_HOOK= export dot=@dot@ export xmllint="@xmllint@" export SHELL="@bash@" @@ -84,9 +89,33 @@ killDaemon() { trap "" EXIT } +if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then + _canUseSandbox=1 +fi + +canUseSandbox() { + if [[ ! $_canUseSandbox ]]; then + echo "Sandboxing not supported, skipping this test..." + return 1 + fi + + return 0 +} + fail() { echo "$1" exit 1 } +expect() { + local expected res + expected="$1" + shift + set +e + "$@" + res="$?" + set -e + [[ $res -eq $expected ]] +} + set -x diff --git a/tests/config.nix b/tests/config.nix.in similarity index 52% rename from tests/config.nix rename to tests/config.nix.in index 76388fdd5b9..a57a8c5962e 100644 --- a/tests/config.nix +++ b/tests/config.nix.in @@ -1,11 +1,9 @@ -with import ; - rec { - inherit shell; + shell = "@bash@"; - path = coreutils; + path = "@coreutils@"; - system = builtins.currentSystem; + system = "@system@"; shared = builtins.getEnv "_NIX_TEST_SHARED"; @@ -13,7 +11,7 @@ rec { derivation ({ inherit system; builder = shell; - args = ["-e" args.builder or (builtins.toFile "builder.sh" "eval \"$buildCommand\"")]; + args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; PATH = path; } // removeAttrs args ["builder" "meta"]) // { meta = args.meta or {}; }; diff --git a/tests/config.sh b/tests/config.sh new file mode 100644 index 00000000000..8fa349f11a3 --- /dev/null +++ b/tests/config.sh @@ -0,0 +1,18 @@ +source common.sh + +# Test that files are loaded from XDG by default +export XDG_CONFIG_HOME=/tmp/home +export XDG_CONFIG_DIRS=/tmp/dir1:/tmp/dir2 +files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | xargs) +[[ $files == "/tmp/home/nix/nix.conf:/tmp/dir1/nix/nix.conf:/tmp/dir2/nix/nix.conf" ]] + +# Test that setting NIX_USER_CONF_FILES overrides all the default user config files +export NIX_USER_CONF_FILES=/tmp/file1.conf:/tmp/file2.conf +files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | xargs) +[[ $files == "/tmp/file1.conf:/tmp/file2.conf" ]] + +# Test that it's possible to load the config from a custom location +here=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") +export NIX_USER_CONF_FILES=$here/config/nix-with-substituters.conf +var=$(nix show-config | grep '^substituters =' | cut -d '=' -f 2 | xargs) +[[ $var == https://example.com ]] diff --git a/tests/config/nix-with-substituters.conf b/tests/config/nix-with-substituters.conf new file mode 100644 index 00000000000..90f359a6fa9 --- /dev/null +++ b/tests/config/nix-with-substituters.conf @@ -0,0 +1,2 @@ +experimental-features = nix-command +substituters = https://example.com diff --git a/tests/dependencies.builder1.sh b/tests/dependencies.builder1.sh deleted file mode 100644 index 4b006a17d70..00000000000 --- a/tests/dependencies.builder1.sh +++ /dev/null @@ -1,2 +0,0 @@ -mkdir $out -echo FOO > $out/foo diff --git a/tests/dependencies.builder2.sh b/tests/dependencies.builder2.sh deleted file mode 100644 index 4f886fdb3a1..00000000000 --- a/tests/dependencies.builder2.sh +++ /dev/null @@ -1,2 +0,0 @@ -mkdir $out -echo BAR > $out/bar diff --git a/tests/dependencies.nix b/tests/dependencies.nix index 687237add82..e320d81c92e 100644 --- a/tests/dependencies.nix +++ b/tests/dependencies.nix @@ -2,21 +2,31 @@ with import ./config.nix; let { + input0 = mkDerivation { + name = "dependencies-input-0"; + buildCommand = "mkdir $out; echo foo > $out/bar"; + }; + input1 = mkDerivation { name = "dependencies-input-1"; - builder = ./dependencies.builder1.sh; + buildCommand = "mkdir $out; echo FOO > $out/foo"; }; input2 = mkDerivation { name = "dependencies-input-2"; - builder = "${./dependencies.builder2.sh}"; + buildCommand = '' + mkdir $out + echo BAR > $out/bar + echo ${input0} > $out/input0 + ''; }; body = mkDerivation { - name = "dependencies"; + name = "dependencies-top"; builder = ./dependencies.builder0.sh + "/FOOBAR/../."; input1 = input1 + "/."; input2 = "${input2}/."; + input1_drv = input1; meta.description = "Random test package"; }; diff --git a/tests/dependencies.sh b/tests/dependencies.sh index df204d185dd..092950aa7ed 100644 --- a/tests/dependencies.sh +++ b/tests/dependencies.sh @@ -6,7 +6,7 @@ drvPath=$(nix-instantiate dependencies.nix) echo "derivation is $drvPath" -nix-store -q --tree "$drvPath" | grep ' +---.*builder1.sh' +nix-store -q --tree "$drvPath" | grep '───.*builder-dependencies-input-1.sh' # Test Graphviz graph generation. nix-store -q --graph "$drvPath" > $TEST_ROOT/graph @@ -22,9 +22,9 @@ nix-store -q --graph "$outPath" > $TEST_ROOT/graph if test -n "$dot"; then # Does it parse? $dot < $TEST_ROOT/graph -fi +fi -nix-store -q --tree "$outPath" | grep '+---.*dependencies-input-2' +nix-store -q --tree "$outPath" | grep '───.*dependencies-input-2' echo "output path is $outPath" @@ -49,4 +49,4 @@ nix-store -q --referrers-closure "$input2OutPath" | grep "$outPath" # Check that the derivers are set properly. test $(nix-store -q --deriver "$outPath") = "$drvPath" -nix-store -q --deriver "$input2OutPath" | grep -q -- "-input-2.drv" +nix-store -q --deriver "$input2OutPath" | grep -q -- "-input-2.drv" diff --git a/tests/export-graph.sh b/tests/export-graph.sh index a6fd6905442..a1449b34ef2 100644 --- a/tests/export-graph.sh +++ b/tests/export-graph.sh @@ -11,7 +11,7 @@ checkRef() { outPath=$(nix-build ./export-graph.nix -A 'foo."bar.runtimeGraph"' -o $TEST_ROOT/result) -test $(nix-store -q --references $TEST_ROOT/result | wc -l) = 2 || fail "bad nr of references" +test $(nix-store -q --references $TEST_ROOT/result | wc -l) = 3 || fail "bad nr of references" checkRef input-2 for i in $(cat $outPath); do checkRef $i; done diff --git a/tests/export.sh b/tests/export.sh index ec7560f1972..2238539bcca 100644 --- a/tests/export.sh +++ b/tests/export.sh @@ -8,6 +8,11 @@ nix-store --export $outPath > $TEST_ROOT/exp nix-store --export $(nix-store -qR $outPath) > $TEST_ROOT/exp_all +if nix-store --export $outPath >/dev/full ; then + echo "exporting to a bad file descriptor should fail" + exit 1 +fi + clearStore diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh new file mode 100644 index 00000000000..d9c9874f568 --- /dev/null +++ b/tests/fetchGit.sh @@ -0,0 +1,159 @@ +source common.sh + +if [[ -z $(type -p git) ]]; then + echo "Git not installed; skipping Git tests" + exit 99 +fi + +clearStore + +repo=$TEST_ROOT/git + +export _NIX_FORCE_HTTP=1 + +rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix $TEST_ROOT/worktree $TEST_ROOT/shallow + +git init $repo +git -C $repo config user.email "foobar@example.com" +git -C $repo config user.name "Foobar" + +echo utrecht > $repo/hello +touch $repo/.gitignore +git -C $repo add hello .gitignore +git -C $repo commit -m 'Bla1' +rev1=$(git -C $repo rev-parse HEAD) + +echo world > $repo/hello +git -C $repo commit -m 'Bla2' -a +git -C $repo worktree add $TEST_ROOT/worktree +echo hello >> $TEST_ROOT/worktree/hello +rev2=$(git -C $repo rev-parse HEAD) + +# Fetch a worktree +unset _NIX_FORCE_HTTP +path0=$(nix eval --raw "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath") +export _NIX_FORCE_HTTP=1 +[[ $(tail -n 1 $path0/hello) = "hello" ]] + +# Fetch the default branch. +path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") +[[ $(cat $path/hello) = world ]] + +# In pure eval mode, fetchGit without a revision should fail. +[[ $(nix eval --raw "(builtins.readFile (fetchGit file://$repo + \"/hello\"))") = world ]] +(! nix eval --pure-eval --raw "(builtins.readFile (fetchGit file://$repo + \"/hello\"))") + +# Fetch using an explicit revision hash. +path2=$(nix eval --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath") +[[ $path = $path2 ]] + +# In pure eval mode, fetchGit with a revision should succeed. +[[ $(nix eval --pure-eval --raw "(builtins.readFile (fetchGit { url = file://$repo; rev = \"$rev2\"; } + \"/hello\"))") = world ]] + +# Fetch again. This should be cached. +mv $repo ${repo}-tmp +path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") +[[ $path = $path2 ]] + +[[ $(nix eval "(builtins.fetchGit file://$repo).revCount") = 2 ]] +[[ $(nix eval --raw "(builtins.fetchGit file://$repo).rev") = $rev2 ]] + +# Fetching with a explicit hash should succeed. +path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath") +[[ $path = $path2 ]] + +path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev1\"; }).outPath") +[[ $(cat $path2/hello) = utrecht ]] + +mv ${repo}-tmp $repo + +# Using a clean working tree should produce the same result. +path2=$(nix eval --raw "(builtins.fetchGit $repo).outPath") +[[ $path = $path2 ]] + +# Using an unclean tree should yield the tracked but uncommitted changes. +mkdir $repo/dir1 $repo/dir2 +echo foo > $repo/dir1/foo +echo bar > $repo/bar +echo bar > $repo/dir2/bar +git -C $repo add dir1/foo +git -C $repo rm hello + +unset _NIX_FORCE_HTTP +path2=$(nix eval --raw "(builtins.fetchGit $repo).outPath") +[ ! -e $path2/hello ] +[ ! -e $path2/bar ] +[ ! -e $path2/dir2/bar ] +[ ! -e $path2/.git ] +[[ $(cat $path2/dir1/foo) = foo ]] + +[[ $(nix eval --raw "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] + +# ... unless we're using an explicit ref or rev. +path3=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") +[[ $path = $path3 ]] + +path3=$(nix eval --raw "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; }).outPath") +[[ $path = $path3 ]] + +# Committing should not affect the store path. +git -C $repo commit -m 'Bla3' -a + +path4=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit file://$repo).outPath") +[[ $path2 = $path4 ]] + +# tarball-ttl should be ignored if we specify a rev +echo delft > $repo/hello +git -C $repo add hello +git -C $repo commit -m 'Bla4' +rev3=$(git -C $repo rev-parse HEAD) +nix eval --tarball-ttl 3600 "(builtins.fetchGit { url = $repo; rev = \"$rev3\"; })" >/dev/null + +# Update 'path' to reflect latest master +path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") + +# Check behavior when non-master branch is used +git -C $repo checkout $rev2 -b dev +echo dev > $repo/hello + +# File URI uses dirty tree unless specified otherwise +path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") +[ $(cat $path2/hello) = dev ] + +# Using local path with branch other than 'master' should work when clean or dirty +path3=$(nix eval --raw "(builtins.fetchGit $repo).outPath") +# (check dirty-tree handling was used) +[[ $(nix eval --raw "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] + +# Committing shouldn't change store path, or switch to using 'master' +git -C $repo commit -m 'Bla5' -a +path4=$(nix eval --raw "(builtins.fetchGit $repo).outPath") +[[ $(cat $path4/hello) = dev ]] +[[ $path3 = $path4 ]] + +# Confirm same as 'dev' branch +path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") +[[ $path3 = $path5 ]] + + +# Nuke the cache +rm -rf $TEST_HOME/.cache/nix + +# Try again, but without 'git' on PATH. This should fail. +NIX=$(command -v nix) +# This should fail +(! PATH= $NIX eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" ) + +# Try again, with 'git' available. This should work. +path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") +[[ $path3 = $path5 ]] + +# Fetching a shallow repo shouldn't work by default, because we can't +# return a revCount. +git clone --depth 1 file://$repo $TEST_ROOT/shallow +(! nix eval --raw "(builtins.fetchGit { url = $TEST_ROOT/shallow; ref = \"dev\"; }).outPath") + +# But you can request a shallow clone, which won't return a revCount. +path6=$(nix eval --raw "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).outPath") +[[ $path3 = $path6 ]] +[[ $(nix eval "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).revCount or 123") == 123 ]] diff --git a/tests/fetchGitRefs.sh b/tests/fetchGitRefs.sh new file mode 100644 index 00000000000..23934698e3e --- /dev/null +++ b/tests/fetchGitRefs.sh @@ -0,0 +1,111 @@ +source common.sh + +if [[ -z $(type -p git) ]]; then + echo "Git not installed; skipping Git tests" + exit 99 +fi + +clearStore + +repo="$TEST_ROOT/git" + +rm -rf "$repo" "${repo}-tmp" "$TEST_HOME/.cache/nix" + +git init "$repo" +git -C "$repo" config user.email "foobar@example.com" +git -C "$repo" config user.name "Foobar" + +echo utrecht > "$repo"/hello +git -C "$repo" add hello +git -C "$repo" commit -m 'Bla1' + +path=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") + +# Test various combinations of ref names +# (taken from the git project) + +# git help check-ref-format +# Git imposes the following rules on how references are named: +# +# 1. They can include slash / for hierarchical (directory) grouping, but no slash-separated component can begin with a dot . or end with the sequence .lock. +# 2. They must contain at least one /. This enforces the presence of a category like heads/, tags/ etc. but the actual names are not restricted. If the --allow-onelevel option is used, this rule is waived. +# 3. They cannot have two consecutive dots .. anywhere. +# 4. They cannot have ASCII control characters (i.e. bytes whose values are lower than \040, or \177 DEL), space, tilde ~, caret ^, or colon : anywhere. +# 5. They cannot have question-mark ?, asterisk *, or open bracket [ anywhere. See the --refspec-pattern option below for an exception to this rule. +# 6. They cannot begin or end with a slash / or contain multiple consecutive slashes (see the --normalize option below for an exception to this rule) +# 7. They cannot end with a dot .. +# 8. They cannot contain a sequence @{. +# 9. They cannot be the single character @. +# 10. They cannot contain a \. + +valid_ref() { + { set +x; printf >&2 '\n>>>>>>>>>> valid_ref %s\b <<<<<<<<<<\n' $(printf %s "$1" | sed -n -e l); set -x; } + git check-ref-format --branch "$1" >/dev/null + git -C "$repo" branch "$1" master >/dev/null + path1=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath") + [[ $path1 = $path ]] + git -C "$repo" branch -D "$1" >/dev/null +} + +invalid_ref() { + { set +x; printf >&2 '\n>>>>>>>>>> invalid_ref %s\b <<<<<<<<<<\n' $(printf %s "$1" | sed -n -e l); set -x; } + # special case for a sole @: + # --branch @ will try to interpret @ as a branch reference and not fail. Thus we need --allow-onelevel + if [ "$1" = "@" ]; then + (! git check-ref-format --allow-onelevel "$1" >/dev/null 2>&1) + else + (! git check-ref-format --branch "$1" >/dev/null 2>&1) + fi + nix --debug eval --raw "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath" 2>&1 | grep 'invalid Git branch/tag name' >/dev/null +} + + +valid_ref 'foox' +valid_ref '1337' +valid_ref 'foo.baz' +valid_ref 'foo/bar/baz' +valid_ref 'foo./bar' +valid_ref 'heads/foo@bar' +valid_ref "$(printf 'heads/fu\303\237')" +valid_ref 'foo-bar-baz' +valid_ref '$1' +valid_ref 'foo.locke' + +invalid_ref 'refs///heads/foo' +invalid_ref 'heads/foo/' +invalid_ref '///heads/foo' +invalid_ref '.foo' +invalid_ref './foo' +invalid_ref './foo/bar' +invalid_ref 'foo/./bar' +invalid_ref 'foo/bar/.' +invalid_ref 'foo bar' +invalid_ref 'foo?bar' +invalid_ref 'foo^bar' +invalid_ref 'foo~bar' +invalid_ref 'foo:bar' +invalid_ref 'foo[bar' +invalid_ref 'foo/bar/.' +invalid_ref '.refs/foo' +invalid_ref 'refs/heads/foo.' +invalid_ref 'heads/foo..bar' +invalid_ref 'heads/foo?bar' +invalid_ref 'heads/foo.lock' +invalid_ref 'heads///foo.lock' +invalid_ref 'foo.lock/bar' +invalid_ref 'foo.lock///bar' +invalid_ref 'heads/v@{ation' +invalid_ref 'heads/foo\.ar' # should fail due to \ +invalid_ref 'heads/foo\bar' # should fail due to \ +invalid_ref "$(printf 'heads/foo\t')" # should fail because it has a TAB +invalid_ref "$(printf 'heads/foo\177')" +invalid_ref '@' + +invalid_ref 'foo/*' +invalid_ref '*/foo' +invalid_ref 'foo/*/bar' +invalid_ref '*' +invalid_ref 'foo/*/*' +invalid_ref '*/foo/*' +invalid_ref '/foo' +invalid_ref '' diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh new file mode 100644 index 00000000000..4c2c13f1ac6 --- /dev/null +++ b/tests/fetchGitSubmodules.sh @@ -0,0 +1,97 @@ +source common.sh + +set -u + +if [[ -z $(type -p git) ]]; then + echo "Git not installed; skipping Git submodule tests" + exit 99 +fi + +clearStore + +rootRepo=$TEST_ROOT/gitSubmodulesRoot +subRepo=$TEST_ROOT/gitSubmodulesSub + +rm -rf ${rootRepo} ${subRepo} $TEST_HOME/.cache/nix + +initGitRepo() { + git init $1 + git -C $1 config user.email "foobar@example.com" + git -C $1 config user.name "Foobar" +} + +addGitContent() { + echo "lorem ipsum" > $1/content + git -C $1 add content + git -C $1 commit -m "Initial commit" +} + +initGitRepo $subRepo +addGitContent $subRepo + +initGitRepo $rootRepo + +git -C $rootRepo submodule init +git -C $rootRepo submodule add $subRepo sub +git -C $rootRepo add sub +git -C $rootRepo commit -m "Add submodule" + +rev=$(git -C $rootRepo rev-parse HEAD) + +r1=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") +r2=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath") +r3=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") + +[[ $r1 == $r2 ]] +[[ $r2 != $r3 ]] + +r4=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; }).outPath") +r5=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = false; }).outPath") +r6=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") +r7=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") +r8=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).outPath") + +[[ $r1 == $r4 ]] +[[ $r4 == $r5 ]] +[[ $r3 == $r6 ]] +[[ $r6 == $r7 ]] +[[ $r7 == $r8 ]] + +have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; }).submodules") +[[ $have_submodules == false ]] + +have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = false; }).submodules") +[[ $have_submodules == false ]] + +have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules") +[[ $have_submodules == true ]] + +pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") +pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +pathWithSubmodulesAgainWithRef=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") + +# The resulting store path cannot be the same. +[[ $pathWithoutSubmodules != $pathWithSubmodules ]] + +# Checking out the same repo with submodules returns in the same store path. +[[ $pathWithSubmodules == $pathWithSubmodulesAgain ]] + +# Checking out the same repo with submodules returns in the same store path. +[[ $pathWithSubmodulesAgain == $pathWithSubmodulesAgainWithRef ]] + +# The submodules flag is actually honored. +[[ ! -e $pathWithoutSubmodules/sub/content ]] +[[ -e $pathWithSubmodules/sub/content ]] + +[[ -e $pathWithSubmodulesAgainWithRef/sub/content ]] + +# No .git directory or submodule reference files must be left +test "$(find "$pathWithSubmodules" -name .git)" = "" + +# Git repos without submodules can be fetched with submodules = true. +subRev=$(git -C $subRepo rev-parse HEAD) +noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; }).outPath") +noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; submodules = true; }).outPath") + +[[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]] diff --git a/tests/fetchMercurial.sh b/tests/fetchMercurial.sh new file mode 100644 index 00000000000..4088dbd3979 --- /dev/null +++ b/tests/fetchMercurial.sh @@ -0,0 +1,93 @@ +source common.sh + +if [[ -z $(type -p hg) ]]; then + echo "Mercurial not installed; skipping Mercurial tests" + exit 99 +fi + +clearStore + +repo=$TEST_ROOT/hg + +rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/hg + +hg init $repo +echo '[ui]' >> $repo/.hg/hgrc +echo 'username = Foobar ' >> $repo/.hg/hgrc + +echo utrecht > $repo/hello +touch $repo/.hgignore +hg add --cwd $repo hello .hgignore +hg commit --cwd $repo -m 'Bla1' +rev1=$(hg log --cwd $repo -r tip --template '{node}') + +echo world > $repo/hello +hg commit --cwd $repo -m 'Bla2' +rev2=$(hg log --cwd $repo -r tip --template '{node}') + +# Fetch the default branch. +path=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath") +[[ $(cat $path/hello) = world ]] + +# In pure eval mode, fetchGit without a revision should fail. +[[ $(nix eval --raw "(builtins.readFile (fetchMercurial file://$repo + \"/hello\"))") = world ]] +(! nix eval --pure-eval --raw "(builtins.readFile (fetchMercurial file://$repo + \"/hello\"))") + +# Fetch using an explicit revision hash. +path2=$(nix eval --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath") +[[ $path = $path2 ]] + +# In pure eval mode, fetchGit with a revision should succeed. +[[ $(nix eval --pure-eval --raw "(builtins.readFile (fetchMercurial { url = file://$repo; rev = \"$rev2\"; } + \"/hello\"))") = world ]] + +# Fetch again. This should be cached. +mv $repo ${repo}-tmp +path2=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath") +[[ $path = $path2 ]] + +[[ $(nix eval --raw "(builtins.fetchMercurial file://$repo).branch") = default ]] +[[ $(nix eval "(builtins.fetchMercurial file://$repo).revCount") = 1 ]] +[[ $(nix eval --raw "(builtins.fetchMercurial file://$repo).rev") = $rev2 ]] + +# But with TTL 0, it should fail. +(! nix eval --tarball-ttl 0 "(builtins.fetchMercurial file://$repo)") + +# Fetching with a explicit hash should succeed. +path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath") +[[ $path = $path2 ]] + +path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev1\"; }).outPath") +[[ $(cat $path2/hello) = utrecht ]] + +mv ${repo}-tmp $repo + +# Using a clean working tree should produce the same result. +path2=$(nix eval --raw "(builtins.fetchMercurial $repo).outPath") +[[ $path = $path2 ]] + +# Using an unclean tree should yield the tracked but uncommitted changes. +mkdir $repo/dir1 $repo/dir2 +echo foo > $repo/dir1/foo +echo bar > $repo/bar +echo bar > $repo/dir2/bar +hg add --cwd $repo dir1/foo +hg rm --cwd $repo hello + +path2=$(nix eval --raw "(builtins.fetchMercurial $repo).outPath") +[ ! -e $path2/hello ] +[ ! -e $path2/bar ] +[ ! -e $path2/dir2/bar ] +[ ! -e $path2/.hg ] +[[ $(cat $path2/dir1/foo) = foo ]] + +[[ $(nix eval --raw "(builtins.fetchMercurial $repo).rev") = 0000000000000000000000000000000000000000 ]] + +# ... unless we're using an explicit rev. +path3=$(nix eval --raw "(builtins.fetchMercurial { url = $repo; rev = \"default\"; }).outPath") +[[ $path = $path3 ]] + +# Committing should not affect the store path. +hg commit --cwd $repo -m 'Bla3' + +path4=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial file://$repo).outPath") +[[ $path2 = $path4 ]] diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index b6fa3a27edd..2535651b09a 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -5,10 +5,47 @@ clearStore # Test fetching a flat file. hash=$(nix-hash --flat --type sha256 ./fetchurl.sh) -outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link) +outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link --hashed-mirrors '') cmp $outPath fetchurl.sh +# Now using a base-64 hash. +clearStore + +hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh) + +outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors '') + +cmp $outPath fetchurl.sh + +# Now using an SRI hash. +clearStore + +hash=$(nix hash-file ./fetchurl.sh) + +[[ $hash =~ ^sha256- ]] + +outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link --hashed-mirrors '') + +cmp $outPath fetchurl.sh + +# Test the hashed mirror feature. +clearStore + +hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh) +hash32=$(nix hash-file --type sha512 --base16 ./fetchurl.sh) + +mirror=$TEST_ROOT/hashed-mirror +rm -rf $mirror +mkdir -p $mirror/sha512 +ln -s $(pwd)/fetchurl.sh $mirror/sha512/$hash32 + +outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors "file://$mirror") + +# Test hashed mirrors with an SRI hash. +nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha512 $hash) \ + --argstr name bla --no-out-link --hashed-mirrors "file://$mirror" + # Test unpacking a NAR. rm -rf $TEST_ROOT/archive mkdir -p $TEST_ROOT/archive diff --git a/tests/fixed.sh b/tests/fixed.sh index cac3f0be91b..8f51403a707 100644 --- a/tests/fixed.sh +++ b/tests/fixed.sh @@ -5,15 +5,22 @@ clearStore export IMPURE_VAR1=foo export IMPURE_VAR2=bar +path=$(nix-store -q $(nix-instantiate fixed.nix -A good.0)) + +echo 'testing bad...' +nix-build fixed.nix -A bad --no-out-link && fail "should fail" + +# Building with the bad hash should produce the "good" output path as +# a side-effect. +[[ -e $path ]] +nix path-info --json $path | grep fixed:md5:2qk15sxzzjlnpjk9brn7j8ppcd + echo 'testing good...' nix-build fixed.nix -A good --no-out-link echo 'testing good2...' nix-build fixed.nix -A good2 --no-out-link -echo 'testing bad...' -nix-build fixed.nix -A bad --no-out-link && fail "should fail" - echo 'testing reallyBad...' nix-instantiate fixed.nix -A reallyBad && fail "should fail" diff --git a/tests/function-trace.sh b/tests/function-trace.sh new file mode 100755 index 00000000000..182a4d5c287 --- /dev/null +++ b/tests/function-trace.sh @@ -0,0 +1,85 @@ +source common.sh + +set +x + +expect_trace() { + expr="$1" + expect="$2" + actual=$( + nix-instantiate \ + --trace-function-calls \ + --expr "$expr" 2>&1 \ + | grep "function-trace" \ + | sed -e 's/ [0-9]*$//' + ); + + echo -n "Tracing expression '$expr'" + set +e + msg=$(diff -swB \ + <(echo "$expect") \ + <(echo "$actual") + ); + result=$? + set -e + if [ $result -eq 0 ]; then + echo " ok." + else + echo " failed. difference:" + echo "$msg" + return $result + fi +} + +# failure inside a tryEval +expect_trace 'builtins.tryEval (throw "example")' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace entered (string):1:19 at +function-trace exited (string):1:19 at +function-trace exited (string):1:1 at +" + +# Missing argument to a formal function +expect_trace '({ x }: x) { }' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Too many arguments to a formal function +expect_trace '({ x }: x) { x = "x"; y = "y"; }' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Not enough arguments to a lambda +expect_trace '(x: y: x + y) 1' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Too many arguments to a lambda +expect_trace '(x: x) 1 2' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Not a function +expect_trace '1 2' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +set -e diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh new file mode 100644 index 00000000000..de1e2cfe405 --- /dev/null +++ b/tests/gc-auto.sh @@ -0,0 +1,70 @@ +source common.sh + +clearStore + +garbage1=$(nix add-to-store --name garbage1 ./nar-access.sh) +garbage2=$(nix add-to-store --name garbage2 ./nar-access.sh) +garbage3=$(nix add-to-store --name garbage3 ./nar-access.sh) + +ls -l $garbage3 +POSIXLY_CORRECT=1 du $garbage3 + +fake_free=$TEST_ROOT/fake-free +export _NIX_TEST_FREE_SPACE_FILE=$fake_free +echo 1100 > $fake_free + +expr=$(cat < \$out/bar + echo 1... + sleep 2 + echo 200 > ${fake_free}.tmp1 + mv ${fake_free}.tmp1 $fake_free + echo 2... + sleep 2 + echo 3... + sleep 2 + echo 4... + [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]] + ''; +} +EOF +) + +expr2=$(cat < \$out/bar + echo 1... + sleep 2 + echo 200 > ${fake_free}.tmp2 + mv ${fake_free}.tmp2 $fake_free + echo 2... + sleep 2 + echo 3... + sleep 2 + echo 4... + ''; +} +EOF +) + +nix build -v -o $TEST_ROOT/result-A -L "($expr)" \ + --min-free 1000 --max-free 2000 --min-free-check-interval 1 & +pid=$! + +nix build -v -o $TEST_ROOT/result-B -L "($expr2)" \ + --min-free 1000 --max-free 2000 --min-free-check-interval 1 + +wait "$pid" + +[[ foo = $(cat $TEST_ROOT/result-A/bar) ]] +[[ foo = $(cat $TEST_ROOT/result-B/bar) ]] diff --git a/tests/gc-concurrent.nix b/tests/gc-concurrent.nix index c0595cc471b..21671ea2c13 100644 --- a/tests/gc-concurrent.nix +++ b/tests/gc-concurrent.nix @@ -4,12 +4,12 @@ rec { input1 = mkDerivation { name = "dependencies-input-1"; - builder = ./dependencies.builder1.sh; + buildCommand = "mkdir $out; echo FOO > $out/foo"; }; input2 = mkDerivation { name = "dependencies-input-2"; - builder = ./dependencies.builder2.sh; + buildCommand = "mkdir $out; echo BAR > $out/bar"; }; test1 = mkDerivation { @@ -23,5 +23,5 @@ rec { builder = ./gc-concurrent2.builder.sh; inherit input1 input2; }; - + } diff --git a/tests/gc.sh b/tests/gc.sh index 0adb05bf173..8b4f8d28218 100644 --- a/tests/gc.sh +++ b/tests/gc.sh @@ -7,7 +7,7 @@ outPath=$(nix-store -rvv "$drvPath") rm -f "$NIX_STATE_DIR"/gcroots/foo ln -sf $outPath "$NIX_STATE_DIR"/gcroots/foo -[ "$(nix-store -q --roots $outPath)" = "$NIX_STATE_DIR"/gcroots/foo ] +[ "$(nix-store -q --roots $outPath)" = "$NIX_STATE_DIR/gcroots/foo -> $outPath" ] nix-store --gc --print-roots | grep $outPath nix-store --gc --print-live | grep $outPath diff --git a/tests/hash.sh b/tests/hash.sh index a95c68683f8..4cfc9790101 100644 --- a/tests/hash.sh +++ b/tests/hash.sh @@ -2,7 +2,7 @@ source common.sh try () { printf "%s" "$2" > $TEST_ROOT/vector - hash=$(nix-hash $EXTRA --flat --type "$1" $TEST_ROOT/vector) + hash=$(nix hash-file --base16 $EXTRA --type "$1" $TEST_ROOT/vector) if test "$hash" != "$3"; then echo "hash $1, expected $3, got $hash" exit 1 @@ -33,6 +33,12 @@ EXTRA=--base32 try sha256 "abc" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" EXTRA= +EXTRA=--sri +try sha512 "" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" +try sha512 "abc" "sha512-3a81oZNherrMQXNJriBBMRLm+k6JqX6iCp7u5ktV05ohkpkqJ0/BqDa6PCOj/uu9RU1EI2Q86A4qmslPpUyknw==" +try sha512 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "sha512-IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" +try sha256 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "sha256-JI1qYdIGOLjlwCaTDD5gOaM85Flk/yFn9uzt1BnbBsE=" + try2 () { hash=$(nix-hash --type "$1" $TEST_ROOT/hash-path) if test "$hash" != "$2"; then @@ -63,11 +69,19 @@ try2 md5 "f78b733a68f5edbdf9413899339eaa4a" # Conversion. try3() { + h64=$(nix to-base64 --type "$1" "$2") + [ "$h64" = "$4" ] + sri=$(nix to-sri --type "$1" "$2") + [ "$sri" = "$1-$4" ] h32=$(nix-hash --type "$1" --to-base32 "$2") [ "$h32" = "$3" ] h16=$(nix-hash --type "$1" --to-base16 "$h32") [ "$h16" = "$2" ] + h16=$(nix to-base16 --type "$1" "$h64") + [ "$h16" = "$2" ] + h16=$(nix to-base16 "$sri") + [ "$h16" = "$2" ] } -try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" -try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" -try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" +try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" +try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" +try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" "IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" diff --git a/tests/import-derivation.nix b/tests/import-derivation.nix index 91adcd288f6..44fa9a45d7e 100644 --- a/tests/import-derivation.nix +++ b/tests/import-derivation.nix @@ -10,7 +10,10 @@ let ''; }; - value = import bar; + value = + # Test that pathExists can check the existence of /nix/store paths + assert builtins.pathExists bar; + import bar; in diff --git a/tests/init.sh b/tests/init.sh index 4571b75b859..c62c4856a93 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -15,10 +15,15 @@ mkdir "$NIX_CONF_DIR" cat > "$NIX_CONF_DIR"/nix.conf < "$NIX_CONF_DIR"/nix.conf.extra < "$file.next" + mv "$file.next" "$file" + fi + done + + for i in $(seq 1 $(sysctl -n hw.ncpu)); do + sudo /usr/bin/dscl . -delete "/Users/nixbld$i" || true + done + sudo /usr/bin/dscl . -delete "/Groups/nixbld" || true + + sudo rm -rf /etc/nix \ + /nix \ + /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels \ + "$HOME/.nix-profile" "$HOME/.nix-defexpr" "$HOME/.nix-channels" +} + +verify() { + set +e + output=$(echo "nix-shell -p bash --run 'echo toow | rev'" | bash -l) + set -e + + test "$output" = "woot" +} + +scratch=$(mktemp -d -t tmp.XXXXXXXXXX) +function finish { + rm -rf "$scratch" +} +trap finish EXIT + +# First setup Nix +cleanup +curl -o install https://nixos.org/nix/install +yes | bash ./install +verify + + +( + set +e + ( + echo "cd $(pwd)" + echo nix-build ./release.nix -A binaryTarball.x86_64-darwin + ) | bash -l + set -e + cp ./result/nix-*.tar.bz2 $scratch/nix.tar.bz2 +) + +( + cd $scratch + tar -xf ./nix.tar.bz2 + + cd nix-* + + set -eux + + cleanup + + yes | ./install + verify + cleanup + + echo -n "" | ./install + verify + cleanup + + sudo mkdir -p /nix/store + sudo touch /nix/store/.silly-hint + echo -n "" | ALLOW_PREEXISTING_INSTALLATION=true ./install + verify + test -e /nix/store/.silly-hint + + cleanup +) diff --git a/tests/lang/binary-data b/tests/lang/binary-data new file mode 100644 index 00000000000..06d74050200 Binary files /dev/null and b/tests/lang/binary-data differ diff --git a/tests/lang/data b/tests/lang/data new file mode 100644 index 00000000000..257cc5642cb --- /dev/null +++ b/tests/lang/data @@ -0,0 +1 @@ +foo diff --git a/tests/lang/eval-fail-hashfile-missing.nix b/tests/lang/eval-fail-hashfile-missing.nix new file mode 100644 index 00000000000..ce098b82380 --- /dev/null +++ b/tests/lang/eval-fail-hashfile-missing.nix @@ -0,0 +1,5 @@ +let + paths = [ ./this-file-is-definitely-not-there-7392097 "/and/neither/is/this/37293620" ]; +in + toString (builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"])) + diff --git a/tests/lang/eval-fail-path-slash.nix b/tests/lang/eval-fail-path-slash.nix index 530105b3210..8c2e104c788 100644 --- a/tests/lang/eval-fail-path-slash.nix +++ b/tests/lang/eval-fail-path-slash.nix @@ -2,5 +2,5 @@ # This restriction could be lifted sometime, # for example if we make '/' a path concatenation operator. # See https://github.com/NixOS/nix/issues/1138 -# and http://lists.science.uu.nl/pipermail/nix-dev/2016-June/020829.html +# and https://nixos.org/nix-dev/2016-June/020829.html /nix/store/ diff --git a/tests/lang/eval-okay-arithmetic.exp b/tests/lang/eval-okay-arithmetic.exp index b195055b7a0..5c54d10b7b4 100644 --- a/tests/lang/eval-okay-arithmetic.exp +++ b/tests/lang/eval-okay-arithmetic.exp @@ -1 +1 @@ -2188 +2216 diff --git a/tests/lang/eval-okay-arithmetic.nix b/tests/lang/eval-okay-arithmetic.nix index bbbbc4691d7..7e9e6a0b666 100644 --- a/tests/lang/eval-okay-arithmetic.nix +++ b/tests/lang/eval-okay-arithmetic.nix @@ -26,6 +26,10 @@ let { (56088 / 123 / 2) (3 + 4 * const 5 0 - 6 / id 2) + (builtins.bitAnd 12 10) # 0b1100 & 0b1010 = 8 + (builtins.bitOr 12 10) # 0b1100 | 0b1010 = 14 + (builtins.bitXor 12 10) # 0b1100 ^ 0b1010 = 6 + (if 3 < 7 then 1 else err) (if 7 < 3 then err else 1) (if 3 < 3 then err else 1) diff --git a/tests/lang/eval-okay-attrs6.exp b/tests/lang/eval-okay-attrs6.exp new file mode 100644 index 00000000000..b46938032e7 --- /dev/null +++ b/tests/lang/eval-okay-attrs6.exp @@ -0,0 +1 @@ +{ __overrides = { bar = "qux"; }; bar = "qux"; foo = "bar"; } diff --git a/tests/lang/eval-okay-attrs6.nix b/tests/lang/eval-okay-attrs6.nix new file mode 100644 index 00000000000..2e5c85483be --- /dev/null +++ b/tests/lang/eval-okay-attrs6.nix @@ -0,0 +1,4 @@ +rec { + "${"foo"}" = "bar"; + __overrides = { bar = "qux"; }; +} diff --git a/tests/lang/eval-okay-backslash-newline-1.exp b/tests/lang/eval-okay-backslash-newline-1.exp new file mode 100644 index 00000000000..3e754364cc9 --- /dev/null +++ b/tests/lang/eval-okay-backslash-newline-1.exp @@ -0,0 +1 @@ +"a\nb" diff --git a/tests/lang/eval-okay-backslash-newline-1.nix b/tests/lang/eval-okay-backslash-newline-1.nix new file mode 100644 index 00000000000..7fef3dddd4d --- /dev/null +++ b/tests/lang/eval-okay-backslash-newline-1.nix @@ -0,0 +1,2 @@ +"a\ +b" diff --git a/tests/lang/eval-okay-backslash-newline-2.exp b/tests/lang/eval-okay-backslash-newline-2.exp new file mode 100644 index 00000000000..3e754364cc9 --- /dev/null +++ b/tests/lang/eval-okay-backslash-newline-2.exp @@ -0,0 +1 @@ +"a\nb" diff --git a/tests/lang/eval-okay-backslash-newline-2.nix b/tests/lang/eval-okay-backslash-newline-2.nix new file mode 100644 index 00000000000..35ddf495c63 --- /dev/null +++ b/tests/lang/eval-okay-backslash-newline-2.nix @@ -0,0 +1,2 @@ +''a''\ +b'' diff --git a/tests/lang/eval-okay-builtins-add.exp b/tests/lang/eval-okay-builtins-add.exp new file mode 100644 index 00000000000..0350b518a7e --- /dev/null +++ b/tests/lang/eval-okay-builtins-add.exp @@ -0,0 +1 @@ +[ 5 4 "int" "tt" "float" 4 ] diff --git a/tests/lang/eval-okay-builtins-add.nix b/tests/lang/eval-okay-builtins-add.nix new file mode 100644 index 00000000000..c841816222a --- /dev/null +++ b/tests/lang/eval-okay-builtins-add.nix @@ -0,0 +1,8 @@ +[ +(builtins.add 2 3) +(builtins.add 2 2) +(builtins.typeOf (builtins.add 2 2)) +("t" + "t") +(builtins.typeOf (builtins.add 2.0 2)) +(builtins.add 2.0 2) +] diff --git a/tests/lang/eval-okay-concatmap.exp b/tests/lang/eval-okay-concatmap.exp new file mode 100644 index 00000000000..3b8be7739de --- /dev/null +++ b/tests/lang/eval-okay-concatmap.exp @@ -0,0 +1 @@ +[ [ 1 3 5 7 9 ] [ "a" "z" "b" "z" ] ] diff --git a/tests/lang/eval-okay-concatmap.nix b/tests/lang/eval-okay-concatmap.nix new file mode 100644 index 00000000000..97da5d37a41 --- /dev/null +++ b/tests/lang/eval-okay-concatmap.nix @@ -0,0 +1,5 @@ +with import ./lib.nix; + +[ (builtins.concatMap (x: if x / 2 * 2 == x then [] else [ x ]) (range 0 10)) + (builtins.concatMap (x: [x] ++ ["z"]) ["a" "b"]) +] diff --git a/tests/lang/eval-okay-context-introspection.exp b/tests/lang/eval-okay-context-introspection.exp new file mode 100644 index 00000000000..27ba77ddaf6 --- /dev/null +++ b/tests/lang/eval-okay-context-introspection.exp @@ -0,0 +1 @@ +true diff --git a/tests/lang/eval-okay-context-introspection.nix b/tests/lang/eval-okay-context-introspection.nix new file mode 100644 index 00000000000..43178bd2eef --- /dev/null +++ b/tests/lang/eval-okay-context-introspection.nix @@ -0,0 +1,24 @@ +let + drv = derivation { + name = "fail"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + + path = "${./eval-okay-context-introspection.nix}"; + + desired-context = { + "${builtins.unsafeDiscardStringContext path}" = { + path = true; + }; + "${builtins.unsafeDiscardStringContext drv.drvPath}" = { + outputs = [ "foo" "out" ]; + allOutputs = true; + }; + }; + + legit-context = builtins.getContext "${path}${drv.outPath}${drv.foo.outPath}${drv.drvPath}"; + + constructed-context = builtins.getContext (builtins.appendContext "" desired-context); +in legit-context == constructed-context diff --git a/tests/lang/eval-okay-float.exp b/tests/lang/eval-okay-float.exp new file mode 100644 index 00000000000..3c50a8adce8 --- /dev/null +++ b/tests/lang/eval-okay-float.exp @@ -0,0 +1 @@ +[ 3.4 3.5 2.5 1.5 ] diff --git a/tests/lang/eval-okay-float.nix b/tests/lang/eval-okay-float.nix new file mode 100644 index 00000000000..b2702c7b166 --- /dev/null +++ b/tests/lang/eval-okay-float.nix @@ -0,0 +1,6 @@ +[ + (1.1 + 2.3) + (builtins.add (0.5 + 0.5) (2.0 + 0.5)) + ((0.5 + 0.5) * (2.0 + 0.5)) + ((1.5 + 1.5) / (0.5 * 4.0)) +] diff --git a/tests/lang/eval-okay-foldlStrict.exp b/tests/lang/eval-okay-foldlStrict.exp new file mode 100644 index 00000000000..837e12b406f --- /dev/null +++ b/tests/lang/eval-okay-foldlStrict.exp @@ -0,0 +1 @@ +500500 diff --git a/tests/lang/eval-okay-foldlStrict.nix b/tests/lang/eval-okay-foldlStrict.nix new file mode 100644 index 00000000000..3b87188d243 --- /dev/null +++ b/tests/lang/eval-okay-foldlStrict.nix @@ -0,0 +1,3 @@ +with import ./lib.nix; + +builtins.foldl' (x: y: x + y) 0 (range 1 1000) diff --git a/tests/lang/eval-okay-fromTOML.exp b/tests/lang/eval-okay-fromTOML.exp new file mode 100644 index 00000000000..d0dd3af2c81 --- /dev/null +++ b/tests/lang/eval-okay-fromTOML.exp @@ -0,0 +1 @@ +[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; fruit = [ { name = "apple"; physical = { color = "red"; shape = "round"; }; variety = [ { name = "red delicious"; } { name = "granny smith"; } ]; } { name = "banana"; variety = [ { name = "plantain"; } ]; } ]; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; oct1 = 342391; oct2 = 493; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } { a = [ [ { b = true; } ] ]; c = [ [ { d = true; } ] ]; e = [ [ 123 ] ]; } ] diff --git a/tests/lang/eval-okay-fromTOML.nix b/tests/lang/eval-okay-fromTOML.nix new file mode 100644 index 00000000000..96393268994 --- /dev/null +++ b/tests/lang/eval-okay-fromTOML.nix @@ -0,0 +1,208 @@ +[ + + (builtins.fromTOML '' + # This is a TOML document. + + title = "TOML Example" + + [owner] + name = "Tom Preston-Werner" + #dob = 1979-05-27T07:32:00-08:00 # First class dates + + [database] + server = "192.168.1.1" + ports = [ 8001, 8001, 8002 ] + connection_max = 5000 + enabled = true + + [servers] + + # Indentation (tabs and/or spaces) is allowed but not required + [servers.alpha] + ip = "10.0.0.1" + dc = "eqdc10" + + [servers.beta] + ip = "10.0.0.2" + dc = "eqdc10" + + [clients] + data = [ ["gamma", "delta"], [1, 2] ] + + # Line breaks are OK when inside arrays + hosts = [ + "alpha", + "omega" + ] + '') + + (builtins.fromTOML '' + key = "value" + bare_key = "value" + bare-key = "value" + 1234 = "value" + + "127.0.0.1" = "value" + "character encoding" = "value" + "ʎǝʞ" = "value" + 'key2' = "value" + 'quoted "value"' = "value" + + name = "Orange" + + physical.color = "orange" + physical.shape = "round" + site."google.com" = true + + # This is legal according to the spec, but cpptoml doesn't handle it. + #a.b.c = 1 + #a.d = 2 + + str = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF." + + int1 = +99 + int2 = 42 + int3 = 0 + int4 = -17 + int5 = 1_000 + int6 = 5_349_221 + int7 = 1_2_3_4_5 + + hex1 = 0xDEADBEEF + hex2 = 0xdeadbeef + hex3 = 0xdead_beef + + oct1 = 0o01234567 + oct2 = 0o755 + + bin1 = 0b11010110 + + flt1 = +1.0 + flt2 = 3.1415 + flt3 = -0.01 + flt4 = 5e+22 + flt5 = 1e6 + flt6 = -2E-2 + flt7 = 6.626e-34 + flt8 = 9_224_617.445_991_228_313 + + bool1 = true + bool2 = false + + # FIXME: not supported because Nix doesn't have a date/time type. + #odt1 = 1979-05-27T07:32:00Z + #odt2 = 1979-05-27T00:32:00-07:00 + #odt3 = 1979-05-27T00:32:00.999999-07:00 + #odt4 = 1979-05-27 07:32:00Z + #ldt1 = 1979-05-27T07:32:00 + #ldt2 = 1979-05-27T00:32:00.999999 + #ld1 = 1979-05-27 + #lt1 = 07:32:00 + #lt2 = 00:32:00.999999 + + arr1 = [ 1, 2, 3 ] + arr2 = [ "red", "yellow", "green" ] + arr3 = [ [ 1, 2 ], [3, 4, 5] ] + arr4 = [ "all", 'strings', """are the same""", ''''type''''] + arr5 = [ [ 1, 2 ], ["a", "b", "c"] ] + + arr7 = [ + 1, 2, 3 + ] + + arr8 = [ + 1, + 2, # this is ok + ] + + [table-1] + key1 = "some string" + key2 = 123 + + + [table-2] + key1 = "another string" + key2 = 456 + + [dog."tater.man"] + type.name = "pug" + + [a.b.c] + [ d.e.f ] + [ g . h . i ] + [ j . "ʞ" . 'l' ] + [x.y.z.w] + + name = { first = "Tom", last = "Preston-Werner" } + point = { x = 1, y = 2 } + animal = { type.name = "pug" } + + [[products]] + name = "Hammer" + sku = 738594937 + + [[products]] + + [[products]] + name = "Nail" + sku = 284758393 + color = "gray" + + [[fruit]] + name = "apple" + + [fruit.physical] + color = "red" + shape = "round" + + [[fruit.variety]] + name = "red delicious" + + [[fruit.variety]] + name = "granny smith" + + [[fruit]] + name = "banana" + + [[fruit.variety]] + name = "plantain" + '') + + (builtins.fromTOML '' + [[package]] + name = "aho-corasick" + version = "0.6.4" + source = "registry+https://github.com/rust-lang/crates.io-index" + dependencies = [ + "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + ] + + [[package]] + name = "ansi_term" + version = "0.9.0" + source = "registry+https://github.com/rust-lang/crates.io-index" + + [[package]] + name = "atty" + version = "0.2.10" + source = "registry+https://github.com/rust-lang/crates.io-index" + dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + ] + + [metadata] + "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4" + "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" + "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6" + "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" + '') + + (builtins.fromTOML '' + a = [[{ b = true }]] + c = [ [ { d = true } ] ] + e = [[123]] + '') + +] diff --git a/tests/lang/eval-okay-fromjson-escapes.exp b/tests/lang/eval-okay-fromjson-escapes.exp new file mode 100644 index 00000000000..add5505a828 --- /dev/null +++ b/tests/lang/eval-okay-fromjson-escapes.exp @@ -0,0 +1 @@ +"quote \" reverse solidus \\ solidus / backspace  formfeed newline \n carriage return \r horizontal tab \t 1 char unicode encoded backspace  1 char unicode encoded e with accent é 2 char unicode encoded s with caron š 3 char unicode encoded rightwards arrow →" diff --git a/tests/lang/eval-okay-fromjson-escapes.nix b/tests/lang/eval-okay-fromjson-escapes.nix new file mode 100644 index 00000000000..f0071350773 --- /dev/null +++ b/tests/lang/eval-okay-fromjson-escapes.nix @@ -0,0 +1,3 @@ +# This string contains all supported escapes in a JSON string, per json.org +# \b and \f are not supported by Nix +builtins.fromJSON ''"quote \" reverse solidus \\ solidus \/ backspace \b formfeed \f newline \n carriage return \r horizontal tab \t 1 char unicode encoded backspace \u0008 1 char unicode encoded e with accent \u00e9 2 char unicode encoded s with caron \u0161 3 char unicode encoded rightwards arrow \u2192"'' diff --git a/tests/lang/eval-okay-getattrpos-functionargs.exp b/tests/lang/eval-okay-getattrpos-functionargs.exp new file mode 100644 index 00000000000..7f9ac40e81b --- /dev/null +++ b/tests/lang/eval-okay-getattrpos-functionargs.exp @@ -0,0 +1 @@ +{ column = 11; file = "eval-okay-getattrpos-functionargs.nix"; line = 2; } diff --git a/tests/lang/eval-okay-getattrpos-functionargs.nix b/tests/lang/eval-okay-getattrpos-functionargs.nix new file mode 100644 index 00000000000..11d6bb0e3ac --- /dev/null +++ b/tests/lang/eval-okay-getattrpos-functionargs.nix @@ -0,0 +1,4 @@ +let + fun = { foo }: {}; + pos = builtins.unsafeGetAttrPos "foo" (builtins.functionArgs fun); +in { inherit (pos) column line; file = baseNameOf pos.file; } diff --git a/tests/lang/eval-okay-getattrpos-undefined.exp b/tests/lang/eval-okay-getattrpos-undefined.exp new file mode 100644 index 00000000000..19765bd501b --- /dev/null +++ b/tests/lang/eval-okay-getattrpos-undefined.exp @@ -0,0 +1 @@ +null diff --git a/tests/lang/eval-okay-getattrpos-undefined.nix b/tests/lang/eval-okay-getattrpos-undefined.nix new file mode 100644 index 00000000000..14dd38f7734 --- /dev/null +++ b/tests/lang/eval-okay-getattrpos-undefined.nix @@ -0,0 +1 @@ +builtins.unsafeGetAttrPos "abort" builtins diff --git a/tests/lang/eval-okay-hash.exp b/tests/lang/eval-okay-hash.exp index d720a082ddb..e69de29bb2d 100644 --- a/tests/lang/eval-okay-hash.exp +++ b/tests/lang/eval-okay-hash.exp @@ -1 +0,0 @@ -[ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ] diff --git a/tests/lang/eval-okay-hashfile.exp b/tests/lang/eval-okay-hashfile.exp new file mode 100644 index 00000000000..ff1e8293ef2 --- /dev/null +++ b/tests/lang/eval-okay-hashfile.exp @@ -0,0 +1 @@ +[ "d3b07384d113edec49eaa6238ad5ff00" "0f343b0931126a20f133d67c2b018a3b" "f1d2d2f924e986ac86fdf7b36c94bcdf32beec15" "60cacbf3d72e1e7834203da608037b1bf83b40e8" "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c" "5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef" "0cf9180a764aba863a67b6d72f0918bc131c6772642cb2dce5a34f0a702f9470ddc2bf125c12198b1995c233c34b4afd346c54a2334c350a948a51b6e8b4e6b6" "8efb4f73c5655351c444eb109230c556d39e2c7624e9c11abc9e3fb4b9b9254218cc5085b454a9698d085cfa92198491f07a723be4574adc70617b73eb0b6461" ] diff --git a/tests/lang/eval-okay-hashfile.nix b/tests/lang/eval-okay-hashfile.nix new file mode 100644 index 00000000000..aff5a185681 --- /dev/null +++ b/tests/lang/eval-okay-hashfile.nix @@ -0,0 +1,4 @@ +let + paths = [ ./data ./binary-data ]; +in + builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"]) diff --git a/tests/lang/eval-okay-hashstring.exp b/tests/lang/eval-okay-hashstring.exp new file mode 100644 index 00000000000..d720a082ddb --- /dev/null +++ b/tests/lang/eval-okay-hashstring.exp @@ -0,0 +1 @@ +[ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ] diff --git a/tests/lang/eval-okay-hash.nix b/tests/lang/eval-okay-hashstring.nix similarity index 100% rename from tests/lang/eval-okay-hash.nix rename to tests/lang/eval-okay-hashstring.nix diff --git a/tests/lang/eval-okay-ind-string.exp b/tests/lang/eval-okay-ind-string.exp index 886219dcf65..9cf4bd2ee78 100644 --- a/tests/lang/eval-okay-ind-string.exp +++ b/tests/lang/eval-okay-ind-string.exp @@ -1 +1 @@ -"This is an indented multi-line string\nliteral. An amount of whitespace at\nthe start of each line matching the minimum\nindentation of all lines in the string\nliteral together will be removed. Thus,\nin this case four spaces will be\nstripped from each line, even though\n THIS LINE is indented six spaces.\n\nAlso, empty lines don't count in the\ndetermination of the indentation level (the\nprevious empty line has indentation 0, but\nit doesn't matter).\nIf the string starts with whitespace\n followed by a newline, it's stripped, but\n that's not the case here. Two spaces are\n stripped because of the \" \" at the start. \nThis line is indented\na bit further.\nAnti-quotations, like so, are\nalso allowed.\n The \\ is not special here.\n' can be followed by any character except another ', e.g. 'x'.\nLikewise for $, e.g. $$ or $varName.\nBut ' followed by ' is special, as is $ followed by {.\nIf you want them, use anti-quotations: '', ${.\n Tabs are not interpreted as whitespace (since we can't guess\n what tab settings are intended), so don't use them.\n\tThis line starts with a space and a tab, so only one\n space will be stripped from each line.\nAlso note that if the last line (just before the closing ' ')\nconsists only of whitespace, it's ignored. But here there is\nsome non-whitespace stuff, so the line isn't removed. \nThis shows a hacky way to preserve an empty line after the start.\nBut there's no reason to do so: you could just repeat the empty\nline.\n Similarly you can force an indentation level,\n in this case to 2 spaces. This works because the anti-quote\n is significant (not whitespace).\nstart on network-interfaces\n\nstart script\n\n rm -f /var/run/opengl-driver\n ln -sf 123 /var/run/opengl-driver\n\n rm -f /var/log/slim.log\n \nend script\n\nenv SLIM_CFGFILE=abc\nenv SLIM_THEMESDIR=def\nenv FONTCONFIG_FILE=/etc/fonts/fonts.conf \t\t\t\t# !!! cleanup\nenv XKB_BINDIR=foo/bin \t\t\t\t# Needed for the Xkb extension.\nenv LD_LIBRARY_PATH=libX11/lib:libXext/lib:/usr/lib/ # related to xorg-sys-opengl - needed to load libglx for (AI)GLX support (for compiz)\n\nenv XORG_DRI_DRIVER_PATH=nvidiaDrivers/X11R6/lib/modules/drivers/ \n\nexec slim/bin/slim\nEscaping of ' followed by ': ''\nEscaping of $ followed by {: ${\nAnd finally to interpret \\n etc. as in a string: \n, \r, \t.\nfoo\n'bla'\nbar\n" +"This is an indented multi-line string\nliteral. An amount of whitespace at\nthe start of each line matching the minimum\nindentation of all lines in the string\nliteral together will be removed. Thus,\nin this case four spaces will be\nstripped from each line, even though\n THIS LINE is indented six spaces.\n\nAlso, empty lines don't count in the\ndetermination of the indentation level (the\nprevious empty line has indentation 0, but\nit doesn't matter).\nIf the string starts with whitespace\n followed by a newline, it's stripped, but\n that's not the case here. Two spaces are\n stripped because of the \" \" at the start. \nThis line is indented\na bit further.\nAnti-quotations, like so, are\nalso allowed.\n The \\ is not special here.\n' can be followed by any character except another ', e.g. 'x'.\nLikewise for $, e.g. $$ or $varName.\nBut ' followed by ' is special, as is $ followed by {.\nIf you want them, use anti-quotations: '', ${.\n Tabs are not interpreted as whitespace (since we can't guess\n what tab settings are intended), so don't use them.\n\tThis line starts with a space and a tab, so only one\n space will be stripped from each line.\nAlso note that if the last line (just before the closing ' ')\nconsists only of whitespace, it's ignored. But here there is\nsome non-whitespace stuff, so the line isn't removed. \nThis shows a hacky way to preserve an empty line after the start.\nBut there's no reason to do so: you could just repeat the empty\nline.\n Similarly you can force an indentation level,\n in this case to 2 spaces. This works because the anti-quote\n is significant (not whitespace).\nstart on network-interfaces\n\nstart script\n\n rm -f /var/run/opengl-driver\n ln -sf 123 /var/run/opengl-driver\n\n rm -f /var/log/slim.log\n \nend script\n\nenv SLIM_CFGFILE=abc\nenv SLIM_THEMESDIR=def\nenv FONTCONFIG_FILE=/etc/fonts/fonts.conf \t\t\t\t# !!! cleanup\nenv XKB_BINDIR=foo/bin \t\t\t\t# Needed for the Xkb extension.\nenv LD_LIBRARY_PATH=libX11/lib:libXext/lib:/usr/lib/ # related to xorg-sys-opengl - needed to load libglx for (AI)GLX support (for compiz)\n\nenv XORG_DRI_DRIVER_PATH=nvidiaDrivers/X11R6/lib/modules/drivers/ \n\nexec slim/bin/slim\nEscaping of ' followed by ': ''\nEscaping of $ followed by {: ${\nAnd finally to interpret \\n etc. as in a string: \n, \r, \t.\nfoo\n'bla'\nbar\ncut -d $'\\t' -f 1\nending dollar $$\n" diff --git a/tests/lang/eval-okay-ind-string.nix b/tests/lang/eval-okay-ind-string.nix index 1556aae9f54..1669dc0648e 100644 --- a/tests/lang/eval-okay-ind-string.nix +++ b/tests/lang/eval-okay-ind-string.nix @@ -117,4 +117,12 @@ let bar ''; -in s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10 + s11 + s12 + s13 + s14 + s15 + # Regression test: accept $'. + s16 = '' + cut -d $'\t' -f 1 + ''; + + # Accept dollars at end of strings + s17 = ''ending dollar $'' + ''$'' + "\n"; + +in s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10 + s11 + s12 + s13 + s14 + s15 + s16 + s17 diff --git a/tests/lang/eval-okay-mapattrs.exp b/tests/lang/eval-okay-mapattrs.exp new file mode 100644 index 00000000000..3f113f17bab --- /dev/null +++ b/tests/lang/eval-okay-mapattrs.exp @@ -0,0 +1 @@ +{ x = "x-foo"; y = "y-bar"; } diff --git a/tests/lang/eval-okay-mapattrs.nix b/tests/lang/eval-okay-mapattrs.nix new file mode 100644 index 00000000000..f075b6275e5 --- /dev/null +++ b/tests/lang/eval-okay-mapattrs.nix @@ -0,0 +1,3 @@ +with import ./lib.nix; + +builtins.mapAttrs (name: value: name + "-" + value) { x = "foo"; y = "bar"; } diff --git a/tests/lang/eval-okay-nested-with.exp b/tests/lang/eval-okay-nested-with.exp new file mode 100644 index 00000000000..0cfbf08886f --- /dev/null +++ b/tests/lang/eval-okay-nested-with.exp @@ -0,0 +1 @@ +2 diff --git a/tests/lang/eval-okay-nested-with.nix b/tests/lang/eval-okay-nested-with.nix new file mode 100644 index 00000000000..ba9d79aa79b --- /dev/null +++ b/tests/lang/eval-okay-nested-with.nix @@ -0,0 +1,3 @@ +with { x = 1; }; +with { x = 2; }; +x diff --git a/tests/lang/eval-okay-path.nix b/tests/lang/eval-okay-path.nix new file mode 100644 index 00000000000..e67168cf3ed --- /dev/null +++ b/tests/lang/eval-okay-path.nix @@ -0,0 +1,7 @@ +builtins.path + { path = ./.; + filter = path: _: baseNameOf path == "data"; + recursive = true; + sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw"; + name = "output"; + } diff --git a/tests/lang/eval-okay-regex-match.nix b/tests/lang/eval-okay-regex-match.nix index ae6501532d1..273e2590713 100644 --- a/tests/lang/eval-okay-regex-match.nix +++ b/tests/lang/eval-okay-regex-match.nix @@ -17,8 +17,11 @@ assert matches "fo+" "foo"; assert matches "fo{1,2}" "foo"; assert !matches "fo{1,2}" "fooo"; assert !matches "fo*" "foobar"; +assert matches "[[:space:]]+([^[:space:]]+)[[:space:]]+" " foo "; +assert !matches "[[:space:]]+([[:upper:]]+)[[:space:]]+" " foo "; assert match "(.*)\\.nix" "foobar.nix" == [ "foobar" ]; +assert match "[[:space:]]+([[:upper:]]+)[[:space:]]+" " FOO " == [ "FOO" ]; assert splitFN "/path/to/foobar.nix" == [ "/path/to/" "/path/to" "foobar" "nix" ]; assert splitFN "foobar.cc" == [ null null "foobar" "cc" ]; diff --git a/tests/lang/eval-okay-regex-split.exp b/tests/lang/eval-okay-regex-split.exp new file mode 100644 index 00000000000..27ba77ddaf6 --- /dev/null +++ b/tests/lang/eval-okay-regex-split.exp @@ -0,0 +1 @@ +true diff --git a/tests/lang/eval-okay-regex-split.nix b/tests/lang/eval-okay-regex-split.nix new file mode 100644 index 00000000000..0073e057787 --- /dev/null +++ b/tests/lang/eval-okay-regex-split.nix @@ -0,0 +1,48 @@ +with builtins; + +# Non capturing regex returns empty lists +assert split "foobar" "foobar" == ["" [] ""]; +assert split "fo*" "f" == ["" [] ""]; +assert split "fo+" "f" == ["f"]; +assert split "fo*" "fo" == ["" [] ""]; +assert split "fo*" "foo" == ["" [] ""]; +assert split "fo+" "foo" == ["" [] ""]; +assert split "fo{1,2}" "foo" == ["" [] ""]; +assert split "fo{1,2}" "fooo" == ["" [] "o"]; +assert split "fo*" "foobar" == ["" [] "bar"]; + +# Capturing regex returns a list of sub-matches +assert split "(fo*)" "f" == ["" ["f"] ""]; +assert split "(fo+)" "f" == ["f"]; +assert split "(fo*)" "fo" == ["" ["fo"] ""]; +assert split "(f)(o*)" "f" == ["" ["f" ""] ""]; +assert split "(f)(o*)" "foo" == ["" ["f" "oo"] ""]; +assert split "(fo+)" "foo" == ["" ["foo"] ""]; +assert split "(fo{1,2})" "foo" == ["" ["foo"] ""]; +assert split "(fo{1,2})" "fooo" == ["" ["foo"] "o"]; +assert split "(fo*)" "foobar" == ["" ["foo"] "bar"]; + +# Matches are greedy. +assert split "(o+)" "oooofoooo" == ["" ["oooo"] "f" ["oooo"] ""]; + +# Matches multiple times. +assert split "(b)" "foobarbaz" == ["foo" ["b"] "ar" ["b"] "az"]; + +# Split large strings containing newlines. null are inserted when a +# pattern within the current did not match anything. +assert split "[[:space:]]+|([',.!?])" '' + Nix Rocks! + That's why I use it. +'' == [ + "Nix" [ null ] "Rocks" ["!"] "" [ null ] + "That" ["'"] "s" [ null ] "why" [ null ] "I" [ null ] "use" [ null ] "it" ["."] "" [ null ] + "" +]; + +# Documentation examples +assert split "(a)b" "abc" == [ "" [ "a" ] "c" ]; +assert split "([ac])" "abc" == [ "" [ "a" ] "b" [ "c" ] "" ]; +assert split "(a)|(c)" "abc" == [ "" [ "a" null ] "b" [ null "c" ] "" ]; +assert split "([[:upper:]]+)" " FOO " == [ " " [ "FOO" ] " " ]; + +true diff --git a/tests/lang/eval-okay-replacestrings.exp b/tests/lang/eval-okay-replacestrings.exp index a2add1b7b14..72e8274d8c5 100644 --- a/tests/lang/eval-okay-replacestrings.exp +++ b/tests/lang/eval-okay-replacestrings.exp @@ -1 +1 @@ -[ "faabar" "fbar" "fubar" "faboor" "fubar" ] +[ "faabar" "fbar" "fubar" "faboor" "fubar" "XaXbXcX" "X" "a_b" ] diff --git a/tests/lang/eval-okay-replacestrings.nix b/tests/lang/eval-okay-replacestrings.nix index 6284a0e660a..bd8031fc004 100644 --- a/tests/lang/eval-okay-replacestrings.nix +++ b/tests/lang/eval-okay-replacestrings.nix @@ -5,4 +5,7 @@ with builtins; (replaceStrings ["oo"] ["u"] "foobar") (replaceStrings ["oo" "a"] ["a" "oo"] "foobar") (replaceStrings ["oo" "oo"] ["u" "i"] "foobar") + (replaceStrings [""] ["X"] "abc") + (replaceStrings [""] ["X"] "") + (replaceStrings ["-"] ["_"] "a-b") ] diff --git a/tests/lang/eval-okay-search-path.nix b/tests/lang/eval-okay-search-path.nix index cca41f821f8..c5a123d042a 100644 --- a/tests/lang/eval-okay-search-path.nix +++ b/tests/lang/eval-okay-search-path.nix @@ -1,7 +1,7 @@ with import ./lib.nix; with builtins; -assert pathExists ; +assert pathExists ; assert length __nixPath == 6; assert length (filter (x: x.prefix == "nix") __nixPath) == 1; diff --git a/tests/lang/eval-okay-splitversion.exp b/tests/lang/eval-okay-splitversion.exp new file mode 100644 index 00000000000..153ceb8186a --- /dev/null +++ b/tests/lang/eval-okay-splitversion.exp @@ -0,0 +1 @@ +[ "1" "2" "3" ] diff --git a/tests/lang/eval-okay-splitversion.nix b/tests/lang/eval-okay-splitversion.nix new file mode 100644 index 00000000000..9e5c99d2e7f --- /dev/null +++ b/tests/lang/eval-okay-splitversion.nix @@ -0,0 +1 @@ +builtins.splitVersion "1.2.3" diff --git a/tests/lang/eval-okay-tojson.exp b/tests/lang/eval-okay-tojson.exp index 33588493f75..e92aae3235f 100644 --- a/tests/lang/eval-okay-tojson.exp +++ b/tests/lang/eval-okay-tojson.exp @@ -1 +1 @@ -"{\"a\":123,\"b\":-456,\"c\":\"foo\",\"d\":\"foo\\n\\\"bar\\\"\",\"e\":true,\"f\":false,\"g\":[1,2,3],\"h\":[\"a\",[\"b\",{\"foo\\nbar\":{}}]],\"i\":3,\"j\":1.44}" +"{\"a\":123,\"b\":-456,\"c\":\"foo\",\"d\":\"foo\\n\\\"bar\\\"\",\"e\":true,\"f\":false,\"g\":[1,2,3],\"h\":[\"a\",[\"b\",{\"foo\\nbar\":{}}]],\"i\":3,\"j\":1.44,\"k\":\"foo\"}" diff --git a/tests/lang/eval-okay-tojson.nix b/tests/lang/eval-okay-tojson.nix index c046ba4ae59..ce67943bead 100644 --- a/tests/lang/eval-okay-tojson.nix +++ b/tests/lang/eval-okay-tojson.nix @@ -9,4 +9,5 @@ builtins.toJSON h = [ "a" [ "b" { "foo\nbar" = {}; } ] ]; i = 1 + 2; j = 1.44; + k = { __toString = self: self.a; a = "foo"; }; } diff --git a/tests/lang/eval-okay-types.exp b/tests/lang/eval-okay-types.exp index 9a8ea0bcbd8..92a15329935 100644 --- a/tests/lang/eval-okay-types.exp +++ b/tests/lang/eval-okay-types.exp @@ -1 +1 @@ -[ true false true false true false true false true true true true true true true true true true true false true false "int" "bool" "string" "null" "set" "list" "lambda" "lambda" "lambda" "lambda" ] +[ true false true false true false true false true true true true true true true true true true true false true true true false "int" "bool" "string" "null" "set" "list" "lambda" "lambda" "lambda" "lambda" ] diff --git a/tests/lang/eval-okay-types.nix b/tests/lang/eval-okay-types.nix index a34775f5e60..9b58be5d1dd 100644 --- a/tests/lang/eval-okay-types.nix +++ b/tests/lang/eval-okay-types.nix @@ -20,6 +20,8 @@ with builtins; (isFloat (1 - 2.0)) (isBool (true && false)) (isBool null) + (isPath /nix/store) + (isPath ./.) (isAttrs { x = 123; }) (isAttrs null) (typeOf (3 * 4)) diff --git a/tests/lang/parse-fail-mixed-nested-attrs1.nix b/tests/lang/parse-fail-mixed-nested-attrs1.nix new file mode 100644 index 00000000000..11e40e66fd1 --- /dev/null +++ b/tests/lang/parse-fail-mixed-nested-attrs1.nix @@ -0,0 +1,4 @@ +{ + x.z = 3; + x = { y = 3; z = 3; }; +} diff --git a/tests/lang/parse-fail-mixed-nested-attrs2.nix b/tests/lang/parse-fail-mixed-nested-attrs2.nix new file mode 100644 index 00000000000..17da82e5f0c --- /dev/null +++ b/tests/lang/parse-fail-mixed-nested-attrs2.nix @@ -0,0 +1,4 @@ +{ + x.y.y = 3; + x = { y.y= 3; z = 3; }; +} diff --git a/tests/lang/parse-fail-uft8.nix b/tests/lang/parse-fail-uft8.nix new file mode 100644 index 00000000000..34948d48aed --- /dev/null +++ b/tests/lang/parse-fail-uft8.nix @@ -0,0 +1 @@ +123 é 4 diff --git a/tests/lang/parse-fail-dup-attrs-6.nix b/tests/lang/parse-okay-dup-attrs-6.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-6.nix rename to tests/lang/parse-okay-dup-attrs-6.nix diff --git a/tests/lang/parse-okay-mixed-nested-attrs-1.nix b/tests/lang/parse-okay-mixed-nested-attrs-1.nix new file mode 100644 index 00000000000..fd1001c8caf --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-1.nix @@ -0,0 +1,4 @@ +{ + x = { y = 3; z = 3; }; + x.q = 3; +} diff --git a/tests/lang/parse-okay-mixed-nested-attrs-2.nix b/tests/lang/parse-okay-mixed-nested-attrs-2.nix new file mode 100644 index 00000000000..ad066b68038 --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-2.nix @@ -0,0 +1,4 @@ +{ + x.q = 3; + x = { y = 3; z = 3; }; +} diff --git a/tests/lang/parse-okay-mixed-nested-attrs-3.nix b/tests/lang/parse-okay-mixed-nested-attrs-3.nix new file mode 100644 index 00000000000..45a33e48037 --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-3.nix @@ -0,0 +1,7 @@ +{ + services.ssh.enable = true; + services.ssh = { port = 123; }; + services = { + httpd.enable = true; + }; +} diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh new file mode 100644 index 00000000000..16abd974c5e --- /dev/null +++ b/tests/linux-sandbox.sh @@ -0,0 +1,37 @@ +source common.sh + +clearStore + +if ! canUseSandbox; then exit; fi + +# Note: we need to bind-mount $SHELL into the chroot. Currently we +# only support the case where $SHELL is in the Nix store, because +# otherwise things get complicated (e.g. if it's in /bin, do we need +# /lib as well?). +if [[ ! $SHELL =~ /nix/store ]]; then exit; fi + +chmod -R u+w $TEST_ROOT/store0 || true +rm -rf $TEST_ROOT/store0 + +export NIX_STORE_DIR=/my/store +export NIX_REMOTE=$TEST_ROOT/store0 + +outPath=$(nix-build dependencies.nix --no-out-link --sandbox-paths /nix/store) + +[[ $outPath =~ /my/store/.*-dependencies ]] + +nix path-info -r $outPath | grep input-2 + +nix ls-store -R -l $outPath | grep foobar + +nix cat-store $outPath/foobar | grep FOOBAR + +# Test --check without hash rewriting. +nix-build dependencies.nix --no-out-link --check --sandbox-paths /nix/store + +# Test that sandboxed builds with --check and -K can move .check directory to store +nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link + +(! nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link --check -K 2> $TEST_ROOT/log) +if grep -q 'error: renaming' $TEST_ROOT/log; then false; fi +grep -q 'may not be deterministic' $TEST_ROOT/log diff --git a/tests/local.mk b/tests/local.mk index b3ce39cda80..536661af88e 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -1,9 +1,9 @@ -check: - @echo "Warning: Nix has no 'make check'. Please install Nix and run 'make installcheck' instead." - nix_tests = \ init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ - build-hook.sh gc.sh gc-concurrent.sh \ + config.sh \ + gc.sh \ + gc-concurrent.sh \ + gc-auto.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh check-refs.sh filter-source.sh \ remote-store.sh export.sh export-graph.sh \ @@ -11,7 +11,27 @@ nix_tests = \ multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \ check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \ - placeholders.sh nix-shell.sh + placeholders.sh nix-shell.sh \ + linux-sandbox.sh \ + build-dry.sh \ + build-remote.sh \ + nar-access.sh \ + structured-attrs.sh \ + fetchGit.sh \ + fetchGitRefs.sh \ + fetchGitSubmodules.sh \ + fetchMercurial.sh \ + signing.sh \ + shell.sh \ + brotli.sh \ + pure-eval.sh \ + check.sh \ + plugins.sh \ + search.sh \ + nix-copy-ssh.sh \ + post-hook.sh \ + function-trace.sh \ + recursive.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) @@ -20,4 +40,4 @@ tests-environment = NIX_REMOTE= $(bash) -e clean-files += $(d)/common.sh -installcheck: $(d)/common.sh +installcheck: $(d)/common.sh $(d)/config.nix $(d)/plugins/libplugintest.$(SO_EXT) diff --git a/tests/logging.sh b/tests/logging.sh index 86f32bade94..c894ad3ff07 100644 --- a/tests/logging.sh +++ b/tests/logging.sh @@ -11,5 +11,5 @@ path=$(nix-build dependencies.nix --no-out-link) clearStore rm -rf $NIX_LOG_DIR (! nix-store -l $path) -nix-build dependencies.nix --no-out-link --option build-compress-log true +nix-build dependencies.nix --no-out-link --compress-build-log [ "$(nix-store -l $path)" = FOO ] diff --git a/tests/misc.sh b/tests/misc.sh index 6d0ab3adcec..fd4908e25dc 100644 --- a/tests/misc.sh +++ b/tests/misc.sh @@ -16,4 +16,6 @@ nix-env --foo 2>&1 | grep "no operation" nix-env -q --foo 2>&1 | grep "unknown flag" # Eval Errors. -nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 | grep "infinite recursion encountered, at (string):1:15$" +eval_res=$(nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 || true) +echo $eval_res | grep "(string) (1:15)" +echo $eval_res | grep "infinite recursion encountered" diff --git a/tests/multiple-outputs.sh b/tests/multiple-outputs.sh index 7ac77908151..bedbc39a4eb 100644 --- a/tests/multiple-outputs.sh +++ b/tests/multiple-outputs.sh @@ -2,6 +2,8 @@ source common.sh clearStore +rm -f $TEST_ROOT/result* + # Test whether read-only evaluation works when referring to the # ‘drvPath’ attribute. echo "evaluating c..." @@ -28,7 +30,7 @@ echo "output path is $outPath" [ "$(cat "$outPath"/file)" = "success" ] # Test nix-build on a derivation with multiple outputs. -nix-build multiple-outputs.nix -A a -o $TEST_ROOT/result +outPath1=$(nix-build multiple-outputs.nix -A a -o $TEST_ROOT/result) [ -e $TEST_ROOT/result-first ] (! [ -e $TEST_ROOT/result-second ]) nix-build multiple-outputs.nix -A a.all -o $TEST_ROOT/result @@ -37,6 +39,17 @@ nix-build multiple-outputs.nix -A a.all -o $TEST_ROOT/result [ "$(cat $TEST_ROOT/result-second/link/file)" = "first" ] hash1=$(nix-store -q --hash $TEST_ROOT/result-second) +outPath2=$(nix-build $(nix-instantiate multiple-outputs.nix -A a) --no-out-link) +[[ $outPath1 = $outPath2 ]] + +outPath2=$(nix-build $(nix-instantiate multiple-outputs.nix -A a.first) --no-out-link) +[[ $outPath1 = $outPath2 ]] + +outPath2=$(nix-build $(nix-instantiate multiple-outputs.nix -A a.second) --no-out-link) +[[ $(cat $outPath2/file) = second ]] + +[[ $(nix-build $(nix-instantiate multiple-outputs.nix -A a.all) --no-out-link | wc -l) -eq 2 ]] + # Delete one of the outputs and rebuild it. This will cause a hash # rewrite. nix-store --delete $TEST_ROOT/result-second --ignore-liveness @@ -59,5 +72,5 @@ fi echo "collecting garbage..." rm $TEST_ROOT/result* -nix-store --gc --option gc-keep-derivations true --option gc-keep-outputs true +nix-store --gc --keep-derivations --keep-outputs nix-store --gc --print-roots diff --git a/tests/nar-access.nix b/tests/nar-access.nix new file mode 100644 index 00000000000..0e2a7f72113 --- /dev/null +++ b/tests/nar-access.nix @@ -0,0 +1,23 @@ +with import ./config.nix; + +rec { + a = mkDerivation { + name = "nar-index-a"; + builder = builtins.toFile "builder.sh" + '' + mkdir $out + mkdir $out/foo + touch $out/foo-x + touch $out/foo/bar + touch $out/foo/baz + touch $out/qux + mkdir $out/zyx + + cat >$out/foo/data < $narFile + +# Check that find and ls-nar match. +( cd $storePath; find . | sort ) > files.find +nix ls-nar -R -d $narFile "" | sort > files.ls-nar +diff -u files.find files.ls-nar + +# Check that file contents of data match. +nix cat-nar $narFile /foo/data > data.cat-nar +diff -u data.cat-nar $storePath/foo/data + +# Check that file contents of baz match. +nix cat-nar $narFile /foo/baz > baz.cat-nar +diff -u baz.cat-nar $storePath/foo/baz + +nix cat-store $storePath/foo/baz > baz.cat-nar +diff -u baz.cat-nar $storePath/foo/baz + +# Test --json. +[[ $(nix ls-nar --json $narFile /) = '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' ]] +[[ $(nix ls-nar --json -R $narFile /foo) = '{"type":"directory","entries":{"bar":{"type":"regular","size":0,"narOffset":368},"baz":{"type":"regular","size":0,"narOffset":552},"data":{"type":"regular","size":58,"narOffset":736}}}' ]] +[[ $(nix ls-nar --json -R $narFile /foo/bar) = '{"type":"regular","size":0,"narOffset":368}' ]] +[[ $(nix ls-store --json $storePath) = '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' ]] +[[ $(nix ls-store --json -R $storePath/foo) = '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' ]] +[[ $(nix ls-store --json -R $storePath/foo/bar) = '{"type":"regular","size":0}' ]] + +# Test missing files. +nix ls-store --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR' +nix ls-store $storePath/xyzzy 2>&1 | grep 'does not exist' + +# Test failure to dump. +if nix-store --dump $storePath >/dev/full ; then + echo "dumping to /dev/full should fail" + exit -1 +fi diff --git a/tests/nix-build.sh b/tests/nix-build.sh index dc0e99c7362..0eb5996087e 100644 --- a/tests/nix-build.sh +++ b/tests/nix-build.sh @@ -2,7 +2,7 @@ source common.sh clearStore -nix-build dependencies.nix -o $TEST_ROOT/result +outPath=$(nix-build dependencies.nix -o $TEST_ROOT/result) test "$(cat $TEST_ROOT/result/foobar)" = FOOBAR # The result should be retained by a GC. @@ -17,3 +17,12 @@ test -e $target/foobar rm $TEST_ROOT/result nix-store --gc if test -e $target/foobar; then false; fi + +outPath2=$(nix-build $(nix-instantiate dependencies.nix) --no-out-link) +[[ $outPath = $outPath2 ]] + +outPath2=$(nix-build $(nix-instantiate dependencies.nix)!out --no-out-link) +[[ $outPath = $outPath2 ]] + +outPath2=$(nix-store -r $(nix-instantiate --indirect --add-root $TEST_ROOT/indirect dependencies.nix)!out) +[[ $outPath = $outPath2 ]] diff --git a/tests/nix-channel.sh b/tests/nix-channel.sh index 553ada51d9f..49c68981acf 100644 --- a/tests/nix-channel.sh +++ b/tests/nix-channel.sh @@ -15,7 +15,7 @@ nix-channel --remove xyzzy # Create a channel. rm -rf $TEST_ROOT/foo mkdir -p $TEST_ROOT/foo -nix copy --recursive --to file://$TEST_ROOT/foo?compression="bzip2" $(nix-store -r $(nix-instantiate dependencies.nix)) +nix copy --to file://$TEST_ROOT/foo?compression="bzip2" $(nix-store -r $(nix-instantiate dependencies.nix)) rm -rf $TEST_ROOT/nixexprs mkdir -p $TEST_ROOT/nixexprs cp config.nix dependencies.nix dependencies.builder*.sh $TEST_ROOT/nixexprs/ @@ -32,11 +32,11 @@ if [ "$xmllint" != false ]; then $xmllint --noout $TEST_ROOT/meta.xml || fail "malformed XML" fi grep -q 'meta.*description.*Random test package' $TEST_ROOT/meta.xml -grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml +grep -q 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml # Do an install. -nix-env -i dependencies -[ -e $TEST_ROOT/var/nix/profiles/default/foobar ] +nix-env -i dependencies-top +[ -e $TEST_HOME/.nix-profile/foobar ] clearProfiles rm -f $TEST_HOME/.nix-channels @@ -51,9 +51,9 @@ if [ "$xmllint" != false ]; then $xmllint --noout $TEST_ROOT/meta.xml || fail "malformed XML" fi grep -q 'meta.*description.*Random test package' $TEST_ROOT/meta.xml -grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml +grep -q 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml # Do an install. -nix-env -i dependencies -[ -e $TEST_ROOT/var/nix/profiles/default/foobar ] +nix-env -i dependencies-top +[ -e $TEST_HOME/.nix-profile/foobar ] diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix index 44126dd64e4..bb5db74107f 100644 --- a/tests/nix-copy-closure.nix +++ b/tests/nix-copy-closure.nix @@ -1,18 +1,18 @@ # Test ‘nix-copy-closure’. -{ system, nix }: +{ nixpkgs, system, nix }: -with import { inherit system; }; +with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { nodes = { client = - { config, pkgs, ... }: + { config, lib, pkgs, ... }: { virtualisation.writableStore = true; virtualisation.pathsInNixDB = [ pkgA ]; nix.package = nix; - nix.binaryCaches = [ ]; + nix.binaryCaches = lib.mkForce [ ]; }; server = @@ -29,10 +29,10 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { startAll; # Create an SSH key on the client. - my $key = `${pkgs.openssh}/bin/ssh-keygen -t dsa -f key -N ""`; + my $key = `${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f key -N ""`; $client->succeed("mkdir -m 700 /root/.ssh"); - $client->copyFileFromHost("key", "/root/.ssh/id_dsa"); - $client->succeed("chmod 600 /root/.ssh/id_dsa"); + $client->copyFileFromHost("key", "/root/.ssh/id_ed25519"); + $client->succeed("chmod 600 /root/.ssh/id_ed25519"); # Install the SSH key on the server. $server->succeed("mkdir -m 700 /root/.ssh"); diff --git a/tests/nix-copy-ssh.sh b/tests/nix-copy-ssh.sh new file mode 100644 index 00000000000..eb801548d2f --- /dev/null +++ b/tests/nix-copy-ssh.sh @@ -0,0 +1,20 @@ +source common.sh + +clearStore +clearCache + +remoteRoot=$TEST_ROOT/store2 +chmod -R u+w "$remoteRoot" || true +rm -rf "$remoteRoot" + +outPath=$(nix-build --no-out-link dependencies.nix) + +nix copy --to "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath + +[ -f $remoteRoot$outPath/foobar ] + +clearStore + +nix copy --no-check-sigs --from "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath + +[ -f $outPath/foobar ] diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index b244815e290..e2e0d109080 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -7,8 +7,3 @@ rm -rf $TEST_HOME $TEST_ROOT/profile-var mkdir -p $TEST_HOME USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh; set" USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh" # test idempotency - -[ -L $TEST_HOME/.nix-profile ] -[ -e $TEST_HOME/.nix-channels ] -[ -e $TEST_ROOT/profile-var/nix/gcroots/per-user/$user ] -[ -e $TEST_ROOT/profile-var/nix/profiles/per-user/$user ] diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index 26cc521bbcb..235e2a5ffbf 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -4,10 +4,37 @@ clearStore # Test nix-shell -A export IMPURE_VAR=foo +export SELECTED_IMPURE_VAR=baz +export NIX_BUILD_SHELL=$SHELL output=$(nix-shell --pure shell.nix -A shellDrv --run \ - 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') -[ "$output" = " - foo - bar" ] +[ "$output" = " - foo - bar - true" ] + +# Test --keep +output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $SELECTED_IMPURE_VAR"') + +[ "$output" = " - foo - bar - baz" ] + +# Test nix-shell on a .drv +[[ $(nix-shell --pure $(nix-instantiate shell.nix -A shellDrv) --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') = " - foo - bar - false" ]] + +[[ $(nix-shell --pure $(nix-instantiate shell.nix -A shellDrv) --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') = " - foo - bar - false" ]] + +# Test nix-shell on a .drv symlink + +# Legacy: absolute path and .drv extension required +nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell.drv +[[ $(nix-shell --pure $TEST_ROOT/shell.drv --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] + +# New behaviour: just needs to resolve to a derivation in the store +nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell +[[ $(nix-shell --pure $TEST_ROOT/shell --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] # Test nix-shell -p output=$(NIX_PATH=nixpkgs=shell.nix nix-shell --pure -p foo bar --run 'echo "$(foo) $(bar)"') @@ -19,3 +46,12 @@ chmod a+rx $TEST_ROOT/shell.shebang.sh output=$($TEST_ROOT/shell.shebang.sh abc def) [ "$output" = "foo bar abc def" ] + +# Test nix-shell shebang mode for ruby +# This uses a fake interpreter that returns the arguments passed +# This, in turn, verifies the `rc` script is valid and the `load()` script (given using `-e`) is as expected. +sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/shell.shebang.rb +chmod a+rx $TEST_ROOT/shell.shebang.rb + +output=$($TEST_ROOT/shell.shebang.rb abc ruby) +[ "$output" = '-e load("'"$TEST_ROOT"'/shell.shebang.rb") -- abc ruby' ] diff --git a/tests/optimise-store.sh b/tests/optimise-store.sh index ea4478693e7..61e3df2f9f7 100644 --- a/tests/optimise-store.sh +++ b/tests/optimise-store.sh @@ -2,17 +2,17 @@ source common.sh clearStore -outPath1=$(echo 'with import ./config.nix; mkDerivation { name = "foo1"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --option auto-optimise-store true) -outPath2=$(echo 'with import ./config.nix; mkDerivation { name = "foo2"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --option auto-optimise-store true) +outPath1=$(echo 'with import ./config.nix; mkDerivation { name = "foo1"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store) +outPath2=$(echo 'with import ./config.nix; mkDerivation { name = "foo2"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store) -inode1="$(perl -e "print ((lstat('$outPath1/foo'))[1])")" -inode2="$(perl -e "print ((lstat('$outPath2/foo'))[1])")" +inode1="$(stat --format=%i $outPath1/foo)" +inode2="$(stat --format=%i $outPath2/foo)" if [ "$inode1" != "$inode2" ]; then echo "inodes do not match" exit 1 fi -nlink="$(perl -e "print ((lstat('$outPath1/foo'))[3])")" +nlink="$(stat --format=%h $outPath1/foo)" if [ "$nlink" != 3 ]; then echo "link count incorrect" exit 1 @@ -20,7 +20,7 @@ fi outPath3=$(echo 'with import ./config.nix; mkDerivation { name = "foo3"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link) -inode3="$(perl -e "print ((lstat('$outPath3/foo'))[1])")" +inode3="$(stat --format=%i $outPath3/foo)" if [ "$inode1" = "$inode3" ]; then echo "inodes match unexpectedly" exit 1 @@ -28,8 +28,8 @@ fi nix-store --optimise -inode1="$(perl -e "print ((lstat('$outPath1/foo'))[1])")" -inode3="$(perl -e "print ((lstat('$outPath3/foo'))[1])")" +inode1="$(stat --format=%i $outPath1/foo)" +inode3="$(stat --format=%i $outPath3/foo)" if [ "$inode1" != "$inode3" ]; then echo "inodes do not match" exit 1 diff --git a/tests/pass-as-file.sh b/tests/pass-as-file.sh index 3dfe10baa23..2c0bc5031ad 100644 --- a/tests/pass-as-file.sh +++ b/tests/pass-as-file.sh @@ -10,6 +10,7 @@ mkDerivation { passAsFile = [ \"foo\" ]; foo = [ \"xyzzy\" ]; builder = builtins.toFile \"builder.sh\" '' + [ \"\$(basename \$fooPath)\" = .attr-1bp7cri8hplaz6hbz0v4f0nl44rl84q1sg25kgwqzipzd1mv89ic ] [ \"\$(cat \$fooPath)\" = xyzzy ] touch \$out ''; diff --git a/tests/placeholders.sh b/tests/placeholders.sh index 071cfe2dc89..cd1bb7bc2aa 100644 --- a/tests/placeholders.sh +++ b/tests/placeholders.sh @@ -18,5 +18,3 @@ nix-build --no-out-link -E ' "; } ' - -echo XYZZY diff --git a/tests/plugins.sh b/tests/plugins.sh new file mode 100644 index 00000000000..4b1baeddce3 --- /dev/null +++ b/tests/plugins.sh @@ -0,0 +1,7 @@ +source common.sh + +set -o pipefail + +res=$(nix eval '(builtins.anotherNull)' --option setting-set true --option plugin-files $PWD/plugins/libplugintest*) + +[ "$res"x = "nullx" ] diff --git a/tests/plugins/local.mk b/tests/plugins/local.mk new file mode 100644 index 00000000000..82ad9940286 --- /dev/null +++ b/tests/plugins/local.mk @@ -0,0 +1,11 @@ +libraries += libplugintest + +libplugintest_DIR := $(d) + +libplugintest_SOURCES := $(d)/plugintest.cc + +libplugintest_ALLOW_UNDEFINED := 1 + +libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1 + +libplugintest_CXXFLAGS := -I src/libutil -I src/libexpr diff --git a/tests/plugins/plugintest.cc b/tests/plugins/plugintest.cc new file mode 100644 index 00000000000..c085d33295b --- /dev/null +++ b/tests/plugins/plugintest.cc @@ -0,0 +1,24 @@ +#include "config.hh" +#include "primops.hh" + +using namespace nix; + +struct MySettings : Config +{ + Setting settingSet{this, false, "setting-set", + "Whether the plugin-defined setting was set"}; +}; + +MySettings mySettings; + +static GlobalConfig::Register rs(&mySettings); + +static void prim_anotherNull (EvalState & state, const Pos & pos, Value ** args, Value & v) +{ + if (mySettings.settingSet) + mkNull(v); + else + mkBool(v, false); +} + +static RegisterPrimOp rp("anotherNull", 0, prim_anotherNull); diff --git a/tests/post-hook.sh b/tests/post-hook.sh new file mode 100644 index 00000000000..a026572154d --- /dev/null +++ b/tests/post-hook.sh @@ -0,0 +1,15 @@ +source common.sh + +clearStore + +export REMOTE_STORE=$TEST_ROOT/remote_store + +# Build the dependencies and push them to the remote store +nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook $PWD/push-to-store.sh + +clearStore + +# Ensure that we the remote store contains both the runtime and buildtime +# closure of what we've just built +nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix +nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix input1_drv diff --git a/tests/pure-eval.nix b/tests/pure-eval.nix new file mode 100644 index 00000000000..ed25b3d4563 --- /dev/null +++ b/tests/pure-eval.nix @@ -0,0 +1,3 @@ +{ + x = 123; +} diff --git a/tests/pure-eval.sh b/tests/pure-eval.sh new file mode 100644 index 00000000000..49c8564487c --- /dev/null +++ b/tests/pure-eval.sh @@ -0,0 +1,18 @@ +source common.sh + +clearStore + +nix eval --pure-eval '(assert 1 + 2 == 3; true)' + +[[ $(nix eval '(builtins.readFile ./pure-eval.sh)') =~ clearStore ]] + +(! nix eval --pure-eval '(builtins.readFile ./pure-eval.sh)') + +(! nix eval --pure-eval '(builtins.currentTime)') +(! nix eval --pure-eval '(builtins.currentSystem)') + +(! nix-instantiate --pure-eval ./simple.nix) + +[[ $(nix eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x)") == 123 ]] +(! nix eval --pure-eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x)") +nix eval --pure-eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; sha256 = \"$(nix hash-file pure-eval.nix --type sha256)\"; })).x)" diff --git a/tests/push-to-store.sh b/tests/push-to-store.sh new file mode 100755 index 00000000000..6aadb916ba0 --- /dev/null +++ b/tests/push-to-store.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +echo Pushing "$@" to "$REMOTE_STORE" +printf "%s" "$OUT_PATHS" | xargs -d: nix copy --to "$REMOTE_STORE" --no-require-sigs diff --git a/tests/recursive.sh b/tests/recursive.sh new file mode 100644 index 00000000000..394ae5ddbb6 --- /dev/null +++ b/tests/recursive.sh @@ -0,0 +1,72 @@ +source common.sh + +# FIXME +if [[ $(uname) != Linux ]]; then exit; fi + +clearStore + +export unreachable=$(nix add-to-store ./recursive.sh) + +nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L '( + with import ./config.nix; + with import ; + mkDerivation { + name = "recursive"; + dummy = builtins.toFile "dummy" "bla bla"; + SHELL = shell; + + # Note: this is a string without context. + unreachable = builtins.getEnv "unreachable"; + + requiredSystemFeatures = [ "recursive-nix" ]; + + buildCommand = '\'\'' + mkdir $out + PATH=${nixBinDir}:$PATH + opts="--experimental-features nix-command" + + # Check that we can query/build paths in our input closure. + nix $opts path-info $dummy + nix $opts build $dummy + + # Make sure we cannot query/build paths not in out input closure. + [[ -e $unreachable ]] + (! nix $opts path-info $unreachable) + (! nix $opts build $unreachable) + + # Add something to the store. + echo foobar > foobar + foobar=$(nix $opts add-to-store ./foobar) + + nix $opts path-info $foobar + nix $opts build $foobar + + # Add it to our closure. + ln -s $foobar $out/foobar + + [[ $(nix $opts path-info --all | wc -l) -eq 3 ]] + + # Build a derivation. + nix $opts build -L '\''( + derivation { + name = "inner1"; + builder = builtins.getEnv "SHELL"; + system = builtins.getEnv "system"; + fnord = builtins.toFile "fnord" "fnord"; + args = [ "-c" "echo $fnord blaat > $out" ]; + } + )'\'' + + [[ $(nix $opts path-info --json ./result) =~ fnord ]] + + ln -s $(nix $opts path-info ./result) $out/inner1 + '\'\''; + }) +' + +[[ $(cat $TEST_ROOT/result/inner1) =~ blaat ]] + +# Make sure the recursively created paths are in the closure. +nix path-info -r $TEST_ROOT/result | grep foobar +nix path-info -r $TEST_ROOT/result | grep fnord +nix path-info -r $TEST_ROOT/result | grep inner1 diff --git a/tests/referrers.sh b/tests/referrers.sh index 8ab8e5ddfe8..614dd8f5b96 100644 --- a/tests/referrers.sh +++ b/tests/referrers.sh @@ -4,9 +4,9 @@ clearStore max=500 -reference=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +reference=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bla touch $reference -(echo $reference && echo && echo 0) | nix-store --register-validity +(echo $reference && echo && echo 0) | nix-store --register-validity echo "making registration..." diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix index d14d6ff7f05..18d4908300e 100644 --- a/tests/remote-builds.nix +++ b/tests/remote-builds.nix @@ -1,20 +1,20 @@ # Test Nix's remote build feature. -{ system, nix }: +{ nixpkgs, system, nix }: -with import { inherit system; }; +with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; makeTest ( let - # The configuration of the build slaves. - slave = + # The configuration of the remote builders. + builder = { config, pkgs, ... }: { services.openssh.enable = true; virtualisation.writableStore = true; nix.package = nix; - nix.useChroot = true; + nix.useSandbox = true; }; # Trivial Nix expression to build remotely. @@ -36,24 +36,23 @@ in { nodes = - { slave1 = slave; - slave2 = slave; + { builder1 = builder; + builder2 = builder; client = - { config, pkgs, ... }: + { config, lib, pkgs, ... }: { nix.maxJobs = 0; # force remote building nix.distributedBuilds = true; - nix.envVars = pkgs.lib.mkAfter { NIX_BUILD_HOOK = "${nix}/libexec/nix/build-remote"; }; nix.buildMachines = - [ { hostName = "slave1"; + [ { hostName = "builder1"; sshUser = "root"; - sshKey = "/root/.ssh/id_dsa"; + sshKey = "/root/.ssh/id_ed25519"; system = "i686-linux"; maxJobs = 1; } - { hostName = "slave2"; + { hostName = "builder2"; sshUser = "root"; - sshKey = "/root/.ssh/id_dsa"; + sshKey = "/root/.ssh/id_ed25519"; system = "i686-linux"; maxJobs = 1; } @@ -61,7 +60,7 @@ in virtualisation.writableStore = true; virtualisation.pathsInNixDB = [ config.system.build.extraUtils ]; nix.package = nix; - nix.binaryCaches = [ ]; + nix.binaryCaches = lib.mkForce [ ]; programs.ssh.extraConfig = "ConnectTimeout 30"; }; }; @@ -71,35 +70,38 @@ in startAll; # Create an SSH key on the client. - my $key = `${pkgs.openssh}/bin/ssh-keygen -t dsa -f key -N ""`; + my $key = `${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f key -N ""`; $client->succeed("mkdir -p -m 700 /root/.ssh"); - $client->copyFileFromHost("key", "/root/.ssh/id_dsa"); - $client->succeed("chmod 600 /root/.ssh/id_dsa"); + $client->copyFileFromHost("key", "/root/.ssh/id_ed25519"); + $client->succeed("chmod 600 /root/.ssh/id_ed25519"); - # Install the SSH key on the slaves. + # Install the SSH key on the builders. $client->waitForUnit("network.target"); - foreach my $slave ($slave1, $slave2) { - $slave->succeed("mkdir -p -m 700 /root/.ssh"); - $slave->copyFileFromHost("key.pub", "/root/.ssh/authorized_keys"); - $slave->waitForUnit("sshd"); - $client->succeed("ssh -o StrictHostKeyChecking=no " . $slave->name() . " 'echo hello world'"); + foreach my $builder ($builder1, $builder2) { + $builder->succeed("mkdir -p -m 700 /root/.ssh"); + $builder->copyFileFromHost("key.pub", "/root/.ssh/authorized_keys"); + $builder->waitForUnit("sshd"); + $client->succeed("ssh -o StrictHostKeyChecking=no " . $builder->name() . " 'echo hello world'"); } - # Perform a build and check that it was performed on the slave. - my $out = $client->succeed("nix-build ${expr nodes.client.config 1}"); - $slave1->succeed("test -e $out"); + # Perform a build and check that it was performed on the builder. + my $out = $client->succeed( + "nix-build ${expr nodes.client.config 1} 2> build-output", + "grep -q Hello build-output" + ); + $builder1->succeed("test -e $out"); # And a parallel build. my ($out1, $out2) = split /\s/, $client->succeed('nix-store -r $(nix-instantiate ${expr nodes.client.config 2})\!out $(nix-instantiate ${expr nodes.client.config 3})\!out'); - $slave1->succeed("test -e $out1 -o -e $out2"); - $slave2->succeed("test -e $out1 -o -e $out2"); + $builder1->succeed("test -e $out1 -o -e $out2"); + $builder2->succeed("test -e $out1 -o -e $out2"); # And a failing build. $client->fail("nix-build ${expr nodes.client.config 5}"); - # Test whether the build hook automatically skips unavailable slaves. - $slave1->block; + # Test whether the build hook automatically skips unavailable builders. + $builder1->block; $client->succeed("nix-build ${expr nodes.client.config 4}"); ''; diff --git a/tests/remote-store.sh b/tests/remote-store.sh index f2f2806d022..4cc73465aea 100644 --- a/tests/remote-store.sh +++ b/tests/remote-store.sh @@ -4,7 +4,7 @@ clearStore startDaemon -storeCleared=1 $SHELL ./user-envs.sh +storeCleared=1 NIX_REMOTE_=$NIX_REMOTE $SHELL ./user-envs.sh nix-store --dump-db > $TEST_ROOT/d1 NIX_REMOTE= nix-store --dump-db > $TEST_ROOT/d2 @@ -13,3 +13,7 @@ cmp $TEST_ROOT/d1 $TEST_ROOT/d2 nix-store --gc --max-freed 1K killDaemon + +user=$(whoami) +[ -e $NIX_STATE_DIR/gcroots/per-user/$user ] +[ -e $NIX_STATE_DIR/profiles/per-user/$user ] diff --git a/tests/repair.sh b/tests/repair.sh index 782838704da..ec7ad5dcaff 100644 --- a/tests/repair.sh +++ b/tests/repair.sh @@ -46,12 +46,12 @@ fi # --verify can fix it. clearCache -nix copy --recursive --to file://$cacheDir $path +nix copy --to file://$cacheDir $path chmod u+w $path2 rm -rf $path2 -nix-store --verify --check-contents --repair --option binary-caches "file://$cacheDir" +nix-store --verify --check-contents --repair --substituters "file://$cacheDir" --no-require-sigs if [ "$(nix-hash $path2)" != "$hash" -o -e $path2/bad ]; then echo "path not repaired properly" >&2 @@ -69,7 +69,7 @@ if nix-store --verify-path $path2; then exit 1 fi -nix-store --repair-path $path2 --option binary-caches "file://$cacheDir" +nix-store --repair-path $path2 --substituters "file://$cacheDir" --no-require-sigs if [ "$(nix-hash $path2)" != "$hash" -o -e $path2/bad ]; then echo "path not repaired properly" >&2 diff --git a/tests/restricted.nix b/tests/restricted.nix new file mode 100644 index 00000000000..e0ef5840209 --- /dev/null +++ b/tests/restricted.nix @@ -0,0 +1 @@ +1 + 2 diff --git a/tests/restricted.sh b/tests/restricted.sh index 19096a9f8dd..e02becc60e3 100644 --- a/tests/restricted.sh +++ b/tests/restricted.sh @@ -2,17 +2,50 @@ source common.sh clearStore -nix-instantiate --option restrict-eval true --eval -E '1 + 2' -(! nix-instantiate --option restrict-eval true ./simple.nix) -nix-instantiate --option restrict-eval true ./simple.nix -I src=. -nix-instantiate --option restrict-eval true ./simple.nix -I src1=simple.nix -I src2=config.nix -I src3=./simple.builder.sh +nix-instantiate --restrict-eval --eval -E '1 + 2' +(! nix-instantiate --restrict-eval ./restricted.nix) +(! nix-instantiate --eval --restrict-eval <(echo '1 + 2')) +nix-instantiate --restrict-eval ./simple.nix -I src=. +nix-instantiate --restrict-eval ./simple.nix -I src1=simple.nix -I src2=config.nix -I src3=./simple.builder.sh -(! nix-instantiate --option restrict-eval true --eval -E 'builtins.readFile ./simple.nix') -nix-instantiate --option restrict-eval true --eval -E 'builtins.readFile ./simple.nix' -I src=.. +(! nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix') +nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=.. -(! nix-instantiate --option restrict-eval true --eval -E 'builtins.readDir ../src/boost') -nix-instantiate --option restrict-eval true --eval -E 'builtins.readDir ../src/boost' -I src=../src +(! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel') +nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel' -I src=../src -(! nix-instantiate --option restrict-eval true --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ') -nix-instantiate --option restrict-eval true --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ' -I src=. +(! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ') +nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ' -I src=. +p=$(nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval --allowed-uris "file://$(pwd)") +cmp $p restricted.sh + +(! nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval) + +(! nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh/") + +nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh" + +(! nix eval --raw "(builtins.fetchurl https://github.com/NixOS/patchelf/archive/master.tar.gz)" --restrict-eval) +(! nix eval --raw "(builtins.fetchTarball https://github.com/NixOS/patchelf/archive/master.tar.gz)" --restrict-eval) +(! nix eval --raw "(fetchGit git://github.com/NixOS/patchelf.git)" --restrict-eval) + +ln -sfn $(pwd)/restricted.nix $TEST_ROOT/restricted.nix +[[ $(nix-instantiate --eval $TEST_ROOT/restricted.nix) == 3 ]] +(! nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix) +(! nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT) +(! nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I .) +nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT -I . + +[[ $(nix eval --raw --restrict-eval -I . '(builtins.readFile "${import ./simple.nix}/hello")') == 'Hello World!' ]] + +# Check whether we can leak symlink information through directory traversal. +traverseDir="$(pwd)/restricted-traverse-me" +ln -sfn "$(pwd)/restricted-secret" "$(pwd)/restricted-innocent" +mkdir -p "$traverseDir" +goUp="..$(echo "$traverseDir" | sed -e 's,[^/]\+,..,g')" +output="$(nix eval --raw --restrict-eval -I "$traverseDir" \ + "(builtins.readFile \"$traverseDir/$goUp$(pwd)/restricted-innocent\")" \ + 2>&1 || :)" +echo "$output" | grep "is forbidden" +! echo "$output" | grep -F restricted-secret diff --git a/tests/search.nix b/tests/search.nix new file mode 100644 index 00000000000..fea6e7a7a64 --- /dev/null +++ b/tests/search.nix @@ -0,0 +1,25 @@ +with import ./config.nix; + +{ + hello = mkDerivation rec { + name = "hello-${version}"; + version = "0.1"; + buildCommand = "touch $out"; + meta.description = "Empty file"; + }; + foo = mkDerivation rec { + name = "foo-5"; + buildCommand = '' + mkdir -p $out + echo ${name} > $out/${name} + ''; + }; + bar = mkDerivation rec { + name = "bar-3"; + buildCommand = '' + echo "Does not build successfully" + exit 1 + ''; + meta.description = "broken bar"; + }; +} diff --git a/tests/search.sh b/tests/search.sh new file mode 100644 index 00000000000..14da3127b0d --- /dev/null +++ b/tests/search.sh @@ -0,0 +1,43 @@ +source common.sh + +clearStore +clearCache + +# No packages +(( $(NIX_PATH= nix search -u|wc -l) == 0 )) + +# Haven't updated cache, still nothing +(( $(nix search -f search.nix hello|wc -l) == 0 )) +(( $(nix search -f search.nix |wc -l) == 0 )) + +# Update cache, search should work +(( $(nix search -f search.nix -u hello|wc -l) > 0 )) + +# Use cache +(( $(nix search -f search.nix foo|wc -l) > 0 )) +(( $(nix search foo|wc -l) > 0 )) + +# Test --no-cache works +# No results from cache +(( $(nix search --no-cache foo |wc -l) == 0 )) +# Does find results from file pointed at +(( $(nix search -f search.nix --no-cache foo |wc -l) > 0 )) + +# Check descriptions are searched +(( $(nix search broken | wc -l) > 0 )) + +# Check search that matches nothing +(( $(nix search nosuchpackageexists | wc -l) == 0 )) + +# Search for multiple arguments +(( $(nix search hello empty | wc -l) == 3 )) + +# Multiple arguments will not exist +(( $(nix search hello broken | wc -l) == 0 )) + +## Search expressions + +# Check that empty search string matches all +nix search|grep -q foo +nix search|grep -q bar +nix search|grep -q hello diff --git a/tests/setuid.nix b/tests/setuid.nix new file mode 100644 index 00000000000..63d3c05cb8e --- /dev/null +++ b/tests/setuid.nix @@ -0,0 +1,108 @@ +# Verify that Linux builds cannot create setuid or setgid binaries. + +{ nixpkgs, system, nix }: + +with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; + +makeTest { + + machine = + { config, lib, pkgs, ... }: + { virtualisation.writableStore = true; + nix.package = nix; + nix.binaryCaches = lib.mkForce [ ]; + nix.nixPath = [ "nixpkgs=${lib.cleanSource pkgs.path}" ]; + virtualisation.pathsInNixDB = [ pkgs.stdenv pkgs.pkgsi686Linux.stdenv ]; + }; + + testScript = { nodes }: + '' + startAll; + + # Copying to /tmp should succeed. + $machine->succeed('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + ")\' '); + + $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + + $machine->succeed("rm /tmp/id"); + + # Creating a setuid binary should fail. + $machine->fail('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + chmod 4755 /tmp/id + ")\' '); + + $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + + $machine->succeed("rm /tmp/id"); + + # Creating a setgid binary should fail. + $machine->fail('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + chmod 2755 /tmp/id + ")\' '); + + $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + + $machine->succeed("rm /tmp/id"); + + # The checks should also work on 32-bit binaries. + $machine->fail('nix-build --no-sandbox -E \'(with import { system = "i686-linux"; }; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + chmod 2755 /tmp/id + ")\' '); + + $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + + $machine->succeed("rm /tmp/id"); + + # The tests above use fchmodat(). Test chmod() as well. + $machine->succeed('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"chmod 0666, qw(/tmp/id) or die\" + ")\' '); + + $machine->succeed('[[ $(stat -c %a /tmp/id) = 666 ]]'); + + $machine->succeed("rm /tmp/id"); + + $machine->fail('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"chmod 04755, qw(/tmp/id) or die\" + ")\' '); + + $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + + $machine->succeed("rm /tmp/id"); + + # And test fchmod(). + $machine->succeed('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 01750, \\\$x or die\" + ")\' '); + + $machine->succeed('[[ $(stat -c %a /tmp/id) = 1750 ]]'); + + $machine->succeed("rm /tmp/id"); + + $machine->fail('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 04777, \\\$x or die\" + ")\' '); + + $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + + $machine->succeed("rm /tmp/id"); + ''; + +} diff --git a/tests/shell-hello.nix b/tests/shell-hello.nix new file mode 100644 index 00000000000..77dcbd2a9df --- /dev/null +++ b/tests/shell-hello.nix @@ -0,0 +1,17 @@ +with import ./config.nix; + +{ + hello = mkDerivation { + name = "hello"; + buildCommand = + '' + mkdir -p $out/bin + cat > $out/bin/hello < $out/bin/foo chmod a+rx $out/bin/foo + ln -s ${shell} $out/bin/bash ''; bar = runCommand "bar" {} '' @@ -43,4 +45,13 @@ rec { ''; bash = shell; -} + + # ruby "interpreter" that outputs "$@" + ruby = runCommand "ruby" {} '' + mkdir -p $out/bin + echo 'printf -- "$*"' > $out/bin/ruby + chmod a+rx $out/bin/ruby + ''; + + inherit pkgs; +}; in pkgs diff --git a/tests/shell.sh b/tests/shell.sh new file mode 100644 index 00000000000..7a9ee8ab099 --- /dev/null +++ b/tests/shell.sh @@ -0,0 +1,28 @@ +source common.sh + +clearStore +clearCache + +nix shell -f shell-hello.nix hello -c hello | grep 'Hello World' +nix shell -f shell-hello.nix hello -c hello NixOS | grep 'Hello NixOS' + +if ! canUseSandbox; then exit; fi + +chmod -R u+w $TEST_ROOT/store0 || true +rm -rf $TEST_ROOT/store0 + +clearStore + +path=$(nix eval --raw -f shell-hello.nix hello) + +# Note: we need the sandbox paths to ensure that the shell is +# visible in the sandbox. +nix shell --sandbox-build-dir /build-tmp \ + --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' \ + --store $TEST_ROOT/store0 -f shell-hello.nix hello -c hello | grep 'Hello World' + +path2=$(nix shell --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store $TEST_ROOT/store0 -f shell-hello.nix hello -c $SHELL -c 'type -p hello') + +[[ $path/bin/hello = $path2 ]] + +[[ -e $TEST_ROOT/store0/nix/store/$(basename $path)/bin/hello ]] diff --git a/tests/shell.shebang.rb b/tests/shell.shebang.rb new file mode 100644 index 00000000000..ea67eb09c1c --- /dev/null +++ b/tests/shell.shebang.rb @@ -0,0 +1,7 @@ +#! @SHELL_PROG@ +#! ruby +#! nix-shell -I nixpkgs=shell.nix --no-substitute +#! nix-shell --pure -p ruby -i ruby + +# Contents doesn't matter. +abort("This shouldn't be executed.") diff --git a/tests/shell.shebang.sh b/tests/shell.shebang.sh index 3dadd591572..f7132043de4 100755 --- a/tests/shell.shebang.sh +++ b/tests/shell.shebang.sh @@ -1,4 +1,4 @@ #! @ENV_PROG@ nix-shell -#! nix-shell -I nixpkgs=shell.nix --option use-binary-caches false +#! nix-shell -I nixpkgs=shell.nix --no-substitute #! nix-shell --pure -i bash -p foo bar echo "$(foo) $(bar) $@" diff --git a/tests/signing.sh b/tests/signing.sh new file mode 100644 index 00000000000..9e29e3fbf06 --- /dev/null +++ b/tests/signing.sh @@ -0,0 +1,105 @@ +source common.sh + +clearStore +clearCache + +nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1 +pk1=$(cat $TEST_ROOT/pk1) +nix-store --generate-binary-cache-key cache2.example.org $TEST_ROOT/sk2 $TEST_ROOT/pk2 +pk2=$(cat $TEST_ROOT/pk2) + +# Build a path. +outPath=$(nix-build dependencies.nix --no-out-link --secret-key-files "$TEST_ROOT/sk1 $TEST_ROOT/sk2") + +# Verify that the path got signed. +info=$(nix path-info --json $outPath) +[[ $info =~ '"ultimate":true' ]] +[[ $info =~ 'cache1.example.org' ]] +[[ $info =~ 'cache2.example.org' ]] + +# Test "nix verify". +nix verify -r $outPath + +expect 2 nix verify -r $outPath --sigs-needed 1 + +nix verify -r $outPath --sigs-needed 1 --trusted-public-keys $pk1 + +expect 2 nix verify -r $outPath --sigs-needed 2 --trusted-public-keys $pk1 + +nix verify -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" + +nix verify --all --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" + +# Build something unsigned. +outPath2=$(nix-build simple.nix --no-out-link) + +nix verify -r $outPath + +# Verify that the path did not get signed but does have the ultimate bit. +info=$(nix path-info --json $outPath2) +[[ $info =~ '"ultimate":true' ]] +(! [[ $info =~ 'signatures' ]]) + +# Test "nix verify". +nix verify -r $outPath2 + +expect 2 nix verify -r $outPath2 --sigs-needed 1 + +expect 2 nix verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 + +# Test "nix sign-paths". +nix sign-paths --key-file $TEST_ROOT/sk1 $outPath2 + +nix verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 + +# Build something content-addressed. +outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0 --no-out-link) + +[[ $(nix path-info --json $outPathCA) =~ '"ca":"fixed:md5:' ]] + +# Content-addressed paths don't need signatures, so they verify +# regardless of --sigs-needed. +nix verify $outPathCA +nix verify $outPathCA --sigs-needed 1000 + +# Check that signing a content-addressed path doesn't overflow validSigs +nix sign-paths --key-file $TEST_ROOT/sk1 $outPathCA +nix verify -r $outPathCA --sigs-needed 1000 --trusted-public-keys $pk1 + +# Copy to a binary cache. +nix copy --to file://$cacheDir $outPath2 + +# Verify that signatures got copied. +info=$(nix path-info --store file://$cacheDir --json $outPath2) +(! [[ $info =~ '"ultimate":true' ]]) +[[ $info =~ 'cache1.example.org' ]] +(! [[ $info =~ 'cache2.example.org' ]]) + +# Verify that adding a signature to a path in a binary cache works. +nix sign-paths --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2 +info=$(nix path-info --store file://$cacheDir --json $outPath2) +[[ $info =~ 'cache1.example.org' ]] +[[ $info =~ 'cache2.example.org' ]] + +# Copying to a diverted store should fail due to a lack of valid signatures. +chmod -R u+w $TEST_ROOT/store0 || true +rm -rf $TEST_ROOT/store0 +(! nix copy --to $TEST_ROOT/store0 $outPath) + +# But succeed if we supply the public keys. +nix copy --to $TEST_ROOT/store0 $outPath --trusted-public-keys $pk1 + +expect 2 nix verify --store $TEST_ROOT/store0 -r $outPath + +nix verify --store $TEST_ROOT/store0 -r $outPath --trusted-public-keys $pk1 +nix verify --store $TEST_ROOT/store0 -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" + +# It should also succeed if we disable signature checking. +(! nix copy --to $TEST_ROOT/store0 $outPath2) +nix copy --to $TEST_ROOT/store0?require-sigs=false $outPath2 + +# But signatures should still get copied. +nix verify --store $TEST_ROOT/store0 -r $outPath2 --trusted-public-keys $pk1 + +# Content-addressed stuff can be copied without signatures. +nix copy --to $TEST_ROOT/store0 $outPathCA diff --git a/tests/structured-attrs.nix b/tests/structured-attrs.nix new file mode 100644 index 00000000000..c39c3a346c5 --- /dev/null +++ b/tests/structured-attrs.nix @@ -0,0 +1,70 @@ +with import ./config.nix; + +let + + dep = mkDerivation { + name = "dep"; + buildCommand = '' + mkdir $out; echo bla > $out/bla + ''; + }; + +in + +mkDerivation { + name = "structured"; + + __structuredAttrs = true; + + outputs = [ "out" "dev" ]; + + buildCommand = '' + set -x + + [[ $int = 123456789 ]] + [[ -z $float ]] + [[ -n $boolTrue ]] + [[ -z $boolFalse ]] + [[ -n ''${hardening[format]} ]] + [[ -z ''${hardening[fortify]} ]] + [[ ''${#buildInputs[@]} = 7 ]] + [[ ''${buildInputs[2]} = c ]] + [[ -v nothing ]] + [[ -z $nothing ]] + + mkdir ''${outputs[out]} ''${outputs[dev]} + echo bar > $dest + echo foo > $dest2 + + json=$(cat .attrs.json) + [[ $json =~ '"narHash":"sha256:1r7yc43zqnzl5b0als5vnyp649gk17i37s7mj00xr8kc47rjcybk"' ]] + [[ $json =~ '"narSize":288' ]] + [[ $json =~ '"closureSize":288' ]] + [[ $json =~ '"references":[]' ]] + ''; + + buildInputs = [ "a" "b" "c" 123 "'" "\"" null ]; + + hardening.format = true; + hardening.fortify = false; + + outer.inner = [ 1 2 3 ]; + + int = 123456789; + + float = 123.456; + + boolTrue = true; + boolFalse = false; + + nothing = null; + + dest = "${placeholder "out"}/foo"; + dest2 = "${placeholder "dev"}/foo"; + + "foo bar" = "BAD"; + "1foobar" = "BAD"; + "foo$" = "BAD"; + + exportReferencesGraph.refs = [ dep ]; +} diff --git a/tests/structured-attrs.sh b/tests/structured-attrs.sh new file mode 100644 index 00000000000..646bdb876d3 --- /dev/null +++ b/tests/structured-attrs.sh @@ -0,0 +1,8 @@ +source common.sh + +clearStore + +nix-build structured-attrs.nix -A all -o $TEST_ROOT/result + +[[ $(cat $TEST_ROOT/result/foo) = bar ]] +[[ $(cat $TEST_ROOT/result-dev/foo) = foo ]] diff --git a/tests/tarball.sh b/tests/tarball.sh index ba534c6261a..b3ec16d4080 100644 --- a/tests/tarball.sh +++ b/tests/tarball.sh @@ -10,19 +10,40 @@ mkdir -p $tarroot cp dependencies.nix $tarroot/default.nix cp config.nix dependencies.builder*.sh $tarroot/ -tarball=$TEST_ROOT/tarball.tar.xz -(cd $TEST_ROOT && tar c tarball) | xz > $tarball +hash=$(nix hash-path $tarroot) -nix-env -f file://$tarball -qa --out-path | grep -q dependencies +test_tarball() { + local ext="$1" + local compressor="$2" -nix-build -o $TEST_ROOT/result file://$tarball + tarball=$TEST_ROOT/tarball.tar$ext + (cd $TEST_ROOT && tar c tarball) | $compressor > $tarball -nix-build -o $TEST_ROOT/result '' -I foo=file://$tarball + nix-env -f file://$tarball -qa --out-path | grep -q dependencies -nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)" + nix-build -o $TEST_ROOT/result file://$tarball -nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar.xz -nix-instantiate --eval -E 'with ; 1 + 2' -I fnord=file://no-such-tarball.tar.xz -(! nix-instantiate --eval -E ' 1' -I fnord=file://no-such-tarball.tar.xz) + nix-build -o $TEST_ROOT/result '' -I foo=file://$tarball -nix-instantiate --eval -E '' -I fnord=file://no-such-tarball.tar.xz -I fnord=. + nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)" + + nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)" + nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })" + nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })" + nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input' + + nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar$ext + nix-instantiate --eval -E 'with ; 1 + 2' -I fnord=file://no-such-tarball$ext + (! nix-instantiate --eval -E ' 1' -I fnord=file://no-such-tarball$ext) + + nix-instantiate --eval -E '' -I fnord=file://no-such-tarball$ext -I fnord=. +} + +test_tarball '' cat +test_tarball .xz xz +test_tarball .gz gzip + +rm -rf $TEST_ROOT/tmp +mkdir -p $TEST_ROOT/tmp +(! TMPDIR=$TEST_ROOT/tmp XDG_RUNTIME_DIR=$TEST_ROOT/tmp nix-env -f file://$(pwd)/bad.tar.xz -qa --out-path) +(! [ -e $TEST_ROOT/tmp/bad ]) diff --git a/tests/timeout.nix b/tests/timeout.nix index 540fba934ff..d0e949e3149 100644 --- a/tests/timeout.nix +++ b/tests/timeout.nix @@ -5,7 +5,8 @@ with import ./config.nix; infiniteLoop = mkDerivation { name = "timeout"; buildCommand = '' - echo "‘timeout’ builder entering an infinite loop" + touch $out + echo "'timeout' builder entering an infinite loop" while true ; do echo -n .; done ''; }; @@ -13,6 +14,7 @@ with import ./config.nix; silent = mkDerivation { name = "silent"; buildCommand = '' + touch $out sleep 60 ''; }; @@ -20,6 +22,7 @@ with import ./config.nix; closeLog = mkDerivation { name = "silent"; buildCommand = '' + touch $out exec > /dev/null 2>&1 sleep 1000000000 ''; diff --git a/tests/timeout.sh b/tests/timeout.sh index ce1ae7d674a..eea9b5731da 100644 --- a/tests/timeout.sh +++ b/tests/timeout.sh @@ -2,10 +2,14 @@ source common.sh -failed=0 -messages="`nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1 || failed=1`" -if [ $failed -ne 0 ]; then - echo "error: ‘nix-store’ succeeded; should have timed out" + +set +e +messages=$(nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1) +status=$? +set -e + +if [ $status -ne 101 ]; then + echo "error: 'nix-store' exited with '$status'; should have exited 101" exit 1 fi @@ -15,7 +19,7 @@ if ! echo "$messages" | grep -q "timed out"; then exit 1 fi -if nix-build -Q timeout.nix -A infiniteLoop --option build-max-log-size 100; then +if nix-build -Q timeout.nix -A infiniteLoop --max-build-log-size 100; then echo "build should have failed" exit 1 fi @@ -29,3 +33,8 @@ if nix-build timeout.nix -A closeLog; then echo "build should have failed" exit 1 fi + +if nix build -f timeout.nix silent --max-silent-time 2; then + echo "build should have failed" + exit 1 +fi diff --git a/tests/user-envs.sh b/tests/user-envs.sh index c4192fdc59b..aebf6a2a2b8 100644 --- a/tests/user-envs.sh +++ b/tests/user-envs.sh @@ -20,10 +20,13 @@ drvPath10=$(nix-env -f ./user-envs.nix -qa --drv-path --no-name '*' | grep foo-1 # Query descriptions. nix-env -f ./user-envs.nix -qa '*' --description | grep -q silly -rm -f $HOME/.nix-defexpr +rm -rf $HOME/.nix-defexpr ln -s $(pwd)/user-envs.nix $HOME/.nix-defexpr nix-env -qa '*' --description | grep -q silly +# Query the system. +nix-env -qa '*' --system | grep -q $system + # Install "foo-1.0". nix-env -i foo-1.0 diff --git a/version b/version deleted file mode 100644 index 35d51f33b34..00000000000 --- a/version +++ /dev/null @@ -1 +0,0 @@ -1.12 \ No newline at end of file