diff --git a/pkgs/build-support/singularity-tools/default.nix b/pkgs/build-support/singularity-tools/default.nix index 9689e41245909..3c38cf9ffd037 100644 --- a/pkgs/build-support/singularity-tools/default.nix +++ b/pkgs/build-support/singularity-tools/default.nix @@ -1,84 +1,56 @@ -{ runCommand -, lib +{ lib , stdenv -, storeDir ? builtins.storeDir -, writeScript -, singularity -, writeReferencesToFile -, bash +, runCommand , vmTools -, gawk +, writeMultipleReferencesToFile +, writeScript +, writeShellScriptBin +, writeText +, e2fsprogs , util-linux +, bash +, coreutils , runtimeShell -, e2fsprogs +, singularity +, storeDir ? builtins.storeDir }: -rec { - shellScript = name: text: - writeScript name '' - #!${runtimeShell} - set -e - ${text} - ''; - - mkLayer = - { name - , contents ? [ ] - # May be "apptainer" instead of "singularity" - , projectName ? (singularity.projectName or "singularity") - }: - runCommand "${projectName}-layer-${name}" - { - inherit contents; - } '' - mkdir $out - for f in $contents ; do - cp -ra $f $out/ - done - ''; - buildImage = +let + defaultSingularity = singularity; +in +rec { + buildSandboxFromContents = let - defaultSingularity = singularity; + shellScript = name: text: + writeScript name '' + #!${runtimeShell} + set -e + ${text} + ''; in { name , contents ? [ ] - , diskSize ? 1024 , runScript ? "#!${stdenv.shell}\nexec /bin/sh" , runAsRoot ? null - , memSize ? 512 - , singularity ? defaultSingularity + , projectName ? singularity.projectName or "singularity" }: let - projectName = singularity.projectName or "singularity"; - layer = mkLayer { - inherit name; - contents = contents ++ [ bash runScriptFile ]; - inherit projectName; - }; + layerClosure = writeMultipleReferencesToFile (contents ++ [ bash runScriptFile ]); runAsRootFile = shellScript "run-as-root.sh" runAsRoot; runScriptFile = shellScript "run-script.sh" runScript; - result = vmTools.runInLinuxVM ( - runCommand "${projectName}-image-${name}.img" - { - buildInputs = [ singularity e2fsprogs util-linux gawk ]; - layerClosure = writeReferencesToFile layer; - preVM = vmTools.createEmptyImage { - size = diskSize; - fullName = "${projectName}-run-disk"; - }; - inherit memSize; - } + buildscriptPackage = + writeShellScriptBin "build-sandbox" '' - rm -rf $out - mkdir disk - mkfs -t ext3 -b 4096 /dev/${vmTools.hd} - mount /dev/${vmTools.hd} disk - mkdir -p disk/img - cd disk/img + if [ "$#" -lt 1 ]; then + echo "Expect SANDBOX_PATH" >&2 + exit 1 + fi + pathSandbox="$1" + cd "$pathSandbox" mkdir proc sys dev - # Run root script ${lib.optionalString (runAsRoot != null) '' + # Run root script mkdir -p ./${storeDir} mount --rbind ${storeDir} ./${storeDir} unshare -imnpuf --mount-proc chroot ./ ${runAsRootFile} @@ -86,9 +58,9 @@ rec { ''} # Build /bin and copy across closure - mkdir -p bin ./${builtins.storeDir} - for f in $(cat $layerClosure) ; do - cp -ar $f ./$f + mkdir -p bin ./${storeDir} + for f in $(cat ${layerClosure}) ; do + cp -r $f ./$f done for c in ${toString contents} ; do @@ -109,14 +81,280 @@ rec { # Fill out .${projectName}.d mkdir -p .${projectName}.d/env touch .${projectName}.d/env/94-appsbase.sh + ''; + in + runCommand "${projectName}-sandbox-${name}" + { + passthru = { + inherit + buildscriptPackage + layerClosure + projectName + runAsRoot + runScript + ; + }; + } '' + runHook preImageBuild + mkdir -p "$out" + "${buildscriptPackage}/bin/${buildscriptPackage.meta.mainProgram}" "$out" + runHook postImageBuild + ''; + + buildImageFromSandbox = + { name + , sandbox ? "" + , contents ? [ ] + , runScript ? "#!${stdenv.shell}\nexec /bin/sh" + , runAsRoot ? null + , singularity ? defaultSingularity + , executableFlags ? [ ] + , buildImageFlags ? [ ] + }: + let + projectName = singularity.projectName or "singularity"; + sandboxFromContents = buildSandboxFromContents { + name = "from-contents"; + inherit contents runScript runAsRoot projectName; + }; + buildscriptPackage = writeShellScriptBin "build-image" '' + if [ "$#" -lt 1 ]; then + echo "Expect IMAGE_PATH" >&2 + exit 1 + fi + pathSIF="$1" + shift + ${if sandbox != "" then '' + pathSandbox=${sandbox} + '' else '' + pathSandbox="$(mktemp -t -d sandbox_XXXXXX)" + trap "rm -rf \"$pathSandbox\"" EXIT INT + "${sandboxFromContents.buildscriptPackage}/bin/${sandboxFromContents.buildscriptPackage.meta.mainProgram}" "$pathSandbox" + ''} + ${projectName} build "$pathSIF" "$pathSandbox" + ''; + in + runCommand "${projectName}-image-${name}.sif" + { + buildInputs = [ singularity util-linux ]; + passthru = { + sandbox = if (sandbox != "") then sandbox else sandboxFromContents; + layerClosure = if (sandbox != "") then sandbox.layerClosure else null; + inherit singularity; + }; + } + '' + runHook preImageBuild + "${buildscriptPackage}/bin/${buildscriptPackage.meta.mainProgram}" "$out" + runHook postImageBuild + ''; + + buildImageInLinuxVM = + { diskSize ? 1024 + , memSize ? 512 + }: + image: + let + projectName = image.projectName or image.singularity.projectName or "singularity"; + in + vmTools.runInLinuxVM (image.overrideAttrs (prevAttrs: { + buildInputs = prevAttrs.buildInputs or [ ] ++ [ + e2fsprogs + ]; + preVM = vmTools.createEmptyImage { + size = diskSize; + fullName = "${projectName}-run-disk"; + }; + preImageBuild = '' + rm -rf $out + mkdir disk + mkfs -t ext3 -b 4096 /dev/${vmTools.hd} + mount /dev/${vmTools.hd} disk + mkdir -p /var/lib/${projectName}/mnt/{container,final,overlay,session,source} + echo "root:x:0:0:System administrator:/root:/bin/sh" > /etc/passwd + echo > /etc/resolv.conf + TMPDIR="$(realpath disk)"; export TMPDIR + cd disk + ''; + })); + + buildImage = + { name + , contents ? [ ] + , diskSize ? 1024 + , runScript ? "#!${stdenv.shell}\nexec /bin/sh" + , runAsRoot ? null + , memSize ? 512 + , singularity ? defaultSingularity + }: + buildImageInLinuxVM + { + inherit diskSize memSize; + } + (buildImageFromSandbox { + inherit name contents runScript runAsRoot singularity; + }); + + # shellScript = name: text: + # writeScript name '' + # #!${runtimeShell} + # set -e + # ${text} + # ''; - cd .. - mkdir -p /var/lib/${projectName}/mnt/{container,final,overlay,session,source} - echo "root:x:0:0:System administrator:/root:/bin/sh" > /etc/passwd - echo > /etc/resolv.conf - TMPDIR=$(pwd -P) ${projectName} build $out ./img - ''); + # buildImage = + # let + # defaultSingularity = singularity; + # in + # { name + # , contents ? [ ] + # , diskSize ? 1024 + # , runScript ? "#!${stdenv.shell}\nexec /bin/sh" + # , runAsRoot ? null + # , memSize ? 512 + # , singularity ? defaultSingularity + # }: + # let + # projectName = singularity.projectName or "singularity"; + # runAsRootFile = shellScript "run-as-root.sh" runAsRoot; + # runScriptFile = shellScript "run-script.sh" runScript; + # result = vmTools.runInLinuxVM ( + # runCommand "${projectName}-image-${name}.sif" + # { + # buildInputs = [ singularity e2fsprogs util-linux ]; + # layerClosure = writeMultipleReferencesToFile (contents ++ [ bash runScriptFile ]); + # preVM = vmTools.createEmptyImage { + # size = diskSize; + # fullName = "${projectName}-run-disk"; + # }; + # inherit memSize; + # } + # '' + # rm -rf $out + # mkdir disk + # mkfs -t ext3 -b 4096 /dev/${vmTools.hd} + # mount /dev/${vmTools.hd} disk + # mkdir -p disk/img + # cd disk/img + # mkdir proc sys dev + # # Run root script + # ${lib.optionalString (runAsRoot != null) '' + # mkdir -p ./${storeDir} + # mount --rbind ${storeDir} ./${storeDir} + # unshare -imnpuf --mount-proc chroot ./ ${runAsRootFile} + # umount -R ./${storeDir} + # ''} + + # # Build /bin and copy across closure + # mkdir -p bin ./${storeDir} + # for f in $(cat $layerClosure) ; do + # cp -ar $f ./$f + # done + + # for c in ${toString contents} ; do + # for f in $c/bin/* ; do + # if [ ! -e bin/$(basename $f) ] ; then + # ln -s $f bin/ + # fi + # done + # done + + # # Create runScript and link shell + # if [ ! -e bin/sh ]; then + # ln -s ${runtimeShell} bin/sh + # fi + # mkdir -p .${projectName}.d + # ln -s ${runScriptFile} .${projectName}.d/runscript + + # # Fill out .${projectName}.d + # mkdir -p .${projectName}.d/env + # touch .${projectName}.d/env/94-appsbase.sh + + # cd .. + # mkdir -p /var/lib/${projectName}/mnt/{container,final,overlay,session,source} + # echo "root:x:0:0:System administrator:/root:/bin/sh" > /etc/passwd + # echo > /etc/resolv.conf + # TMPDIR=$(pwd -P) ${projectName} build $out ./img + # ''); + + # in + # result; + + inherit (import ./definition-lib.nix { inherit lib; }) + knownPrimarySectionNamesDefault + knownAppSectionNamesDefault + toSingularityDef + ; + + contentsToDef = + { contents ? [ ] + , definitionOverrider ? null + }: + let + layerClosure = writeMultipleReferencesToFile (contents ++ [ bash coreutils ]); + in + ( + if lib.isFunction definitionOverrider then + definitionOverrider + else if builtins.isAttrs definitionOverrider then + (d: lib.recursiveUpdate d definitionOverrider) + else + lib.id + ) { + header.Bootstrap = "scratch"; + setup = '' + mkdir -p ''${SINGULARITY_ROOTFS}/${storeDir} + for f in $(cat ${layerClosure}) ; do + cp -r "$f" "''${SINGULARITY_ROOTFS}/${storeDir}" + done + mkdir -p "''${SINGULARITY_ROOTFS}/bin" + "${coreutils}/bin/ln" -s "${runtimeShell}" "''${SINGULARITY_ROOTFS}/bin/sh" + mkdir -p "''${SINGULARITY_ROOTFS}/usr/bin" + "${coreutils}/bin/ln" -s "${coreutils}/bin/env" "''${SINGULARITY_ROOTFS}/usr/bin/env" + ''; + environment = { + PATH = "${lib.makeBinPath contents}:\${PATH:-}"; + }; + labels = { + inherit layerClosure; + }; + }; + + buildImageFromDef = + args@{ name + , definition ? contentsToDef { inherit contents definitionOverrider; } + , contents ? [ ] + , definitionOverrider ? null + , executableFlags ? [ ] + , buildImageFlags ? [ ] + , singularity ? defaultSingularity + , toSingularityDefArgs ? { } + , ... + }: + let + # May be "apptainer" instead of "singularity" + projectName = singularity.projectName or "singularity"; + definitionFile = writeText "${name}.def" (toSingularityDef toSingularityDefArgs definition); + # Pass for users who want to build from the command line instead of inside a VM. + buildscriptPackage = writeShellScriptBin "build-image" '' + if [ "$#" -lt 1 ]; then + echo "Expect IMAGE_PATH" >&2 + exit 1 + fi + pathSIF="$1" + shift + ${lib.toUpper projectName}ENV_PATH="$PATH" "${singularity}/bin/${projectName}" ${toString executableFlags} build ${toString buildImageFlags} "$@" "$pathSIF" "${definitionFile}" + ''; in - result; + (runCommand "${projectName}-image-${name}.sif" + (removeAttrs args [ "name" "contents" "definition" "definitionOverrider" ] // { + inherit executableFlags buildImageFlags; + passthru = args.passthru or { } // { + inherit singularity definition definitionFile buildscriptPackage; + layerClosure = definition.labels.layerClosure or null; + }; + }) '' + "${buildscriptPackage}/bin/${buildscriptPackage.meta.mainProgram}" "$out" $buildImageFlags "''${buildImageFlagsArray[@]}" + ''); } diff --git a/pkgs/build-support/singularity-tools/definition-lib.nix b/pkgs/build-support/singularity-tools/definition-lib.nix new file mode 100644 index 0000000000000..c794a1cd8ccec --- /dev/null +++ b/pkgs/build-support/singularity-tools/definition-lib.nix @@ -0,0 +1,131 @@ +{ lib }: + +let + # Increase the readability of the output file + # by arranging each section into the specified order + knownPrimarySectionNamesDefault = [ "pre" "setup" "files" "post" "environment" "runscript" "startscript" "test" "labels" "help" ]; + knownAppSectionNamesDefault = [ "appfiles" "appinstall" "appenv" "apprun" "applabels" "apphelp" ]; + + # TODO: Implement a type for the Apptainer/Singularity definition + + ### Structurize the definition file + /**! + * Here is the structured representation of the + * demo definition file in the Apptainer/Singularity menu: + * ~~~{.nix} + * { + * # header.Bootstrap is REQUIRED + * header = { + * Bootstrap = "docker"; + * From = "ubuntu"; + * }; + * setup = '' + * touch /file1 + * touch ''${SINGULARITY_ROOTFS}/file2 + * ''; + * # A string instead of a list is also acceptable + * files = [ + * # Element as a list will be concatenated with whitespace as the delimiter + * [ "/file1" ] + * [ "/file1" "/opt" ] + * # If the element is a string instead of a list, it would be composed directly + * "some ad-hoc stuff" + * ]; + * # A string or a list instead of a set is also acceptable + * # The value part is NOT escaped as shell strings. + * # Use `lib.escapeShellArg` to escape special contents + * environment = { + * __sectionPreScript = "some ad-hoc stuff beforehand"; + * LISTEN_PORT = "12345" + * LC_ALL = "C"; + * # This will evaluates to + * # `export PATH=/nix/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-hello-0.0.0/bin/hello:/nix/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-cowsay-0.0.0/bin/cowsay:${PATH:-}` + * # when writing as a definition string + * PATH = "${lib.makeBinPath [ hello cowsay ]}:\${PATH:-}" + * __sectionPostScript = "some ad-hoc stuff afterward"; + * }; + * post = '' + * # ... + * ''; + * # ... + * # A string instead of a set is also acceptable + * labels = { + * Author = "d@sylabs.io"; + * Version = "v0.0.1"; + * }; + * # ... + * # Sections for apps goes here + * apps.foo = { + * apprun = '' + * exec echo "RUNNING FOO" + * ''; + * applabels = { + * BOOTSTRAP = "FOO"; + * }; + * # ... + * }; + * } + * ~~~ + * The original version can be found here: + * https://apptainer.org/user-docs/3.1/definition_files.html + **/ + toSingularityDef = + { dropUnknownSections ? false + , knownPrimarySectionNames ? knownPrimarySectionNamesDefault + , knownAppSectionNames ? knownAppSectionNamesDefault + , ... + }: + let + orderAs = template: listToOrder: (lib.intersectLists listToOrder template) ++ lib.optionals dropUnknownSections (lib.subtractLists template listToOrder); + sectionMappingFunction = sectionName: sectionContent: + map (s: " ${s}") ( + if (builtins.typeOf sectionContent == "string") then + (ss: if (ss != [ ]) && (lib.last ss == "\n") then lib.take ((lib.count ss) - 1) ss else ss) (lib.splitString "\n" sectionContent) + else if (builtins.typeOf sectionContent == "list") then + map (lineContent: if (builtins.typeOf lineContent == "list") then builtins.concatStringsSep " " lineContent else lineContent) sectionContent + else if (builtins.typeOf sectionContent == "set") then + if sectionName == "environment" || sectionName == "appenv" then + (sectionContent.__sectionPreScript or [ ]) + ++ map (name: "export ${name}=${sectionContent.${name}}") (builtins.attrNames sectionContent) + ++ (sectionContent.__sectionPostScript or [ ]) + else + map (name: "${name} ${sectionContent.${name}}") (builtins.attrNames sectionContent) + else [ ] + ); + in + definition: + builtins.concatStringsSep "\n" ( + [ + (builtins.concatStringsSep "\n" ( + [ "Bootstrap: ${definition.header.Bootstrap}" ] + ++ map + (headerName: "${headerName}: ${definition.header.${headerName}}") + (lib.subtractLists [ "Bootstrap" ] (builtins.attrNames definition.header)) + )) + ] + ++ map + (sectionName: builtins.concatStringsSep "\n" ( + [ "%${sectionName}" ] + ++ sectionMappingFunction sectionName definition.${sectionName} + )) + (orderAs knownPrimarySectionNames (lib.subtractLists [ "header" "apps" ] (builtins.attrNames definition))) + ++ lib.optionals (builtins.hasAttr "apps" definition) (builtins.concatLists (map + (appName: + map + (sectionName: builtins.concatStringsSep "\n" ( + [ "%${sectionName} ${appName}" ] + ++ sectionMappingFunction sectionName definition.apps.${appName}.${sectionName} + )) + (orderAs knownAppSectionNames (builtins.attrNames definition.apps.${appName})) + ) + (builtins.attrNames definition.apps) + )) + ); +in +{ + inherit + knownPrimarySectionNamesDefault + knownAppSectionNamesDefault + toSingularityDef + ; +} diff --git a/pkgs/build-support/trivial-builders.nix b/pkgs/build-support/trivial-builders.nix index e4c2ffb703494..a0aa42a76ceaf 100644 --- a/pkgs/build-support/trivial-builders.nix +++ b/pkgs/build-support/trivial-builders.nix @@ -229,7 +229,7 @@ rec { */ - writeScriptBin = name: text: writeTextFile {inherit name text; executable = true; destination = "/bin/${name}";}; + writeScriptBin = name: text: writeTextFile {inherit name text; executable = true; destination = "/bin/${name}"; meta.mainProgram = name;}; /* Similar to writeScript. Writes a Shell script and checks its syntax. @@ -287,6 +287,7 @@ rec { checkPhase = '' ${stdenv.shellDryRun} "$target" ''; + meta.mainProgram = name; }; /* @@ -761,6 +762,36 @@ rec { writeDirectReferencesToFile (writeText "string-file" string); + /** + * Write the references (i.e. the runtime dependencies in the Nix store) of the elements in `paths' to a file. + * The elements are sorted according to their hashes, and duplicated lines are removed. + */ + writeMultipleReferencesToFile = paths: runCommand "runtime-deps-multiple" { + # Uniquely sort the input paths to reduce unnessesary rebuilds + referencesFiles = map writeReferencesToFile (lib.pipe (map toString paths) [ (builtins.sort builtins.lessThan) lib.unique ]); + } '' + touch "$out" + declare -A referencesDict=() + for refFile in $referencesFiles; do + while read ref; do + if [[ "''${ref:0:${toString (lib.stringLength builtins.storeDir + 1)}}" == "${builtins.storeDir}/" ]]; then + refHash="''${ref:${toString (lib.stringLength builtins.storeDir + 1)}:32}" + refTail="$(stripHash "$ref")" + # Set if not set before + : "''${referencesDict["$refHash"]="$refTail"}" + fi + done < "$refFile" + done + if (( "''${#referencesDict[@]}" )); then + declare -a hashesSorted + IFS=$'\n' hashesSorted=( $(sort <<<"''${!referencesDict[*]}") ) + for refHash in ''${hashesSorted[@]}; do + echo "${builtins.storeDir}/$refHash-''${referencesDict[$refHash]}" >> "$out" + done + fi + ''; + + /* Print an error message if the file with the specified name and hash doesn't exist in the Nix store. This function should only be used by non-redistributable software with an unfree license