Skip to content

Commit

Permalink
Merge pull request #26974 from obsidiansystems/response-file-parsing-…
Browse files Browse the repository at this point in the history
…speed

cc-wrapper: improve response file parsing speed
  • Loading branch information
Ryan Trinkle authored Jul 5, 2017
2 parents c23dcd7 + 754c3f6 commit 7004641
Show file tree
Hide file tree
Showing 6 changed files with 134 additions and 53 deletions.
19 changes: 16 additions & 3 deletions pkgs/build-support/cc-wrapper/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
, zlib ? null, extraPackages ? [], extraBuildCommands ? ""
, dyld ? null # TODO: should this be a setup-hook on dyld?
, isGNU ? false, isClang ? cc.isClang or false, gnugrep ? null
, hostPlatform, targetPlatform
, buildPackages ? {}, hostPlatform, targetPlatform
, runCommand ? null
}:

Expand Down Expand Up @@ -120,6 +120,17 @@ let
null)
else "";

expand-response-params = if buildPackages.stdenv.cc or null != null && buildPackages.stdenv.cc != "/dev/null"
then buildPackages.stdenv.mkDerivation {
name = "expand-response-params";
src = ./expand-response-params.c;
buildCommand = ''
# Work around "stdenv-darwin-boot-2 is not allowed to refer to path /nix/store/...-expand-response-params.c"
cp "$src" expand-response-params.c
"$CC" -std=c99 -O3 -o "$out" expand-response-params.c
'';
} else "";

in

stdenv.mkDerivation {
Expand Down Expand Up @@ -368,11 +379,13 @@ stdenv.mkDerivation {
+ ''
substituteAll ${preWrap ./add-flags.sh} $out/nix-support/add-flags.sh
substituteAll ${preWrap ./add-hardening.sh} $out/nix-support/add-hardening.sh
cp -p ${preWrap ./utils.sh} $out/nix-support/utils.sh
substituteAll ${preWrap ./utils.sh} $out/nix-support/utils.sh
''
+ extraBuildCommands;

inherit dynamicLinker;
inherit dynamicLinker expand-response-params;

expandResponseParams = expand-response-params; # for substitution in utils.sh

crossAttrs = {
shell = shell.crossDrv + shell.crossDrv.shellPath;
Expand Down
84 changes: 84 additions & 0 deletions pkgs/build-support/cc-wrapper/expand-response-params.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
#include <assert.h>
#include <ctype.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

typedef struct { char *data; size_t len, cap; } String;

void resize(String *s, size_t len) {
s->len = len;
if (s->cap < s->len) {
s->cap = s->len * 2;
s->data = (char *)realloc(s->data, s->cap);
assert(s->data);
}
}

void append(String *s, const char *data, size_t len) {
resize(s, s->len + len);
memcpy(s->data + s->len - len, data, len);
}

typedef enum { space = 0, other = 1, backslash = 2, apostrophe = 3, quotation_mark = 4 } CharClass;
typedef enum { outside, unq, unq_esc, sq, sq_esc, dq, dq_esc } State;

// current State -> CharClass -> next State
const State transitions[][5] = {
[outside] = {outside, unq, unq_esc, sq, dq},
[unq] = {outside, unq, unq_esc, sq, dq},
[unq_esc] = {unq, unq, unq, unq, unq},
[sq] = {sq, sq, sq_esc, unq, sq},
[sq_esc] = {sq, sq, sq, sq, sq},
[dq] = {dq, dq, dq_esc, dq, unq},
[dq_esc] = {dq, dq, dq, dq, dq},
};

CharClass charClass(int c) {
return c == '\\' ? backslash : c == '\'' ? apostrophe : c == '"' ? quotation_mark :
isspace(c) ? space : other;
}

// expandArg writes NULL-terminated expansions of `arg', a NULL-terminated
// string, to stdout. If arg does not begin with `@' or does not refer to a
// file, it is written as is. Otherwise the contents of the file are
// recursively expanded. On unexpected EOF in malformed response files an
// incomplete final argument is written, even if it is empty, to parse like GCC.
void expandArg(String *arg) {
FILE *f;
if (arg->data[0] != '@' || !(f = fopen(&arg->data[1], "r"))) {
fwrite(arg->data, 1, arg->len, stdout);
return;
}

resize(arg, 0);
State cur = outside;
int c;
do {
c = fgetc(f);
State next = transitions[cur][charClass(c)];
if ((cur == unq && next == outside) || (cur != outside && c == EOF)) {
append(arg, "", 1);
expandArg(arg);
resize(arg, 0);
} else if (cur == unq_esc || cur == sq_esc || cur == dq_esc ||
(cur == outside ? next == unq : cur == next)) {
char s = c;
append(arg, &s, 1);
}
cur = next;
} while (c != EOF);

fclose(f);
}

int main(int argc, char **argv) {
String arg = { 0 };
while (*++argv) {
resize(&arg, 0);
append(&arg, *argv, strlen(*argv) + 1);
expandArg(&arg);
}
free(arg.data);
return EXIT_SUCCESS;
}
56 changes: 11 additions & 45 deletions pkgs/build-support/cc-wrapper/utils.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,52 +23,18 @@ badPath() {
"${p:0:${#NIX_BUILD_TOP}}" != "$NIX_BUILD_TOP"
}

# @args.rsp parser.
# Char classes: space, other, backslash, single quote, double quote.
# States: 0 - outside, 1/2 - unquoted arg/slash, 3/4 - 'arg'/slash, 5/6 - "arg"/slash.
# State transitions:
rspT=(01235 01235 11111 33413 33333 55651 55555)
# Push (a) arg or (c) char on transition:
rspP[10]=a rspP[01]=c rspP[11]=c rspP[21]=c rspP[33]=c rspP[43]=c rspP[55]=c rspP[65]=c

rspParse() {
rsp=()
local state=0
local arg=''
local c

while read -r -N1 c; do
local cls=1
case "$c" in
' ' | $'\t' | $'\r' | $'\n') cls=0 ;;
'\') cls=2 ;;
"'") cls=3 ;;
'"') cls=4 ;;
esac
local nextstates="${rspT[$state]}"
local nextstate="${nextstates:$cls:1}"
case "${rspP[$state$nextstate]}" in
'c') arg+="$c" ;;
'a') rsp+=("$arg"); arg='' ;;
esac
state="$nextstate"
done

if [ "$state" -ne 0 ]; then
rsp+=("$arg")
fi
}

expandResponseParams() {
params=()
while [ $# -gt 0 ]; do
local p="$1"
shift
if [ "${p:0:1}" = '@' -a -e "${p:1}" ]; then
rspParse <"${p:1}"
set -- "${rsp[@]}" "$@"
else
params+=("$p")
params=("$@")
local arg
for arg in "$@"; do
if [[ "$arg" == @* ]]; then
if [ -n "@expandResponseParams@" ]; then
readarray -d '' params < <("@expandResponseParams@" "$@")
return 0
else
echo "Response files aren't supported during bootstrapping" >&2
return 1
fi
fi
done
}
6 changes: 4 additions & 2 deletions pkgs/os-specific/darwin/ios-cross/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@
, stdenv
, coreutils
, gnugrep
, hostPlatform, targetPlatform
, buildPackages
, hostPlatform
, targetPlatform
}:

/* As of this writing, known-good prefix/arch/simulator triples:
Expand All @@ -29,7 +31,7 @@ let
sdk = "/Applications/Xcode.app/Contents/Developer/Platforms/iPhone${sdkType}.platform/Developer/SDKs/iPhone${sdkType}${sdkVer}.sdk";

in (import ../../../build-support/cc-wrapper {
inherit stdenv coreutils gnugrep runCommand;
inherit stdenv coreutils gnugrep runCommand buildPackages;
nativeTools = false;
nativeLibc = false;
inherit binutils;
Expand Down
16 changes: 13 additions & 3 deletions pkgs/stdenv/darwin/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -54,14 +54,17 @@ in rec {
__sandboxProfile = binShClosure + libSystemProfile;
};

stageFun = step: last: {shell ? "${bootstrapTools}/bin/sh",
stageFun = step: last: {shell ? "${bootstrapTools}/bin/bash",
overrides ? (self: super: {}),
extraPreHook ? "",
extraBuildInputs,
allowedRequisites ? null}:
let
thisStdenv = import ../generic {
inherit config shell extraBuildInputs allowedRequisites;
inherit config shell extraBuildInputs;
allowedRequisites = if allowedRequisites == null then null else allowedRequisites ++ [
thisStdenv.cc.expand-response-params
];

name = "stdenv-darwin-boot-${toString step}";

Expand All @@ -73,14 +76,17 @@ in rec {
nativeTools = true;
nativePrefix = bootstrapTools;
nativeLibc = false;
buildPackages = lib.optionalAttrs (last ? stdenv) {
inherit (last) stdenv;
};
hostPlatform = localSystem;
targetPlatform = localSystem;
libc = last.pkgs.darwin.Libsystem;
isClang = true;
cc = { name = "clang-9.9.9"; outPath = bootstrapTools; };
};

preHook = stage0.stdenv.lib.optionalString (shell == "${bootstrapTools}/bin/sh") ''
preHook = stage0.stdenv.lib.optionalString (shell == "${bootstrapTools}/bin/bash") ''
# Don't patch #!/interpreter because it leads to retained
# dependencies on the bootstrapTools in the final stdenv.
dontPatchShebangs=1
Expand Down Expand Up @@ -297,6 +303,9 @@ in rec {
inherit shell;
nativeTools = false;
nativeLibc = false;
buildPackages = {
inherit (prevStage) stdenv;
};
hostPlatform = localSystem;
targetPlatform = localSystem;
inherit (pkgs) coreutils binutils gnugrep;
Expand All @@ -319,6 +328,7 @@ in rec {
gzip ncurses.out ncurses.dev ncurses.man gnused bash gawk
gnugrep llvmPackages.clang-unwrapped patch pcre.out binutils-raw.out
binutils-raw.dev binutils gettext
cc.expand-response-params
]) ++ (with pkgs.darwin; [
dyld Libsystem CF cctools ICU libiconv locale
]);
Expand Down
6 changes: 6 additions & 0 deletions pkgs/stdenv/linux/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,9 @@ let
else lib.makeOverridable (import ../../build-support/cc-wrapper) {
nativeTools = false;
nativeLibc = false;
buildPackages = lib.optionalAttrs (prevStage ? stdenv) {
inherit (prevStage) stdenv;
};
hostPlatform = localSystem;
targetPlatform = localSystem;
cc = prevStage.gcc-unwrapped;
Expand Down Expand Up @@ -241,6 +244,9 @@ in
nativeTools = false;
nativeLibc = false;
isGNU = true;
buildPackages = {
inherit (prevStage) stdenv;
};
hostPlatform = localSystem;
targetPlatform = localSystem;
cc = prevStage.gcc-unwrapped;
Expand Down

0 comments on commit 7004641

Please sign in to comment.