Just: introduce common "verbs"

java/justfile

@@ -0,0 +1,3 @@
+import '../lib.just'
+
+build: (_build "java")

java/ql/justfile

@@ -0,0 +1,6 @@
+import "../../lib.just"
+
+[no-cd]
+format *ARGS=".": (_ql_format ARGS)
+
+consistency_queries := source_dir() / "consistency-queries"

java/ql/test/justfile

@@ -0,0 +1,15 @@
+import "../justfile"
+
+base_flags := """\
+     CODEQL_EXTRACTOR_KOTLIN_DIAGNOSTIC_LIMIT="\\ " \
+"""
+
+all_checks := default_db_checks + """\
+    --check-undefined-labels \
+    --check-repeated-labels \
+    --check-redefined-labels \
+    --check-use-before-definition \
+    --consistency-queries=""" + consistency_queries
+
+[no-cd]
+test *ARGS=".": (_codeql_test "java" base_flags all_checks ARGS)

justfile

@@ -0,0 +1,2 @@
+import 'misc/just/lib.just'
+import 'misc/just/forward.just'

lib.just

@@ -0,0 +1 @@
+import "misc/just/lib.just"

misc/bazel/justfile

@@ -0,0 +1,5 @@
+import '../just/lib.just'
+
+[no-cd, positional-arguments, no-exit-message]
+hello +ARGS:
+    @echo "hello from bzl" "$@"

misc/codegen/justfile

@@ -0,0 +1,5 @@
+import "../just/lib.just"
+
+test *ARGS="": (_bazel "test" "@codeql//misc/codegen/...")
+
+format *ARGS=".": (_black ARGS)

misc/just/codeql-test-run.ts

@@ -0,0 +1,138 @@
+import * as child_process from "child_process";
+import * as path from "path";
+import * as os from "os";
+import * as fs from "fs";
+
+function invoke(
+    invocation: string[],
+    options: { cwd?: string; log_prefix?: string } = {},
+): number {
+    const log_prefix =
+        options.log_prefix && options.log_prefix !== ""
+            ? `${options.log_prefix} `
+            : "";
+    console.log(
+        `${process.env["CMD_BEGIN"] || ""}${log_prefix}${invocation.join(" ")}${process.env["CMD_END"] || ""}`,
+    );
+    try {
+        child_process.execFileSync(invocation[0], invocation.slice(1), {
+            stdio: "inherit",
+            cwd: options.cwd,
+        });
+    } catch (error) {
+        return 1;
+    }
+    return 0;
+}
+
+type Args = {
+    tests: string[];
+    flags: string[];
+    env: string[];
+    codeql: string;
+    all: boolean;
+};
+
+function parseArgs(args: Args, argv: string) {
+    argv.split(/(?<!\\) /)
+        .map((arg) => arg.replace("\\ ", " "))
+        .forEach((arg) => {
+            if (arg.startsWith("--codeql=")) {
+                args.codeql = arg.split("=")[1];
+            } else if (arg === "+" || arg === "--all-checks") {
+                args.all = true;
+            } else if (arg.startsWith("-")) {
+                args.flags.push(arg);
+            } else if (/^[A-Z_][A-Z_0-9]*=.*$/.test(arg)) {
+                args.env.push(arg);
+            } else if (arg !== "") {
+                args.tests.push(arg);
+            }
+        });
+}
+
+function codeqlTestRun(argv: string[]): number {
+    const semmle_code = process.env["SEMMLE_CODE"];
+    const [language, base_args, all_args, extra_args] = argv;
+    const ram_per_thread = process.platform === "linux" ? 3000 : 2048;
+    const cpus = os.cpus().length;
+    let args: Args = {
+        tests: [],
+        flags: [`--ram=${ram_per_thread * cpus}`, `-j${cpus}`],
+        env: [],
+        codeql: semmle_code ? "build" : "host",
+        all: false,
+    };
+    parseArgs(args, base_args);
+    parseArgs(args, extra_args);
+    if (args.all) {
+        parseArgs(args, all_args);
+    }
+    if (!semmle_code && (args.codeql === "build" || args.codeql === "built")) {
+        console.error(
+            "Using `--codeql=build` or `--codeql=built` requires working with the internal repository",
+        );
+        return 1;
+    }
+    if (args.tests.length === 0) {
+        args.tests.push(".");
+    }
+    if (args.codeql === "build") {
+        if (
+            invoke(["python3", "build", `target/intree/codeql-${language}`], {
+                cwd: semmle_code,
+            }) !== 0
+        ) {
+            return 1;
+        }
+    }
+    if (args.codeql !== "host") {
+        // disable the default implicit config file, but keep an explicit one
+        // this is the same behavior wrt to `--codeql` as the integration test runner
+        process.env["CODEQL_CONFIG_FILE"] ||= ".";
+    }
+    // Set and unset environment variables
+    args.env.forEach((envVar) => {
+        const [key, value] = envVar.split("=", 2);
+        if (key) {
+            if (value === undefined) {
+                delete process.env[key];
+            } else {
+                process.env[key] = value;
+            }
+        } else {
+            console.error(`Invalid environment variable assignment: ${envVar}`);
+            process.exit(1);
+        }
+    });
+    let codeql;
+    if (args.codeql === "built" || args.codeql === "build") {
+        codeql = path.join(
+            semmle_code!,
+            "target",
+            "intree",
+            `codeql-${language}`,
+            "codeql",
+        );
+    } else if (args.codeql === "host") {
+        codeql = "codeql";
+    } else {
+        codeql = args.codeql;
+        if (fs.lstatSync(codeql).isDirectory()) {
+            codeql = path.join(codeql, "codeql");
+            if (process.platform === "win32") {
+                codeql += ".exe";
+            }
+        }
+        if (!fs.existsSync(codeql)) {
+            console.error(`CodeQL executable not found: ${codeql}`);
+            return 1;
+        }
+    }
+
+    return invoke([codeql, "test", "run", ...args.flags, "--", ...args.tests], {
+        log_prefix: args.env.join(" "),
+    });
+}
+
+process.exit(codeqlTestRun(process.argv.slice(2)));

misc/just/forward-command.ts

@@ -0,0 +1,92 @@
+import * as child_process from "child_process";
+import * as path from "path";
+import * as fs from "fs";
+
+function commonDir(paths: string[]): string {
+    if (paths.length === 0) return "";
+    const splitPaths = paths.map((p) => p.split(path.sep));
+    let i;
+    for (i = 0; i < splitPaths[0].length; i++) {
+        if (!splitPaths.every((parts) => parts[i] === splitPaths[0][i])) {
+            break;
+        }
+    }
+    const commonParts = splitPaths[0].slice(0, i);
+    let ret = commonParts.join(path.sep);
+    if (!fs.existsSync(ret)) {
+        throw new Error(`Common directory does not exist: ${ret}`);
+    }
+    if (!fs.lstatSync(ret).isDirectory()) {
+        ret = path.dirname(ret);
+    }
+    return ret;
+}
+
+function forwardCommand(args: string[]): number {
+    // Avoid infinite recursion
+    if (args.length == 0) {
+        console.error("No command provided");
+        return 1;
+    }
+    const cmd = args[0];
+    const envVariable = `__JUST_FORWARD_${cmd}`;
+    if (process.env[envVariable]) {
+        console.error(`No common ${cmd} handler found`);
+        return 1;
+    }
+    process.env[envVariable] = "true";
+    const cmdArgs = args.slice(1);
+    // non-positional arguments are flags, + (used by language tests) or environment variable settings
+    const is_non_positional = /^(-.*|\+|[A-Z_][A-Z_0-9]*=.*)$/;
+    const flags = cmdArgs.filter((arg) => is_non_positional.test(arg));
+    const positionalArgs = cmdArgs.filter(
+        (arg) => !is_non_positional.test(arg),
+    );
+
+    if (positionalArgs.length === 0) {
+        console.error("No positional arguments provided");
+        return 1;
+    }
+
+    const commonPath = commonDir(positionalArgs);
+    let relativeArgs = positionalArgs.map(
+        (arg) => path.relative(commonPath, arg) || ".",
+    );
+    if (relativeArgs.length === 1 && relativeArgs[0] === ".") {
+        relativeArgs = [];
+    }
+    let relativeFlags = flags.map((arg) => {
+        // this might break in specific corner cases, but is good enough for most uses
+        // workaround if this doesn't work is to not use the forwarder (call just directly in the relevant directory)
+        if (arg.includes("=") && arg.includes(path.sep)) {
+            let [flags, flag_arg] = arg.split("=", 2);
+            flag_arg = flag_arg
+                .split(path.delimiter)
+                .map((p) =>
+                    path.isAbsolute(p) ? p : path.relative(commonPath, p),
+                )
+                .join(path.delimiter);
+            return `${flags}=${flag_arg}`;
+        }
+        return arg;
+    });
+
+    const invocation = [
+        process.env["JUST_EXECUTABLE"] || "just",
+        cmd,
+        ...relativeFlags,
+        ...relativeArgs,
+    ];
+    console.log(`-> ${commonPath}: just ${invocation.slice(1).join(" ")}`);
+    try {
+        child_process.execFileSync(invocation[0], invocation.slice(1), {
+            stdio: "inherit",
+            cwd: commonPath,
+        });
+    } catch (error) {
+        return 1;
+    }
+    return 0;
+}
+
+process.exit(forwardCommand(process.argv.slice(2)));

misc/just/forward.just

@@ -0,0 +1,28 @@
+import "lib.just"
+
+# copy&paste necessary for each command until proper forwarding of multiple args is implemented
+# see https://github.com/casey/just/issues/1988
+
+_forward := tsx + ' "' + source_dir() + '/forward-command.ts"'
+
+
+[no-cd, positional-arguments, no-exit-message]
+@test *ARGS=".":
+    {{ _forward }} test "$@"
+
+
+[no-cd, positional-arguments, no-exit-message]
+@build *ARGS=".":
+    {{ _forward }} build "$@"
+
+[no-cd, positional-arguments, no-exit-message]
+@generate *ARGS=".":
+    {{ _forward }} generate "$@"
+
+[no-cd, positional-arguments, no-exit-message]
+@lint *ARGS=".":
+    {{ _forward }} lint "$@"
+
+[no-cd, positional-arguments, no-exit-message]
+@format *ARGS=".":
+    {{ _forward }} format "$@"

misc/just/justfile

@@ -0,0 +1,2 @@
+format *ARGS=".":
+    npx prettier --write {{ ARGS }}

misc/just/lib.just

@@ -0,0 +1,74 @@
+set fallback
+set allow-duplicate-recipes
+set allow-duplicate-variables
+set unstable
+
+export PATH_SEP := if os() == "windows" { ";" } else { ":" }
+export JUST_EXECUTABLE := just_executable()
+
+error := style("error") + "error" + NORMAL + ": "
+cmd_sep := "\n#--------------------------------------------------------\n"
+export CMD_BEGIN := style("command") + cmd_sep
+export CMD_END := cmd_sep + NORMAL
+
+tsx := "npx tsx@4.19.0"
+
+import? '../../../semmle-code.just'  # internal repo just file, if present
+import 'semmle-code-stub.just'
+
+
+[no-exit-message]
+@_require_semmle_code:
+    {{ if SEMMLE_CODE == "" { '''
+        echo "''' + error + ''' running this recipe requires doing so from an internal repository checkout" >&2
+        exit 1
+    ''' } else { "" } }}
+
+_build LANGUAGE: _require_semmle_code (_maybe_build LANGUAGE)
+
+[no-exit-message]
+_maybe_build LANGUAGE:
+    {{ cmd_sep }}{{ if SEMMLE_CODE == "" { '# using codeql from PATH, if any' } else { 'cd "$SEMMLE_CODE"; ./build target/intree/codeql-' + LANGUAGE } }}{{ cmd_sep }}
+
+
+default_db_checks := """\
+    --check-databases \
+    --check-diff-informed \
+    --fail-on-trap-errors \
+"""
+
+[no-cd, positional-arguments, no-exit-message]
+@_codeql_test LANGUAGE BASE_FLAGS ALL_CHECKS_FLAGS EXTRA_ARGS:
+    {{ tsx }} "{{ source_dir() }}/codeql-test-run.ts" "$@"
+
+
+[no-cd, no-exit-message]
+_ql_format +ARGS: (_maybe_build "nolang")
+    {{ cmd_sep }}{{ if SEMMLE_CODE != "" { '"$SEMMLE_CODE/target/intree/codeql-nolang/codeql"' } else { 'codeql' } }} query format --in-place $(find {{ ARGS }} -type f -name '*.ql' -or -name '*.qll'){{ cmd_sep }}
+
+
+[no-cd, no-exit-message]
+_bazel COMMAND *ARGS:
+    {{ cmd_sep }}{{ if SEMMLE_CODE != "" { '"$SEMMLE_CODE/tools/bazel"' } else { 'bazel' } }} {{ COMMAND }} {{ ARGS }}{{ cmd_sep }}
+
+[no-cd, no-exit-message]
+_sembuild *ARGS: (_run_in_semmle_code "./build" ARGS)
+
+[no-cd, no-exit-message]
+_integration_test *ARGS: _require_semmle_code (_run "$SEMMLE_CODE/tools/pytest" ARGS)
+
+[no-cd]
+_run +ARGS:
+    {{ cmd_sep }}{{ ARGS }}{{ cmd_sep }}
+
+[no-cd]
+_run_in_semmle_code +ARGS: _require_semmle_code
+    {{ cmd_sep }}cd "$SEMMLE_CODE"; {{ ARGS }}{{ cmd_sep }}
+
+
+[no-cd, positional-arguments, no-exit-message]
+_just +ARGS:
+    "{{ JUST_EXECUTABLE }}" "$@"
+
+[no-cd]
+_black *ARGS=".": (_run "uv" "run" "black" ARGS)

misc/just/semmle-code-stub.just

@@ -0,0 +1 @@
+export SEMMLE_CODE := ""

rust/justfile

@@ -0,0 +1,11 @@
+import '../lib.just'
+
+install: (_bazel "run" "@codeql//rust:install")
+
+build: generate (_build "rust")
+
+generate: (_bazel "run" "@codeql//rust/codegen")
+
+lint: (_run "python3" "lint.py")
+
+format: (_run "python3" "lint.py" "--format-only")