Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 19 additions & 6 deletions codex-cli/scripts/build_npm_package.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,12 +69,24 @@

PACKAGE_NATIVE_COMPONENTS: dict[str, list[str]] = {
"codex": [],
"codex-linux-x64": ["codex", "rg"],
"codex-linux-arm64": ["codex", "rg"],
"codex-darwin-x64": ["codex", "rg"],
"codex-darwin-arm64": ["codex", "rg"],
"codex-win32-x64": ["codex", "rg", "codex-windows-sandbox-setup", "codex-command-runner"],
"codex-win32-arm64": ["codex", "rg", "codex-windows-sandbox-setup", "codex-command-runner"],
"codex-linux-x64": ["codex", "rg", "node"],
"codex-linux-arm64": ["codex", "rg", "node"],
"codex-darwin-x64": ["codex", "rg", "node"],
"codex-darwin-arm64": ["codex", "rg", "node"],
"codex-win32-x64": [
"codex",
"rg",
"node",
"codex-windows-sandbox-setup",
"codex-command-runner",
],
"codex-win32-arm64": [
"codex",
"rg",
"node",
"codex-windows-sandbox-setup",
"codex-command-runner",
],
"codex-responses-api-proxy": ["codex-responses-api-proxy"],
"codex-sdk": [],
}
Expand All @@ -92,6 +104,7 @@
"codex-windows-sandbox-setup": "codex",
"codex-command-runner": "codex",
"rg": "path",
"node": "node",
}


Expand Down
96 changes: 94 additions & 2 deletions codex-cli/scripts/install_native_deps.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
DEFAULT_WORKFLOW_URL = "https://github.com/openai/codex/actions/runs/17952349351" # rust-v0.40.0
VENDOR_DIR_NAME = "vendor"
RG_MANIFEST = CODEX_CLI_ROOT / "bin" / "rg"
NODE_VERSION_FILE = CODEX_CLI_ROOT.parent / "codex-rs" / "node-version.txt"
BINARY_TARGETS = (
"x86_64-unknown-linux-musl",
"aarch64-unknown-linux-musl",
Expand Down Expand Up @@ -79,6 +80,41 @@ class BinaryComponent:
RG_TARGET_TO_PLATFORM = {target: platform for target, platform in RG_TARGET_PLATFORM_PAIRS}
DEFAULT_RG_TARGETS = [target for target, _ in RG_TARGET_PLATFORM_PAIRS]

# Node distribution asset mapping: target -> (archive_format, filename, member_path)
NODE_ASSETS: dict[str, tuple[str, str, str]] = {
"x86_64-apple-darwin": (
"tar.gz",
"node-v{ver}-darwin-x64.tar.gz",
"node-v{ver}-darwin-x64/bin/node",
),
"aarch64-apple-darwin": (
"tar.gz",
"node-v{ver}-darwin-arm64.tar.gz",
"node-v{ver}-darwin-arm64/bin/node",
),
"x86_64-unknown-linux-musl": (
"tar.xz",
"node-v{ver}-linux-x64.tar.xz",
"node-v{ver}-linux-x64/bin/node",
),
"aarch64-unknown-linux-musl": (
"tar.xz",
"node-v{ver}-linux-arm64.tar.xz",
"node-v{ver}-linux-arm64/bin/node",
),
"x86_64-pc-windows-msvc": (
"zip",
"node-v{ver}-win-x64.zip",
"node-v{ver}-win-x64/node.exe",
),
"aarch64-pc-windows-msvc": (
"zip",
"node-v{ver}-win-arm64.zip",
"node-v{ver}-win-arm64/node.exe",
),
}
DEFAULT_NODE_TARGETS = tuple(NODE_ASSETS.keys())

# urllib.request.urlopen() defaults to no timeout (can hang indefinitely), which is painful in CI.
DOWNLOAD_TIMEOUT_SECS = 60

Expand Down Expand Up @@ -132,11 +168,11 @@ def parse_args() -> argparse.Namespace:
"--component",
dest="components",
action="append",
choices=tuple(list(BINARY_COMPONENTS) + ["rg"]),
choices=tuple(list(BINARY_COMPONENTS) + ["rg", "node"]),
help=(
"Limit installation to the specified components."
" May be repeated. Defaults to codex, codex-windows-sandbox-setup,"
" codex-command-runner, and rg."
" codex-command-runner, node, and rg."
),
)
parser.add_argument(
Expand All @@ -163,6 +199,7 @@ def main() -> int:
"codex-windows-sandbox-setup",
"codex-command-runner",
"rg",
"node",
]

workflow_url = (args.workflow_url or DEFAULT_WORKFLOW_URL).strip()
Expand All @@ -187,6 +224,11 @@ def main() -> int:
print("Fetching ripgrep binaries...")
fetch_rg(vendor_dir, DEFAULT_RG_TARGETS, manifest_path=RG_MANIFEST)

if "node" in components:
with _gha_group("Fetch Node runtime"):
print("Fetching Node runtime...")
fetch_node(vendor_dir, load_node_version(), DEFAULT_NODE_TARGETS)

print(f"Installed native dependencies into {vendor_dir}")
return 0

Expand Down Expand Up @@ -259,6 +301,41 @@ def fetch_rg(
return [results[target] for target in targets]


def fetch_node(vendor_dir: Path, version: str, targets: Sequence[str]) -> None:
version = version.strip().lstrip("v")
vendor_dir.mkdir(parents=True, exist_ok=True)

for target in targets:
asset = NODE_ASSETS.get(target)
if asset is None:
raise RuntimeError(f"unsupported Node target {target}")
archive_format, filename_tmpl, member_tmpl = asset
filename = filename_tmpl.format(ver=version)
member = member_tmpl.format(ver=version)
url = f"https://nodejs.org/dist/v{version}/{filename}"
dest_dir = vendor_dir / target / "node"
dest_dir.mkdir(parents=True, exist_ok=True)
binary_name = "node.exe" if "windows" in target else "node"
dest = dest_dir / binary_name

with tempfile.TemporaryDirectory() as tmp_dir_str:
tmp_dir = Path(tmp_dir_str)
download_path = tmp_dir / filename
print(f" downloading node {version} for {target} from {url}", flush=True)
_download_file(url, download_path)
dest.unlink(missing_ok=True)
extract_archive(download_path, archive_format, member, dest)
if "windows" not in target:
dest.chmod(0o755)


def load_node_version() -> str:
try:
return NODE_VERSION_FILE.read_text(encoding="utf-8").strip()
except OSError as exc:
raise RuntimeError(f"failed to read node version from {NODE_VERSION_FILE}: {exc}") from exc


def _download_artifacts(workflow_id: str, dest_dir: Path) -> None:
cmd = [
"gh",
Expand Down Expand Up @@ -437,6 +514,21 @@ def extract_archive(
shutil.move(str(extracted), dest)
return

if archive_format == "tar.xz":
if not archive_member:
raise RuntimeError("Missing 'path' for tar.xz archive in archive.")
with tarfile.open(archive_path, "r:xz") as tar:
try:
member = tar.getmember(archive_member)
except KeyError as exc:
raise RuntimeError(
f"Entry '{archive_member}' not found in archive {archive_path}."
) from exc
tar.extract(member, path=archive_path.parent, filter="data")
extracted = archive_path.parent / archive_member
shutil.move(str(extracted), dest)
return

if archive_format == "zip":
if not archive_member:
raise RuntimeError("Missing 'path' for zip archive in DotSlash manifest.")
Expand Down
6 changes: 6 additions & 0 deletions codex-rs/core/config.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,9 @@
"js_repl": {
"type": "boolean"
},
"js_repl_polling": {
"type": "boolean"
},
"js_repl_tools_only": {
"type": "boolean"
},
Expand Down Expand Up @@ -1401,6 +1404,9 @@
"js_repl": {
"type": "boolean"
},
"js_repl_polling": {
"type": "boolean"
},
"js_repl_tools_only": {
"type": "boolean"
},
Expand Down
3 changes: 1 addition & 2 deletions codex-rs/core/src/codex.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4864,8 +4864,7 @@ async fn run_sampling_request(

let model_supports_parallel = turn_context.model_info.supports_parallel_tool_calls;

let tools =
crate::tools::spec::filter_tools_for_model(router.specs(), &turn_context.tools_config);
let tools = router.specs();
let base_instructions = sess.get_base_instructions().await;

let prompt = Prompt {
Expand Down
12 changes: 12 additions & 0 deletions codex-rs/core/src/features.rs
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@ pub enum Feature {
// Experimental
/// Enable JavaScript REPL tools backed by a persistent Node kernel.
JsRepl,
/// Enable js_repl polling helpers and tool.
JsReplPolling,
/// Only expose js_repl tools directly to the model.
JsReplToolsOnly,
/// Use the single unified PTY-backed exec tool.
Expand Down Expand Up @@ -336,6 +338,10 @@ impl Features {
tracing::warn!("js_repl_tools_only requires js_repl; disabling js_repl_tools_only");
features.disable(Feature::JsReplToolsOnly);
}
if features.enabled(Feature::JsReplPolling) && !features.enabled(Feature::JsRepl) {
tracing::warn!("js_repl_polling requires js_repl; disabling js_repl_polling");
features.disable(Feature::JsReplPolling);
}

features
}
Expand Down Expand Up @@ -451,6 +457,12 @@ pub const FEATURES: &[FeatureSpec] = &[
stage: Stage::UnderDevelopment,
default_enabled: false,
},
FeatureSpec {
id: Feature::JsReplPolling,
key: "js_repl_polling",
stage: Stage::UnderDevelopment,
default_enabled: false,
},
FeatureSpec {
id: Feature::JsReplToolsOnly,
key: "js_repl_tools_only",
Expand Down
24 changes: 24 additions & 0 deletions codex-rs/core/src/project_doc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,15 @@ fn render_js_repl_instructions(config: &Config) -> Option<String> {
section.push_str("- Top-level bindings persist across cells. If you hit `SyntaxError: Identifier 'x' has already been declared`, reuse the binding, pick a new name, wrap in `{ ... }` for block scope, or reset the kernel with `js_repl_reset`.\n");
section.push_str("- Top-level static import declarations (for example `import x from \"pkg\"`) are currently unsupported in `js_repl`; use dynamic imports with `await import(\"pkg\")` instead.\n");

if config.features.enabled(Feature::JsReplPolling) {
section.push_str("- Polling mode is two-step: (1) call `js_repl` with first-line pragma `// codex-js-repl: poll=true` to get an `exec_id`; (2) call `js_repl_poll` with that `exec_id` until `status` is `completed` or `error`.\n");
section.push_str(
"- Use `js_repl_cancel` with an `exec_id` to cancel a long-running polled execution.\n",
);
section.push_str("- `js_repl_poll`/`js_repl_cancel` must not be called before a successful `js_repl` submission returns an `exec_id`.\n");
section.push_str("- If `js_repl` rejects your payload format, resend raw JS with the pragma; do not retry with JSON, quoted strings, or markdown fences.\n");
}

if config.features.enabled(Feature::JsReplToolsOnly) {
section.push_str("- Do not call tools directly; use `js_repl` + `codex.tool(...)` for all tool calls, including shell commands.\n");
section
Expand Down Expand Up @@ -434,6 +443,21 @@ mod tests {
assert_eq!(res, expected);
}

#[tokio::test]
async fn js_repl_polling_instructions_are_feature_gated() {
let tmp = tempfile::tempdir().expect("tempdir");
let mut cfg = make_config(&tmp, 4096, None).await;
cfg.features
.enable(Feature::JsRepl)
.enable(Feature::JsReplPolling);

let res = get_user_instructions(&cfg, None)
.await
.expect("js_repl instructions expected");
let expected = "## JavaScript REPL (Node)\n- Use `js_repl` for Node-backed JavaScript with top-level await in a persistent kernel. `codex.state` persists for the session (best effort) and is cleared by `js_repl_reset`.\n- `js_repl` is a freeform/custom tool. Direct `js_repl` calls must send raw JavaScript tool input (optionally with first-line `// codex-js-repl: timeout_ms=15000`). Do not wrap code in JSON (for example `{\"code\":\"...\"}`), quotes, or markdown code fences.\n- Helpers: `codex.state`, `codex.tmpDir`, and `codex.tool(name, args?)`.\n- `codex.tool` executes a normal tool call and resolves to the raw tool output object. Use it for shell and non-shell tools alike.\n- To share generated images with the model, write a file under `codex.tmpDir`, call `await codex.tool(\"view_image\", { path: \"/absolute/path\" })`, then delete the file.\n- Top-level bindings persist across cells. If you hit `SyntaxError: Identifier 'x' has already been declared`, reuse the binding, pick a new name, wrap in `{ ... }` for block scope, or reset the kernel with `js_repl_reset`.\n- Top-level static import declarations (for example `import x from \"pkg\"`) are currently unsupported in `js_repl`; use dynamic imports with `await import(\"pkg\")` instead.\n- Polling mode is two-step: (1) call `js_repl` with first-line pragma `// codex-js-repl: poll=true` to get an `exec_id`; (2) call `js_repl_poll` with that `exec_id` until `status` is `completed` or `error`.\n- Use `js_repl_cancel` with an `exec_id` to cancel a long-running polled execution.\n- `js_repl_poll`/`js_repl_cancel` must not be called before a successful `js_repl` submission returns an `exec_id`.\n- If `js_repl` rejects your payload format, resend raw JS with the pragma; do not retry with JSON, quoted strings, or markdown fences.\n- Avoid direct access to `process.stdout` / `process.stderr` / `process.stdin`; it can corrupt the JSON line protocol. Use `console.log` and `codex.tool(...)`.";
assert_eq!(res, expected);
}

/// When both system instructions *and* a project doc are present the two
/// should be concatenated with the separator.
#[tokio::test]
Expand Down
Loading
Loading