From 93e7e092ad54f2d8dfa09ba158c1e2fded15ecbf Mon Sep 17 00:00:00 2001 From: OmarTawfik <15987992+OmarTawfik@users.noreply.github.com> Date: Mon, 13 May 2024 22:21:10 -0700 Subject: [PATCH] render runtime crates from templates dynamically --- Cargo.lock | 47 ++- Cargo.toml | 14 +- crates/codegen/language/tests/src/fail/mod.rs | 1 - crates/codegen/parser/generator/src/lib.rs | 11 - .../generator/src/typescript_generator.rs | 36 -- crates/codegen/parser/runtime/src/lib.rs | 27 -- .../src/napi_interface/ast_selectors.rs | 1 - .../templates/ast_selectors.rs.jinja2 | 332 ---------------- .../templates/ast_types.ts.jinja2 | 185 --------- .../runtime/src/query/user_defined_queries.rs | 3 - .../runtime/src/templates/language.rs.jinja2 | 271 ------------- .../templates/user_defined_queries.rs.jinja2 | 16 - .../runtime => runtime/cargo}/Cargo.toml | 11 +- crates/codegen/runtime/cargo/build.rs | 6 + crates/codegen/runtime/cargo/src/lib.rs | 6 + .../src => runtime/cargo/src/runtime}/cst.rs | 0 .../cargo/src/runtime}/cursor.rs | 0 .../cargo/src/runtime/generated}/kinds.rs | 75 ++-- .../cargo/src/runtime/generated/language.rs | 131 +++++++ .../cargo/src/runtime}/kinds.rs.jinja2 | 92 +++-- .../cargo/src/runtime/language.rs.jinja2 | 303 +++++++++++++++ .../cargo/src/runtime}/lexer.rs | 0 .../cargo/src/runtime/mod.rs} | 11 +- .../napi_interface/ast_selectors.rs.jinja2 | 356 ++++++++++++++++++ .../cargo/src/runtime}/napi_interface/cst.rs | 2 + .../src/runtime}/napi_interface/cursor.rs | 0 .../napi_interface/generated/ast_selectors.rs | 158 ++++++++ .../cargo/src/runtime}/napi_interface/mod.rs | 4 +- .../runtime}/napi_interface/parse_error.rs | 0 .../runtime}/napi_interface/parse_output.rs | 0 .../src/runtime}/napi_interface/query.rs | 0 .../src/runtime}/napi_interface/text_index.rs | 0 .../cargo/src/runtime}/parse_error.rs | 0 .../cargo/src/runtime}/parse_output.rs | 0 .../runtime}/parser_support/choice_helper.rs | 0 .../src/runtime}/parser_support/context.rs | 1 + .../cargo/src/runtime}/parser_support/mod.rs | 0 .../parser_support/optional_helper.rs | 0 .../parser_support/parser_function.rs | 0 .../runtime}/parser_support/parser_result.rs | 0 .../parser_support/precedence_helper.rs | 0 .../src/runtime}/parser_support/recovery.rs | 0 .../parser_support/repetition_helper.rs | 0 .../runtime}/parser_support/scanner_macros.rs | 0 .../parser_support/separated_helper.rs | 0 .../parser_support/sequence_helper.rs | 0 .../cargo/src/runtime}/query/engine.rs | 0 .../query/generated/user_defined_queries.rs | 3 + .../cargo/src/runtime}/query/mod.rs | 2 + .../cargo/src/runtime}/query/model.rs | 0 .../cargo/src/runtime}/query/parser.rs | 0 .../query/user_defined_queries.rs.jinja2 | 16 + .../cargo/src/runtime}/text_index.rs | 0 .../runtime/cargo/src/user_defined/mod.rs | 1 + .../runtime/cargo/src/user_defined/query.rs | 7 + .../{parser => runtime}/generator/Cargo.toml | 5 +- .../generator/src/ast.rs} | 0 .../src/keyword_scanner_definition.rs | 0 crates/codegen/runtime/generator/src/lib.rs | 67 ++++ .../generator/src/model.rs} | 159 ++------ .../generator/src/parser_definition.rs | 0 .../src/precedence_parser_definition.rs | 0 .../generator/src/scanner_definition.rs | 0 .../{parser => runtime}/generator/src/trie.rs | 0 crates/codegen/runtime/node_addon/Cargo.toml | 49 +++ crates/codegen/runtime/node_addon/build.rs | 6 + crates/codegen/runtime/npm/CHANGELOG.md | 318 ++++++++++++++++ crates/codegen/runtime/npm/Cargo.toml | 13 + crates/codegen/runtime/npm/LICENSE | 21 ++ crates/codegen/runtime/npm/README.md | 0 crates/codegen/runtime/npm/build.rs | 6 + crates/codegen/runtime/npm/package.json | 17 + crates/codegen/runtime/npm/src/lib.rs | 1 + .../npm/src/runtime/ast/ast_types.ts.jinja2 | 189 ++++++++++ .../src/runtime/ast/generated/ast_types.ts | 3 + .../runtime/npm/src/runtime}/ast/index.ts | 0 .../runtime/npm/src/runtime}/cst/index.ts | 2 +- .../runtime/npm/src/runtime}/cursor/index.ts | 2 +- .../runtime/npm/src/runtime}/index.ts | 0 .../runtime/npm/src/runtime}/kinds/index.ts | 2 +- .../npm/src/runtime}/language/index.ts | 2 +- .../napi-bindings/generated/index.d.ts | 152 ++++++++ .../runtime/napi-bindings/generated/index.js | 304 +++++++++++++++ .../npm/src/runtime}/parse_error/index.ts | 2 +- .../npm/src/runtime}/parse_output/index.ts | 2 +- .../runtime/npm/src/runtime/query/index.ts | 7 + .../npm/src/runtime}/text_index/index.ts | 2 +- crates/codegen/runtime/npm/tsconfig.json | 20 + .../codegen/spec/src/generators/navigation.rs | 12 +- crates/codegen/spec/src/lib.rs | 12 +- crates/codegen/testing/src/cst_output.rs | 21 +- crates/infra/cli/Cargo.toml | 2 + crates/infra/cli/src/commands/check/mod.rs | 7 +- crates/infra/cli/src/commands/lint/mod.rs | 13 +- .../src/commands/publish/changesets/mod.rs | 4 +- .../infra/cli/src/commands/publish/npm/mod.rs | 10 +- .../napi/bindings}/index.d.ts.jinja2 | 0 .../toolchains/napi/bindings/index.js.jinja2} | 6 +- crates/infra/cli/src/toolchains/napi/cli.rs | 6 +- .../infra/cli/src/toolchains/napi/compiler.rs | 39 +- .../infra/cli/src/toolchains/napi/config.rs | 4 +- crates/infra/cli/src/toolchains/napi/glibc.rs | 2 +- .../infra/cli/src/toolchains/napi/resolver.rs | 66 ++-- crates/infra/utils/Cargo.toml | 1 + .../utils/src/codegen/common/file_system.rs | 49 --- crates/infra/utils/src/codegen/common/mod.rs | 3 - crates/infra/utils/src/codegen/common/tera.rs | 74 ---- crates/infra/utils/src/codegen/file_system.rs | 159 ++++++++ .../src/codegen/{common => }/formatting.rs | 0 crates/infra/utils/src/codegen/mod.rs | 26 +- crates/infra/utils/src/codegen/read_write.rs | 109 ------ crates/infra/utils/src/codegen/templates.rs | 187 +++++++++ crates/infra/utils/src/codegen/tera.rs | 101 +++++ crates/infra/utils/src/codegen/write_only.rs | 77 ---- crates/infra/utils/src/paths/extensions.rs | 32 +- crates/infra/utils/src/paths/walker.rs | 4 +- .../outputs/cargo/slang_solidity/Cargo.toml | 9 +- .../outputs/cargo/slang_solidity/build.rs | 9 +- .../src/generated/{ => generated}/kinds.rs | 51 +-- .../src/generated/{ => generated}/language.rs | 33 +- .../cargo/slang_solidity/src/generated/mod.rs | 11 +- .../src/generated/napi_interface/cst.rs | 2 + .../{ => generated}/ast_selectors.rs | 2 - .../src/generated/napi_interface/mod.rs | 4 +- .../src/generated/parser_support/context.rs | 1 + .../query/generated/user_defined_queries.rs | 13 + .../slang_solidity/src/generated/query/mod.rs | 2 + .../generated/query/user_defined_queries.rs | 11 - .../slang_solidity_node_addon/Cargo.toml | 3 - .../cargo/slang_solidity_node_addon/build.rs | 4 - .../cargo/tests/src/cst_output/runner.rs | 8 +- .../solidity/outputs/npm/package/Cargo.toml | 3 +- crates/solidity/outputs/npm/package/build.rs | 10 +- .../ast/generated/ast_types.ts | 2 +- .../npm/package/src/generated/ast/index.ts | 3 + .../package/src/{ => generated}/cst/index.ts | 4 +- .../npm/package/src/generated/cursor/index.ts | 6 + .../npm/package/src/generated}/index.ts | 3 + .../src/{ => generated}/kinds/index.ts | 4 +- .../package/src/generated/language/index.ts | 6 + .../{ => napi-bindings/generated}/index.d.ts | 12 +- .../{ => napi-bindings/generated}/index.js | 4 +- .../src/generated/parse_error/index.ts | 6 + .../src/generated/parse_output/index.ts | 6 + .../npm/package/src/generated/query/index.ts | 9 + .../package/src/generated/text_index/index.ts | 7 + .../npm/package/src/parse_error/index.ts | 4 - .../npm/package/src/parse_output/index.ts | 4 - .../outputs/npm/package/src/query/index.ts | 4 - .../npm/package/src/text_index/index.ts | 5 - .../npm/package/templates/index.js.jinja2 | 6 - .../outputs/npm/package/tsconfig.json | 2 +- .../tests/src/doc-examples/using-queries.ts | 5 +- .../outputs/cargo/slang_testlang/Cargo.toml | 3 +- .../outputs/cargo/slang_testlang/build.rs | 9 +- .../src/generated/{ => generated}/kinds.rs | 38 +- .../src/generated/{ => generated}/language.rs | 36 +- .../cargo/slang_testlang/src/generated/mod.rs | 11 +- .../src/generated/napi_interface/cst.rs | 2 + .../{ => generated}/ast_selectors.rs | 2 - .../src/generated/napi_interface/mod.rs | 4 +- .../src/generated/parser_support/context.rs | 1 + .../query/generated/user_defined_queries.rs | 13 + .../slang_testlang/src/generated/query/mod.rs | 2 + .../generated/query/user_defined_queries.rs | 11 - .../slang_testlang_node_addon/Cargo.toml | 3 - .../cargo/slang_testlang_node_addon/build.rs | 4 - .../outputs/cargo/tests/src/errors/mod.rs | 5 +- .../testlang/outputs/npm/package/Cargo.toml | 3 +- crates/testlang/outputs/npm/package/build.rs | 10 +- .../outputs/npm/package/src/ast/index.ts | 1 - .../outputs/npm/package/src/cursor/index.ts | 4 - .../ast/generated/ast_types.ts | 2 +- .../npm/package/src/generated/ast/index.ts | 3 + .../npm/package/src/generated/cst/index.ts | 14 + .../npm/package/src/generated/cursor/index.ts | 6 + .../npm/package/src/generated/index.ts | 11 + .../npm/package/src/generated/kinds/index.ts | 12 + .../package/src/generated/language/index.ts | 6 + .../{ => napi-bindings/generated}/index.d.ts | 12 +- .../{ => napi-bindings/generated}/index.js | 4 +- .../src/generated/parse_error/index.ts | 6 + .../src/generated/parse_output/index.ts | 6 + .../npm/package/src/generated/query/index.ts | 9 + .../package/src/generated/text_index/index.ts | 7 + .../outputs/npm/package/src/language/index.ts | 4 - .../outputs/npm/package/src/query/index.ts | 4 - .../npm/package/templates/index.js.jinja2 | 6 - .../outputs/npm/package/tsconfig.json | 2 +- .../outputs/npm/tests/src/tests/errors.ts | 2 +- package-lock.json | 14 + package.json | 1 + tsconfig.json | 1 + 193 files changed, 3273 insertions(+), 1805 deletions(-) delete mode 100644 crates/codegen/parser/generator/src/lib.rs delete mode 100644 crates/codegen/parser/generator/src/typescript_generator.rs delete mode 100644 crates/codegen/parser/runtime/src/lib.rs delete mode 100644 crates/codegen/parser/runtime/src/napi_interface/ast_selectors.rs delete mode 100644 crates/codegen/parser/runtime/src/napi_interface/templates/ast_selectors.rs.jinja2 delete mode 100644 crates/codegen/parser/runtime/src/napi_interface/templates/ast_types.ts.jinja2 delete mode 100644 crates/codegen/parser/runtime/src/query/user_defined_queries.rs delete mode 100644 crates/codegen/parser/runtime/src/templates/language.rs.jinja2 delete mode 100644 crates/codegen/parser/runtime/src/templates/user_defined_queries.rs.jinja2 rename crates/codegen/{parser/runtime => runtime/cargo}/Cargo.toml (74%) create mode 100644 crates/codegen/runtime/cargo/build.rs create mode 100644 crates/codegen/runtime/cargo/src/lib.rs rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/cst.rs (100%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/cursor.rs (100%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime/generated}/kinds.rs (67%) create mode 100644 crates/codegen/runtime/cargo/src/runtime/generated/language.rs rename crates/codegen/{parser/runtime/src/templates => runtime/cargo/src/runtime}/kinds.rs.jinja2 (53%) create mode 100644 crates/codegen/runtime/cargo/src/runtime/language.rs.jinja2 rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/lexer.rs (100%) rename crates/codegen/{parser/runtime/src/mod_for_destination.rs => runtime/cargo/src/runtime/mod.rs} (71%) create mode 100644 crates/codegen/runtime/cargo/src/runtime/napi_interface/ast_selectors.rs.jinja2 rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/napi_interface/cst.rs (99%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/napi_interface/cursor.rs (100%) create mode 100644 crates/codegen/runtime/cargo/src/runtime/napi_interface/generated/ast_selectors.rs rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/napi_interface/mod.rs (95%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/napi_interface/parse_error.rs (100%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/napi_interface/parse_output.rs (100%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/napi_interface/query.rs (100%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/napi_interface/text_index.rs (100%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/parse_error.rs (100%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/parse_output.rs (100%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/parser_support/choice_helper.rs (100%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/parser_support/context.rs (99%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/parser_support/mod.rs (100%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/parser_support/optional_helper.rs (100%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/parser_support/parser_function.rs (100%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/parser_support/parser_result.rs (100%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/parser_support/precedence_helper.rs (100%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/parser_support/recovery.rs (100%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/parser_support/repetition_helper.rs (100%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/parser_support/scanner_macros.rs (100%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/parser_support/separated_helper.rs (100%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/parser_support/sequence_helper.rs (100%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/query/engine.rs (100%) create mode 100644 crates/codegen/runtime/cargo/src/runtime/query/generated/user_defined_queries.rs rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/query/mod.rs (80%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/query/model.rs (100%) rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/query/parser.rs (100%) create mode 100644 crates/codegen/runtime/cargo/src/runtime/query/user_defined_queries.rs.jinja2 rename crates/codegen/{parser/runtime/src => runtime/cargo/src/runtime}/text_index.rs (100%) create mode 100644 crates/codegen/runtime/cargo/src/user_defined/mod.rs create mode 100644 crates/codegen/runtime/cargo/src/user_defined/query.rs rename crates/codegen/{parser => runtime}/generator/Cargo.toml (93%) rename crates/codegen/{parser/generator/src/ast_model.rs => runtime/generator/src/ast.rs} (100%) rename crates/codegen/{parser => runtime}/generator/src/keyword_scanner_definition.rs (100%) create mode 100644 crates/codegen/runtime/generator/src/lib.rs rename crates/codegen/{parser/generator/src/rust_generator.rs => runtime/generator/src/model.rs} (73%) rename crates/codegen/{parser => runtime}/generator/src/parser_definition.rs (100%) rename crates/codegen/{parser => runtime}/generator/src/precedence_parser_definition.rs (100%) rename crates/codegen/{parser => runtime}/generator/src/scanner_definition.rs (100%) rename crates/codegen/{parser => runtime}/generator/src/trie.rs (100%) create mode 100644 crates/codegen/runtime/node_addon/Cargo.toml create mode 100644 crates/codegen/runtime/node_addon/build.rs create mode 100644 crates/codegen/runtime/npm/CHANGELOG.md create mode 100644 crates/codegen/runtime/npm/Cargo.toml create mode 100644 crates/codegen/runtime/npm/LICENSE create mode 100644 crates/codegen/runtime/npm/README.md create mode 100644 crates/codegen/runtime/npm/build.rs create mode 100644 crates/codegen/runtime/npm/package.json create mode 100644 crates/codegen/runtime/npm/src/lib.rs create mode 100644 crates/codegen/runtime/npm/src/runtime/ast/ast_types.ts.jinja2 create mode 100644 crates/codegen/runtime/npm/src/runtime/ast/generated/ast_types.ts rename crates/{solidity/outputs/npm/package/src => codegen/runtime/npm/src/runtime}/ast/index.ts (100%) rename crates/{testlang/outputs/npm/package/src => codegen/runtime/npm/src/runtime}/cst/index.ts (85%) rename crates/{solidity/outputs/npm/package/src => codegen/runtime/npm/src/runtime}/cursor/index.ts (62%) rename crates/{solidity/outputs/npm/package/src => codegen/runtime/npm/src/runtime}/index.ts (100%) rename crates/{testlang/outputs/npm/package/src => codegen/runtime/npm/src/runtime}/kinds/index.ts (84%) rename crates/{solidity/outputs/npm/package/src => codegen/runtime/npm/src/runtime}/language/index.ts (65%) create mode 100644 crates/codegen/runtime/npm/src/runtime/napi-bindings/generated/index.d.ts create mode 100644 crates/codegen/runtime/npm/src/runtime/napi-bindings/generated/index.js rename crates/{testlang/outputs/npm/package/src => codegen/runtime/npm/src/runtime}/parse_error/index.ts (67%) rename crates/{testlang/outputs/npm/package/src => codegen/runtime/npm/src/runtime}/parse_output/index.ts (68%) create mode 100644 crates/codegen/runtime/npm/src/runtime/query/index.ts rename crates/{testlang/outputs/npm/package/src => codegen/runtime/npm/src/runtime}/text_index/index.ts (66%) create mode 100644 crates/codegen/runtime/npm/tsconfig.json rename crates/{solidity/outputs/npm/package/templates => infra/cli/src/toolchains/napi/bindings}/index.d.ts.jinja2 (100%) rename crates/{testlang/outputs/npm/package/templates/index.d.ts.jinja2 => infra/cli/src/toolchains/napi/bindings/index.js.jinja2} (77%) delete mode 100644 crates/infra/utils/src/codegen/common/file_system.rs delete mode 100644 crates/infra/utils/src/codegen/common/mod.rs delete mode 100644 crates/infra/utils/src/codegen/common/tera.rs create mode 100644 crates/infra/utils/src/codegen/file_system.rs rename crates/infra/utils/src/codegen/{common => }/formatting.rs (100%) delete mode 100644 crates/infra/utils/src/codegen/read_write.rs create mode 100644 crates/infra/utils/src/codegen/templates.rs create mode 100644 crates/infra/utils/src/codegen/tera.rs delete mode 100644 crates/infra/utils/src/codegen/write_only.rs rename crates/solidity/outputs/cargo/slang_solidity/src/generated/{ => generated}/kinds.rs (94%) rename crates/solidity/outputs/cargo/slang_solidity/src/generated/{ => generated}/language.rs (99%) rename crates/solidity/outputs/cargo/slang_solidity/src/generated/napi_interface/{ => generated}/ast_selectors.rs (99%) create mode 100644 crates/solidity/outputs/cargo/slang_solidity/src/generated/query/generated/user_defined_queries.rs delete mode 100644 crates/solidity/outputs/cargo/slang_solidity/src/generated/query/user_defined_queries.rs rename crates/solidity/outputs/npm/package/src/{ => generated}/ast/generated/ast_types.ts (99%) create mode 100644 crates/solidity/outputs/npm/package/src/generated/ast/index.ts rename crates/solidity/outputs/npm/package/src/{ => generated}/cst/index.ts (68%) create mode 100644 crates/solidity/outputs/npm/package/src/generated/cursor/index.ts rename crates/{testlang/outputs/npm/package/src => solidity/outputs/npm/package/src/generated}/index.ts (70%) rename crates/solidity/outputs/npm/package/src/{ => generated}/kinds/index.ts (66%) create mode 100644 crates/solidity/outputs/npm/package/src/generated/language/index.ts rename crates/solidity/outputs/npm/package/src/generated/{ => napi-bindings/generated}/index.d.ts (100%) rename crates/solidity/outputs/npm/package/src/generated/{ => napi-bindings/generated}/index.js (98%) create mode 100644 crates/solidity/outputs/npm/package/src/generated/parse_error/index.ts create mode 100644 crates/solidity/outputs/npm/package/src/generated/parse_output/index.ts create mode 100644 crates/solidity/outputs/npm/package/src/generated/query/index.ts create mode 100644 crates/solidity/outputs/npm/package/src/generated/text_index/index.ts delete mode 100644 crates/solidity/outputs/npm/package/src/parse_error/index.ts delete mode 100644 crates/solidity/outputs/npm/package/src/parse_output/index.ts delete mode 100644 crates/solidity/outputs/npm/package/src/query/index.ts delete mode 100644 crates/solidity/outputs/npm/package/src/text_index/index.ts delete mode 100644 crates/solidity/outputs/npm/package/templates/index.js.jinja2 rename crates/testlang/outputs/cargo/slang_testlang/src/generated/{ => generated}/kinds.rs (80%) rename crates/testlang/outputs/cargo/slang_testlang/src/generated/{ => generated}/language.rs (98%) rename crates/testlang/outputs/cargo/slang_testlang/src/generated/napi_interface/{ => generated}/ast_selectors.rs (99%) create mode 100644 crates/testlang/outputs/cargo/slang_testlang/src/generated/query/generated/user_defined_queries.rs delete mode 100644 crates/testlang/outputs/cargo/slang_testlang/src/generated/query/user_defined_queries.rs delete mode 100644 crates/testlang/outputs/npm/package/src/ast/index.ts delete mode 100644 crates/testlang/outputs/npm/package/src/cursor/index.ts rename crates/testlang/outputs/npm/package/src/{ => generated}/ast/generated/ast_types.ts (99%) create mode 100644 crates/testlang/outputs/npm/package/src/generated/ast/index.ts create mode 100644 crates/testlang/outputs/npm/package/src/generated/cst/index.ts create mode 100644 crates/testlang/outputs/npm/package/src/generated/cursor/index.ts create mode 100644 crates/testlang/outputs/npm/package/src/generated/index.ts create mode 100644 crates/testlang/outputs/npm/package/src/generated/kinds/index.ts create mode 100644 crates/testlang/outputs/npm/package/src/generated/language/index.ts rename crates/testlang/outputs/npm/package/src/generated/{ => napi-bindings/generated}/index.d.ts (100%) rename crates/testlang/outputs/npm/package/src/generated/{ => napi-bindings/generated}/index.js (98%) create mode 100644 crates/testlang/outputs/npm/package/src/generated/parse_error/index.ts create mode 100644 crates/testlang/outputs/npm/package/src/generated/parse_output/index.ts create mode 100644 crates/testlang/outputs/npm/package/src/generated/query/index.ts create mode 100644 crates/testlang/outputs/npm/package/src/generated/text_index/index.ts delete mode 100644 crates/testlang/outputs/npm/package/src/language/index.ts delete mode 100644 crates/testlang/outputs/npm/package/src/query/index.ts delete mode 100644 crates/testlang/outputs/npm/package/templates/index.js.jinja2 diff --git a/Cargo.lock b/Cargo.lock index 04d814972e..6c3b86fa32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -378,7 +378,25 @@ dependencies = [ ] [[package]] -name = "codegen_parser_generator" +name = "codegen_runtime_cargo" +version = "0.14.2" +dependencies = [ + "anyhow", + "ariadne", + "codegen_runtime_generator", + "napi", + "napi-derive", + "nom", + "semver", + "serde", + "serde_json", + "strum", + "strum_macros", + "thiserror", +] + +[[package]] +name = "codegen_runtime_generator" version = "0.14.2" dependencies = [ "Inflector", @@ -395,17 +413,29 @@ dependencies = [ ] [[package]] -name = "codegen_parser_runtime" +name = "codegen_runtime_node_addon" version = "0.14.2" dependencies = [ "ariadne", "napi", + "napi-build", "napi-derive", "nom", + "semver", "serde", "serde_json", "strum", "strum_macros", + "thiserror", +] + +[[package]] +name = "codegen_runtime_npm" +version = "0.14.2" +dependencies = [ + "anyhow", + "codegen_runtime_generator", + "infra_utils", ] [[package]] @@ -947,6 +977,8 @@ dependencies = [ "semver", "serde", "serde_json", + "strum", + "strum_macros", "tempfile", "toml", ] @@ -957,6 +989,7 @@ version = "0.14.2" dependencies = [ "Inflector", "anyhow", + "ariadne", "cargo-emit", "console", "ignore", @@ -1758,7 +1791,7 @@ dependencies = [ "anyhow", "ariadne", "clap", - "codegen_parser_generator", + "codegen_runtime_generator", "infra_utils", "nom", "semver", @@ -1782,7 +1815,6 @@ dependencies = [ "semver", "serde", "serde_json", - "slang_solidity", "strum", "strum_macros", "thiserror", @@ -1794,7 +1826,7 @@ version = "0.14.2" dependencies = [ "anyhow", "ariadne", - "codegen_parser_generator", + "codegen_runtime_generator", "infra_utils", "nom", "semver", @@ -1817,7 +1849,6 @@ dependencies = [ "semver", "serde", "serde_json", - "slang_testlang", "strum", "strum_macros", "thiserror", @@ -1883,7 +1914,7 @@ name = "solidity_npm_package" version = "0.14.2" dependencies = [ "anyhow", - "codegen_parser_generator", + "codegen_runtime_generator", "infra_utils", "solidity_language", ] @@ -2082,7 +2113,7 @@ name = "testlang_npm_package" version = "0.14.2" dependencies = [ "anyhow", - "codegen_parser_generator", + "codegen_runtime_generator", "infra_utils", "testlang_language", ] diff --git a/Cargo.toml b/Cargo.toml index fc6caf299a..c33003c0a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,10 +13,13 @@ members = [ "crates/codegen/language/internal_macros", "crates/codegen/language/macros", "crates/codegen/language/tests", - "crates/codegen/parser/generator", - "crates/codegen/parser/runtime", + "crates/codegen/runtime/cargo", + "crates/codegen/runtime/generator", + "crates/codegen/runtime/node_addon", + "crates/codegen/runtime/npm", "crates/codegen/spec", "crates/codegen/testing", + "crates/infra/cli", "crates/infra/utils", @@ -47,10 +50,13 @@ codegen_language_definition = { path = "crates/codegen/language/definition" } codegen_language_internal_macros = { path = "crates/codegen/language/internal_macros" } codegen_language_macros = { path = "crates/codegen/language/macros" } codegen_language_tests = { path = "crates/codegen/language/tests" } -codegen_parser_generator = { path = "crates/codegen/parser/generator" } -codegen_parser_runtime = { path = "crates/codegen/parser/runtime" } +codegen_runtime_cargo = { path = "crates/codegen/runtime/cargo" } +codegen_runtime_generator = { path = "crates/codegen/runtime/generator" } +codegen_runtime_node_addon = { path = "crates/codegen/runtime/node_addon" } +codegen_runtime_npm = { path = "crates/codegen/runtime/npm" } codegen_spec = { path = "crates/codegen/spec" } codegen_testing = { path = "crates/codegen/testing" } + infra_cli = { path = "crates/infra/cli" } infra_utils = { path = "crates/infra/utils" } diff --git a/crates/codegen/language/tests/src/fail/mod.rs b/crates/codegen/language/tests/src/fail/mod.rs index e5156a88ac..1b1426e485 100644 --- a/crates/codegen/language/tests/src/fail/mod.rs +++ b/crates/codegen/language/tests/src/fail/mod.rs @@ -21,7 +21,6 @@ fn run_trybuild() { let tests = FileWalker::from_directory(crate_dir) .find(["src/fail/**/test.rs"]) .unwrap() - .into_iter() .collect::>(); assert!(!tests.is_empty(), "No tests found."); diff --git a/crates/codegen/parser/generator/src/lib.rs b/crates/codegen/parser/generator/src/lib.rs deleted file mode 100644 index 6f3cda2e24..0000000000 --- a/crates/codegen/parser/generator/src/lib.rs +++ /dev/null @@ -1,11 +0,0 @@ -mod ast_model; -mod keyword_scanner_definition; -mod parser_definition; -mod precedence_parser_definition; -mod rust_generator; -mod scanner_definition; -mod trie; -mod typescript_generator; - -pub use rust_generator::RustGenerator; -pub use typescript_generator::TypeScriptGenerator; diff --git a/crates/codegen/parser/generator/src/typescript_generator.rs b/crates/codegen/parser/generator/src/typescript_generator.rs deleted file mode 100644 index 282ba1d84c..0000000000 --- a/crates/codegen/parser/generator/src/typescript_generator.rs +++ /dev/null @@ -1,36 +0,0 @@ -use std::path::Path; - -use anyhow::Result; -use codegen_language_definition::model::Language; -use infra_utils::cargo::CargoWorkspace; -use infra_utils::codegen::Codegen; -use serde::Serialize; - -use crate::ast_model::AstModel; - -pub struct TypeScriptGenerator; - -impl TypeScriptGenerator { - pub fn generate(language: &Language, output_dir: &Path) -> Result<()> { - let runtime_dir = - CargoWorkspace::locate_source_crate("codegen_parser_runtime")?.join("src"); - - let mut codegen = Codegen::read_write(&runtime_dir)?; - - { - #[derive(Serialize)] - struct Context { - ast_model: AstModel, - } - codegen.render( - Context { - ast_model: AstModel::create(language), - }, - runtime_dir.join("napi_interface/templates/ast_types.ts.jinja2"), - output_dir.join("src/ast/generated/ast_types.ts"), - )?; - } - - Ok(()) - } -} diff --git a/crates/codegen/parser/runtime/src/lib.rs b/crates/codegen/parser/runtime/src/lib.rs deleted file mode 100644 index 9f521422e5..0000000000 --- a/crates/codegen/parser/runtime/src/lib.rs +++ /dev/null @@ -1,27 +0,0 @@ -#![allow(dead_code)] - -#[macro_use] -mod parser_support; - -pub mod cst; -pub mod cursor; -pub mod kinds; -pub(crate) mod lexer; -pub mod parse_error; -pub mod parse_output; -pub mod query; -pub mod text_index; - -#[cfg(feature = "slang_napi_interfaces")] -pub mod napi_interface; - -// TODO(#863): replace with the same hierarchy as the product crate: -mod user_defined { - pub mod query { - pub struct UserDefinedQueriesImpl; - - impl crate::query::UserDefinedQueries for UserDefinedQueriesImpl { - // Empty Stub - } - } -} diff --git a/crates/codegen/parser/runtime/src/napi_interface/ast_selectors.rs b/crates/codegen/parser/runtime/src/napi_interface/ast_selectors.rs deleted file mode 100644 index 2234ae1a26..0000000000 --- a/crates/codegen/parser/runtime/src/napi_interface/ast_selectors.rs +++ /dev/null @@ -1 +0,0 @@ -//! Generated by the templating engine. diff --git a/crates/codegen/parser/runtime/src/napi_interface/templates/ast_selectors.rs.jinja2 b/crates/codegen/parser/runtime/src/napi_interface/templates/ast_selectors.rs.jinja2 deleted file mode 100644 index 201ac27a01..0000000000 --- a/crates/codegen/parser/runtime/src/napi_interface/templates/ast_selectors.rs.jinja2 +++ /dev/null @@ -1,332 +0,0 @@ -#![allow(clippy::too_many_lines)] - -use std::rc::Rc; - -use napi::Either; -use napi_derive::napi; - -use crate::napi_interface::cst::{RuleNode, TokenNode}; -use crate::napi_interface::{RuleKind, RustLabeledNode, RustNode, RustRuleNode, TokenKind}; - -// -// Sequences: -// - -#[napi( - namespace = "ast_internal", - ts_return_type = "Array", - catch_unwind, -)] -pub fn select_sequence( - #[napi(ts_arg_type = "cst.RuleNode")] node: &RuleNode, -) -> Result>>> { - let mut selector = Selector::new(node); - - let result = match node.kind() { - {%- for sequence in ast_model.sequences -%} - RuleKind::{{ sequence.name }} => { - selector.{{ sequence.name | snake_case }}()? - }, - {%- endfor -%} - _ => { - return Error::UnexpectedParent(node.kind()).into(); - } - }; - - selector.finalize()?; - Ok(result) -} - -{% for sequence in ast_model.sequences %} - impl Selector { - fn {{ sequence.name | snake_case }}(&mut self) -> Result>>> { - Ok(vec![ - {%- for field in sequence.fields -%} - {%- if field.is_optional -%} - - self.try_select(|node| { - {%- if field.is_terminal -%} - node.is_token_with_kind(TokenKind::{{ field.reference }}) - {%- else -%} - node.is_rule_with_kind(RuleKind::{{ field.reference }}) - {%- endif -%} - })?, - - {%- else -%} - - Some(self.select(|node| { - {%- if field.is_terminal -%} - node.is_token_with_kind(TokenKind::{{ field.reference }}) - {%- else -%} - node.is_rule_with_kind(RuleKind::{{ field.reference }}) - {%- endif -%} - })?), - - {%- endif -%} - {%- endfor -%} - ]) - } - } -{% endfor %} - -// -// Choices: -// - -#[napi( - namespace = "ast_internal", - ts_return_type = "cst.Node", - catch_unwind, -)] -pub fn select_choice( - #[napi(ts_arg_type = "cst.RuleNode")] node: &RuleNode, -) -> Result> { - let mut selector = Selector::new(node); - - let result = match node.kind() { - {%- for choice in ast_model.choices -%} - RuleKind::{{ choice.name }} => { - selector.{{ choice.name | snake_case }}()? - }, - {%- endfor -%} - _ => { - return Error::UnexpectedParent(node.kind()).into(); - } - }; - - selector.finalize()?; - Ok(result) -} - -{% for choice in ast_model.choices %} - impl Selector { - fn {{ choice.name | snake_case }}(&mut self) -> Result> { - self.select(|node| { - {%- set non_terminals_len = choice.non_terminals | length -%} - {%- set terminals_len = choice.terminals | length -%} - - {%- if non_terminals_len == 1 -%} - node.is_rule_with_kind(RuleKind::{{ choice.non_terminals[0] }}) - {%- elif non_terminals_len > 1 -%} - node.is_rule_with_kinds(&[ - {%- for non_terminal in choice.non_terminals -%} - RuleKind::{{ non_terminal }}, - {%- endfor -%} - ]) - {%- endif -%} - - {%- if non_terminals_len > 0 and terminals_len > 0 -%} - || - {%- endif -%} - - {%- if terminals_len == 1 -%} - node.is_token_with_kind(TokenKind::{{ choice.terminals[0] }}) - {%- elif terminals_len > 1 -%} - node.is_token_with_kinds(&[ - {%- for terminal in choice.terminals -%} - TokenKind::{{ terminal }}, - {%- endfor -%} - ]) - {%- endif -%} - }) - } - } -{% endfor %} - -// -// Repeated: -// - -#[napi( - namespace = "ast_internal", - ts_return_type = "Array", - catch_unwind, -)] -pub fn select_repeated( - #[napi(ts_arg_type = "cst.RuleNode")] node: &RuleNode, -) -> Result>> { - let mut selector = Selector::new(node); - - let result = match node.kind() { - {%- for repeated in ast_model.repeated -%} - RuleKind::{{ repeated.name }} => { - selector.{{ repeated.name | snake_case }}()? - }, - {%- endfor -%} - _ => { - return Error::UnexpectedParent(node.kind()).into(); - } - }; - - selector.finalize()?; - Ok(result) -} - -{% for repeated in ast_model.repeated %} - impl Selector { - fn {{ repeated.name | snake_case }}(&mut self) -> Result>> { - let mut items = vec![]; - - while let Some(item) = self.try_select(|node| { - {%- if repeated.is_terminal -%} - node.is_token_with_kind(TokenKind::{{ repeated.reference }}) - {%- else -%} - node.is_rule_with_kind(RuleKind::{{ repeated.reference }}) - {%- endif -%} - })? { - items.push(item); - } - - Ok(items) - } - } -{% endfor %} - -// -// Separated: -// - -#[napi( - namespace = "ast_internal", - ts_return_type = "[Array, Array]", - catch_unwind, -)] -pub fn select_separated( - #[napi(ts_arg_type = "cst.RuleNode")] node: &RuleNode, -) -> Result>>> { - let mut selector = Selector::new(node); - - let result = match node.kind() { - {%- for separated in ast_model.separated -%} - RuleKind::{{ separated.name }} => { - selector.{{ separated.name | snake_case }}()? - }, - {%- endfor -%} - _ => { - return Error::UnexpectedParent(node.kind()).into(); - } - }; - - selector.finalize()?; - Ok(result) -} - -{% for separated in ast_model.separated %} - impl Selector { - fn {{ separated.name | snake_case }}(&mut self) -> Result>>> { - let mut separated = vec![]; - let mut separators = vec![]; - - if let Some(first) = self.try_select(|node| { - {%- if separated.is_terminal -%} - node.is_token_with_kind(TokenKind::{{ separated.reference }}) - {%- else -%} - node.is_rule_with_kind(RuleKind::{{ separated.reference }}) - {%- endif -%} - })? { - separated.push(first); - - while let Some(separator) = self.try_select(|node| node.is_token_with_kind(TokenKind::{{ separated.separator }}))? { - separators.push(separator); - - separated.push(self.select(|node| { - {%- if separated.is_terminal -%} - node.is_token_with_kind(TokenKind::{{ separated.reference }}) - {%- else -%} - node.is_rule_with_kind(RuleKind::{{ separated.reference }}) - {%- endif -%} - })?); - } - } - - Ok(vec![separated, separators]) - } - } -{% endfor %} - -// -// Common: -// - -struct Selector { - node: Rc, - index: usize, -} - -impl Selector { - fn new(node: &RuleNode) -> Self { - Self { - node: Rc::clone(&node.0), - index: 0, - } - } - - fn select(&mut self, filter: impl FnOnce(&RustNode) -> bool) -> Result> { - match self.try_select(filter)? { - Some(node) => Ok(node), - None => Error::MissingChild(self.index).into(), - } - } - - fn try_select(&mut self, filter: impl FnOnce(&RustNode) -> bool) -> Result>> { - while let Some(child) = self.node.children.get(self.index) { - match child { - node if node.is_trivia() => { - // skip trivia, since it's not part of the AST - self.index += 1; - continue; - } - RustLabeledNode { - label: _, - node: RustNode::Token(token), - } if matches!(token.kind, TokenKind::SKIPPED) => { - return Error::SkippedToken(self.index).into(); - } - labeled if filter(labeled) => { - self.index += 1; - return Ok(Some(labeled.node.clone().into_js_either_node())); - } - _ => { - break; - }, - } - } - - Ok(None) - } - - fn finalize(mut self) -> Result<()> { - if self.try_select(|_| true)?.is_some() { - return Error::UnexpectedTrailing(self.index - 1).into(); - } - - Ok(()) - } -} - -type Result = std::result::Result; - -#[derive(Debug, thiserror::Error)] -enum Error { - // Should not theoretically happen, since we're only called from our own generated AST types. - #[error("Unexpected parent node with RuleKind '{0}'.")] - UnexpectedParent(RuleKind), - - // Should not theoretically happen, since we're only called from our own generated AST types. - #[error("Unexpected trailing children at index '{0}'.")] - UnexpectedTrailing(usize), - - // Should not theoretically happen, unless AST error recovery was changed. - #[error("Missing child node at index '{0}'.")] - MissingChild(usize), - - // Can happen if the user decided to use an incorrect/incomplete CST node. - #[error("Unexpected SKIPPED token at index '{0}'. Creating AST types from incorrect/incomplete CST nodes is not supported yet.")] - SkippedToken(usize), -} - -impl From for Result { - fn from(error: Error) -> Self { - Err(napi::Error::from_reason(error.to_string())) - } -} diff --git a/crates/codegen/parser/runtime/src/napi_interface/templates/ast_types.ts.jinja2 b/crates/codegen/parser/runtime/src/napi_interface/templates/ast_types.ts.jinja2 deleted file mode 100644 index a63320d661..0000000000 --- a/crates/codegen/parser/runtime/src/napi_interface/templates/ast_types.ts.jinja2 +++ /dev/null @@ -1,185 +0,0 @@ -import * as assert from "node:assert"; -import { ast_internal } from "../../generated"; -import { RuleNode, TokenNode } from "../../cst"; -import { RuleKind, TokenKind } from "../../kinds"; - -/* - * Sequences: - */ - -{% for sequence in ast_model.sequences %} - export class {{ sequence.name }} { - private readonly fetch = once(() => { - const [ - {%- for field in sequence.fields %} - ${{ field.name | camel_case }}, - {%- endfor %} - ] = ast_internal.selectSequence(this.cst); - - return { - {%- for field in sequence.fields %} - {{ field.name | camel_case }}: - {%- if field.is_optional -%} - ${{ field.name | camel_case }} === null ? undefined : - {%- endif -%} - {%- if field.is_terminal -%} - ${{ field.name | camel_case }} as TokenNode, - {%- else -%} - new {{ field.reference }}(${{ field.name | camel_case }} as RuleNode), - {%- endif -%} - {% endfor -%} - }; - }); - - public constructor(public readonly cst: RuleNode) { - assertKind(this.cst.kind, RuleKind.{{ sequence.name }}); - } - - {% for field in sequence.fields %} - public get {{ field.name | camel_case }}() : - {%- if field.is_terminal -%} - TokenNode - {%- else -%} - {{ field.reference }} - {%- endif -%} - {%- if field.is_optional -%} - | undefined - {%- endif -%} - { - return this.fetch().{{ field.name | camel_case }}; - } - {% endfor %} - } -{% endfor %} - -/* - * Choices: - */ - -{% for choice in ast_model.choices %} - export class {{ choice.name }} { - {%- set variant_types = choice.non_terminals -%} - {%- if choice.terminals | length > 0 -%} - {%- set variant_types = variant_types | concat(with = "TokenNode") -%} - {%- endif -%} - {%- set variant_types = variant_types | join(sep = " | ") -%} - - private readonly fetch: () => {{ variant_types }} = once(() => { - const variant = ast_internal.selectChoice(this.cst); - - switch (variant.kind) { - {%- for non_terminal in choice.non_terminals %} - case RuleKind.{{ non_terminal }}: - return new {{ non_terminal }}(variant as RuleNode); - {%- endfor %} - - {% if choice.terminals | length > 0 %} - {%- for terminal in choice.terminals %} - case TokenKind.{{ terminal }}: - {%- endfor %} - return variant as TokenNode; - {%- endif %} - - default: - assert.fail(`Unexpected variant: ${variant.kind}`); - } - }); - - public constructor(public readonly cst: RuleNode) { - assertKind(this.cst.kind, RuleKind.{{ choice.name }}); - } - - public get variant(): {{ variant_types }} { - return this.fetch(); - } - } -{% endfor %} - -/* - * Repeated: - */ - -{% for repeated in ast_model.repeated %} - export class {{ repeated.name }} { - private readonly fetch = once(() => { - const items = ast_internal.selectRepeated(this.cst); - - {%- if repeated.is_terminal -%} - return items as TokenNode[]; - {%- else -%} - return items.map((item) => new {{ repeated.reference }}(item as RuleNode)); - {%- endif -%} - }); - - public constructor(public readonly cst: RuleNode) { - assertKind(this.cst.kind, RuleKind.{{ repeated.name }}); - } - - public get items(): - {%- if repeated.is_terminal -%} - readonly TokenNode[] - {%- else -%} - readonly {{ repeated.reference }}[] - {%- endif -%} - { - return this.fetch(); - } - } -{% endfor %} - -/* - * Separated: - */ - -{% for separated in ast_model.separated %} - export class {{ separated.name }} { - private readonly fetch = once(() => { - const [items, separators] = ast_internal.selectSeparated(this.cst); - - return { - {%- if separated.is_terminal -%} - items: items as TokenNode[], - {%- else -%} - items: items.map((item) => new {{ separated.reference }}(item as RuleNode)), - {%- endif -%} - separators: separators as TokenNode[], - }; - }); - - public constructor(public readonly cst: RuleNode) { - assertKind(this.cst.kind, RuleKind.{{ separated.name }}); - } - - public get items(): - {%- if separated.is_terminal -%} - readonly TokenNode[] - {%- else -%} - readonly {{ separated.reference }}[] - {%- endif -%} - { - return this.fetch().items; - } - - public get separators(): readonly TokenNode[] { - return this.fetch().separators; - } - } -{% endfor %} - -/* - * Helpers: - */ - -function once(factory: () => T): () => T { - let value: T | undefined; - return () => { - if (value === undefined) { - value = factory(); - } - return value; - }; -} - -function assertKind(actual: RuleKind, expected: RuleKind): void { - assert.equal(actual, expected, `${expected} can only be initialized with a CST node of the same kind.`); -} diff --git a/crates/codegen/parser/runtime/src/query/user_defined_queries.rs b/crates/codegen/parser/runtime/src/query/user_defined_queries.rs deleted file mode 100644 index df3ba8113b..0000000000 --- a/crates/codegen/parser/runtime/src/query/user_defined_queries.rs +++ /dev/null @@ -1,3 +0,0 @@ -// Empty stub. Generated by codegen in output crates. - -pub(crate) trait UserDefinedQueries {} diff --git a/crates/codegen/parser/runtime/src/templates/language.rs.jinja2 b/crates/codegen/parser/runtime/src/templates/language.rs.jinja2 deleted file mode 100644 index 95c6ad98d9..0000000000 --- a/crates/codegen/parser/runtime/src/templates/language.rs.jinja2 +++ /dev/null @@ -1,271 +0,0 @@ -// This file is generated; we can't reasonably satisfy some of these lints. -#![allow( - clippy::if_not_else, - clippy::too_many_lines, - clippy::unused_self, - clippy::struct_excessive_bools, - clippy::similar_names, - unused_imports -)] - -use semver::Version; -#[cfg(feature = "slang_napi_interfaces")] -use napi_derive::napi; - -use crate::cst; -use crate::kinds::{ - NodeLabel, IsLexicalContext, LexicalContext, LexicalContextType, RuleKind, TokenKind, -}; -use crate::lexer::{KeywordScan, Lexer, ScannedToken}; -#[cfg(feature = "slang_napi_interfaces")] -use crate::napi_interface::parse_output::ParseOutput as NAPIParseOutput; -use crate::parse_output::ParseOutput; -use crate::parser_support::{ - ChoiceHelper, OneOrMoreHelper, OptionalHelper, ParserContext, ParserFunction, ParserResult, - PrecedenceHelper, SeparatedHelper, SequenceHelper, TokenAcceptanceThreshold, ZeroOrMoreHelper, -}; - -#[derive(Debug)] -#[cfg_attr(feature = "slang_napi_interfaces", napi(namespace = "language"))] -pub struct Language { - pub(crate) version: Version, - {%- for version in generator.referenced_versions -%} - pub(crate) version_is_at_least_{{ version | replace(from=".", to="_") }}: bool, - {%- endfor -%} -} - -#[derive(thiserror::Error, Debug)] -pub enum Error { - #[error("Unsupported {{ language_name }} language version '{0}'.")] - UnsupportedLanguageVersion(Version), - - #[cfg(feature = "slang_napi_interfaces")] - #[error("Invalid semantic version '{0}'.")] - InvalidSemanticVersion(String), -} - -#[cfg(feature = "slang_napi_interfaces")] -impl From for napi::Error { - fn from(value: Error) -> Self { - napi::Error::from_reason(value.to_string()) - } -} - -impl Language { - pub const SUPPORTED_VERSIONS: &'static [Version] = &[ - {% for version in versions %} - Version::new({{ version | split(pat=".") | join(sep=", ") }}), - {% endfor %} - ]; - - pub fn new(version: Version) -> std::result::Result { - if Self::SUPPORTED_VERSIONS.binary_search(&version).is_ok() { - Ok(Self { - {%- for version in generator.referenced_versions %} - version_is_at_least_{{ version | replace(from=".", to="_") }}: Version::new({{ version | split(pat=".") | join(sep=", ") }}) <= version, - {%- endfor -%} - version, - }) - } else { - Err(Error::UnsupportedLanguageVersion(version)) - } - } - - pub fn version(&self) -> &Version { - &self.version - } - - /******************************************** - * Parser Functions - ********************************************/ - - {% for parser_name, parser_code in generator.parser_functions %} - #[allow(unused_assignments, unused_parens)] - fn {{ parser_name | snake_case }}(&self, input: &mut ParserContext<'_>) -> ParserResult { {{ parser_code }} } - {% endfor %} - - {% for parser_name, parser_code in generator.trivia_parser_functions %} - #[allow(unused_assignments, unused_parens)] - fn {{ parser_name | snake_case }}(&self, input: &mut ParserContext<'_>) -> ParserResult { {{ parser_code }} } - {% endfor %} - - /******************************************** - * Scanner Functions - ********************************************/ - - {% for scanner_name, scanner_code in generator.scanner_functions %} - #[allow(unused_assignments, unused_parens)] - fn {{ scanner_name | snake_case }}(&self, input: &mut ParserContext<'_>) -> bool { {{ scanner_code }} } - {% endfor %} - - // Keyword scanners - {%- for keyword_name, keyword_code in generator.keyword_compound_scanners %} - #[inline] - fn {{ keyword_name | snake_case }}(&self, input: &mut ParserContext<'_>, ident: &str) -> KeywordScan { {{ keyword_code }} } - {% endfor %} - - pub fn parse(&self, kind: RuleKind, input: &str) -> ParseOutput { - match kind { - {%- for parser_name, _ in generator.parser_functions -%} - RuleKind::{{ parser_name }} => Self::{{ parser_name | snake_case }}.parse(self, input), - {%- endfor -%} - } - } -} - -impl Lexer for Language { - fn leading_trivia(&self, input: &mut ParserContext<'_>) -> ParserResult { - Language::leading_trivia(self, input) - } - - fn trailing_trivia(&self, input: &mut ParserContext<'_>) -> ParserResult { - Language::trailing_trivia(self, input) - } - - fn delimiters() -> &'static [(TokenKind, TokenKind)] { - match LexCtx::value() { - {%- for context_name, context in generator.scanner_contexts %} - LexicalContext::{{ context_name }} => &[ - {%- for open, close in context.delimiters %} - (TokenKind::{{ open }}, TokenKind::{{ close }}), - {%- endfor %} - ], - {%- endfor %} - } - } - - fn next_token(&self, input: &mut ParserContext<'_>) -> Option { - let save = input.position(); - let mut furthest_position = input.position(); - let mut longest_token = None; - - macro_rules! longest_match { - ($( { $kind:ident = $function:ident } )*) => { - $( - if self.$function(input) && input.position() > furthest_position { - furthest_position = input.position(); - - longest_token = Some(TokenKind::$kind); - } - input.set_position(save); - )* - }; - } - - match LexCtx::value() { - {%- for context_name, context in generator.scanner_contexts %} - LexicalContext::{{ context_name }} => { - if let Some(kind) = {{ context.literal_scanner }} { - furthest_position = input.position(); - longest_token = Some(kind); - } - input.set_position(save); - - longest_match! { - {%- for name in context.compound_scanner_names %} - {%- if name not in context.promotable_identifier_scanners %} - { {{name }} = {{ name | snake_case }} } - {%- endif -%} - {%- endfor %} - } - // Make sure promotable identifiers are last so they don't grab other things - longest_match! { - {%- for name in context.promotable_identifier_scanners %} - { {{ name }} = {{ name | snake_case }} } - {%- endfor %} - } - - // We have an identifier; we need to check if it's a keyword - if let Some(identifier) = longest_token.filter(|tok| - [ - {% for name in context.promotable_identifier_scanners %} - TokenKind::{{ name }}, - {% endfor %} - ] - .contains(tok) - ) { - let kw_scan = {{ context.keyword_trie_scanner }}; - let kw_scan = match kw_scan { - // Strict prefix; we need to match the whole identifier to promote - _ if input.position() < furthest_position => KeywordScan::Absent, - value => value, - }; - - {% if context.keyword_compound_scanners | length > 0 %} - // Perf: only scan for a compound keyword if we didn't already find one - let mut kw_scan = kw_scan; - if kw_scan == KeywordScan::Absent { - input.set_position(save); - - // TODO(#638): Don't allocate a string here - let ident_value = input.content(save.utf8..furthest_position.utf8); - - for keyword_compound_scanner in [ - {%- for keyword_name, _ in context.keyword_compound_scanners %} - Self::{{ keyword_name | snake_case }}, - {%- endfor %} - ] { - match keyword_compound_scanner(self, input, &ident_value) { - _ if input.position() < furthest_position => {/* Strict prefix */}, - KeywordScan::Absent => {}, - value => kw_scan = value, - } - input.set_position(save); - } - } - {% endif %} - - input.set_position(furthest_position); - return Some(ScannedToken::IdentifierOrKeyword { identifier, kw: kw_scan }); - } - }, - {%- endfor %} - } - - match longest_token { - Some(token) => { - input.set_position(furthest_position); - Some(ScannedToken::Single(token)) - }, - // Skip a character if possible and if we didn't recognize a token - None if input.peek().is_some() => { - let _ = input.next(); - Some(ScannedToken::Single(TokenKind::SKIPPED)) - }, - None => None, - } - } -} - -#[cfg(feature = "slang_napi_interfaces")] -// NAPI-exposed functions have to accept owned values. -#[allow(clippy::needless_pass_by_value)] -#[napi(namespace = "language")] -impl Language { - - #[napi(constructor, catch_unwind)] - pub fn new_napi(version: String) -> std::result::Result { - let version = Version::parse(&version).map_err(|_| Error::InvalidSemanticVersion(version))?; - Self::new(version).map_err(|e| e.into()) - } - - #[napi(getter, js_name = "version", catch_unwind)] - pub fn version_napi(&self) -> String { - self.version.to_string() - } - - #[napi(js_name = "supportedVersions", catch_unwind)] - pub fn supported_versions_napi() -> Vec { - return Self::SUPPORTED_VERSIONS.iter().map(|v| v.to_string()).collect(); - } - - #[napi(js_name = "parse", ts_return_type = "parse_output.ParseOutput", catch_unwind)] - pub fn parse_napi( - &self, - #[napi(ts_arg_type = "kinds.RuleKind")] kind: RuleKind, - input: String - ) -> NAPIParseOutput { - self.parse(kind, input.as_str()).into() - } - -} diff --git a/crates/codegen/parser/runtime/src/templates/user_defined_queries.rs.jinja2 b/crates/codegen/parser/runtime/src/templates/user_defined_queries.rs.jinja2 deleted file mode 100644 index fbf1bc9820..0000000000 --- a/crates/codegen/parser/runtime/src/templates/user_defined_queries.rs.jinja2 +++ /dev/null @@ -1,16 +0,0 @@ -use crate::cursor::Cursor; - -pub(crate) trait UserDefinedQueries { - {%- for query, captures in queries -%} - {# - # TODO(#554): not sure if this type is expected to be side-effect free implementation, - # or is it going to carry an incremental state. We can add other callbacks as needed. - #} - - fn {{ query }}( - {%- for capture in captures -%} - {{ capture }}: &Cursor, - {%- endfor -%} - ); - {%- endfor -%} -} diff --git a/crates/codegen/parser/runtime/Cargo.toml b/crates/codegen/runtime/cargo/Cargo.toml similarity index 74% rename from crates/codegen/parser/runtime/Cargo.toml rename to crates/codegen/runtime/cargo/Cargo.toml index 9fa7816a75..a22296ac93 100644 --- a/crates/codegen/parser/runtime/Cargo.toml +++ b/crates/codegen/runtime/cargo/Cargo.toml @@ -1,21 +1,27 @@ [package] -name = "codegen_parser_runtime" +name = "codegen_runtime_cargo" version.workspace = true rust-version.workspace = true edition.workspace = true publish = false -description = "Language-agnostic parser runtime copied over by codegen" +description = "Cargo runtime copied over by codegen" + +[build-dependencies] +anyhow = { workspace = true } +codegen_runtime_generator = { workspace = true } [dependencies] ariadne = { workspace = true } napi = { workspace = true, optional = true } napi-derive = { workspace = true, optional = true } nom = { workspace = true } +semver = { workspace = true } serde = { workspace = true } serde_json = { workspace = true, optional = true } strum = { workspace = true } strum_macros = { workspace = true } +thiserror = { workspace = true } # Since the source code is copied over as-is, we need to retain exact feature set with target crate, # i.e `slang_solidity_node_addon`. @@ -25,4 +31,3 @@ slang_napi_interfaces = ["dep:napi", "dep:napi-derive", "dep:serde_json"] [lints] workspace = true - diff --git a/crates/codegen/runtime/cargo/build.rs b/crates/codegen/runtime/cargo/build.rs new file mode 100644 index 0000000000..6ab086731c --- /dev/null +++ b/crates/codegen/runtime/cargo/build.rs @@ -0,0 +1,6 @@ +use anyhow::Result; +use codegen_runtime_generator::OutputLanguage; + +fn main() -> Result<()> { + OutputLanguage::Cargo.generate_stubs() +} diff --git a/crates/codegen/runtime/cargo/src/lib.rs b/crates/codegen/runtime/cargo/src/lib.rs new file mode 100644 index 0000000000..9bccc0fff3 --- /dev/null +++ b/crates/codegen/runtime/cargo/src/lib.rs @@ -0,0 +1,6 @@ +#![allow(dead_code)] + +mod runtime; +mod user_defined; + +pub use runtime::*; diff --git a/crates/codegen/parser/runtime/src/cst.rs b/crates/codegen/runtime/cargo/src/runtime/cst.rs similarity index 100% rename from crates/codegen/parser/runtime/src/cst.rs rename to crates/codegen/runtime/cargo/src/runtime/cst.rs diff --git a/crates/codegen/parser/runtime/src/cursor.rs b/crates/codegen/runtime/cargo/src/runtime/cursor.rs similarity index 100% rename from crates/codegen/parser/runtime/src/cursor.rs rename to crates/codegen/runtime/cargo/src/runtime/cursor.rs diff --git a/crates/codegen/parser/runtime/src/kinds.rs b/crates/codegen/runtime/cargo/src/runtime/generated/kinds.rs similarity index 67% rename from crates/codegen/parser/runtime/src/kinds.rs rename to crates/codegen/runtime/cargo/src/runtime/generated/kinds.rs index 519e38591a..9d4219b298 100644 --- a/crates/codegen/parser/runtime/src/kinds.rs +++ b/crates/codegen/runtime/cargo/src/runtime/generated/kinds.rs @@ -1,3 +1,5 @@ +// This file is generated automatically by infrastructure scripts. Please don't edit by hand. + #[cfg(feature = "slang_napi_interfaces")] use napi_derive::napi; @@ -13,21 +15,12 @@ use napi_derive::napi; strum_macros::Display, strum_macros::EnumString, )] -#[cfg_attr( feature = "slang_napi_interfaces", /* derives `Clone` and `Copy` */ napi(string_enum, namespace = "kinds") )] +#[cfg_attr(feature = "slang_napi_interfaces", /* derives `Clone` and `Copy` */ napi(string_enum, namespace = "kinds"))] #[cfg_attr(not(feature = "slang_napi_interfaces"), derive(Clone, Copy))] -pub enum TokenKind { - SKIPPED, - // Used for testing this crate, this is generated in the client code - Identifier, - Token1, - Token2, - Token3, -} - -impl TokenKind { - pub fn is_trivia(&self) -> bool { - unreachable!("Expanded by the template") - } +pub enum RuleKind { + Stub1, + Stub2, + Stub3, } #[derive( @@ -42,15 +35,24 @@ impl TokenKind { strum_macros::Display, strum_macros::EnumString, )] -#[cfg_attr( feature = "slang_napi_interfaces", /* derives `Clone` and `Copy` */ napi(string_enum, namespace = "kinds") )] +#[strum(serialize_all = "snake_case")] +#[cfg_attr(feature = "slang_napi_interfaces", /* derives `Clone` and `Copy` */ napi(string_enum, namespace = "kinds"))] #[cfg_attr(not(feature = "slang_napi_interfaces"), derive(Clone, Copy))] -pub enum RuleKind { +pub enum NodeLabel { + // Built-in: + Item, + Variant, + Separator, + Operand, + LeftOperand, + RightOperand, LeadingTrivia, TrailingTrivia, - // Used for testing this crate, this is generated in the client code - Rule1, - Rule2, - Rule3, + + // Generated: + Stub1, + Stub2, + Stub3, } #[derive( @@ -67,27 +69,28 @@ pub enum RuleKind { )] #[cfg_attr(feature = "slang_napi_interfaces", /* derives `Clone` and `Copy` */ napi(string_enum, namespace = "kinds"))] #[cfg_attr(not(feature = "slang_napi_interfaces"), derive(Clone, Copy))] -pub enum NodeLabel { - // Built-in labels - // _SLANG_INTERNAL_RESERVED_NODE_LABELS_ (keep in sync) - Item, - Variant, - Separator, - Operand, - LeftOperand, - RightOperand, - LeadingTrivia, - TrailingTrivia, - // Used for testing this crate, this is generated in the client code - Label1, - Label2, - Label3, +pub enum TokenKind { + // Built-in: + SKIPPED, + + // Generated: + Stub1, + Stub2, + Stub3, +} + +impl TokenKind { + pub fn is_trivia(&self) -> bool { + false + } } /// The lexical context of the scanner. #[derive(strum_macros::FromRepr, Clone, Copy)] pub(crate) enum LexicalContext { - // Expanded by the template engine + Stub1, + Stub2, + Stub3, } /// Marker trait for type-level [`LexicalContext`] variants. diff --git a/crates/codegen/runtime/cargo/src/runtime/generated/language.rs b/crates/codegen/runtime/cargo/src/runtime/generated/language.rs new file mode 100644 index 0000000000..ff12d1b9c8 --- /dev/null +++ b/crates/codegen/runtime/cargo/src/runtime/generated/language.rs @@ -0,0 +1,131 @@ +// This file is generated automatically by infrastructure scripts. Please don't edit by hand. + +// This file is generated; we can't reasonably satisfy some of these lints. +#![allow( + clippy::if_not_else, + clippy::too_many_lines, + clippy::unused_self, + clippy::struct_excessive_bools, + clippy::similar_names, + unused_imports +)] + +#[cfg(feature = "slang_napi_interfaces")] +use napi_derive::napi; +use semver::Version; + +use crate::cst; +use crate::kinds::{ + IsLexicalContext, LexicalContext, LexicalContextType, NodeLabel, RuleKind, TokenKind, +}; +use crate::lexer::{KeywordScan, Lexer, ScannedToken}; +#[cfg(feature = "slang_napi_interfaces")] +use crate::napi_interface::parse_output::ParseOutput as NAPIParseOutput; +use crate::parse_output::ParseOutput; +use crate::parser_support::{ + ChoiceHelper, OneOrMoreHelper, OptionalHelper, ParserContext, ParserFunction, ParserResult, + PrecedenceHelper, SeparatedHelper, SequenceHelper, TokenAcceptanceThreshold, ZeroOrMoreHelper, +}; + +#[derive(Debug)] +#[cfg_attr(feature = "slang_napi_interfaces", napi(namespace = "language"))] +pub struct Language { + pub(crate) version: Version, +} + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("Unsupported language version '{0}'.")] + UnsupportedLanguageVersion(Version), + + #[cfg(feature = "slang_napi_interfaces")] + #[error("Invalid semantic version '{0}'.")] + InvalidSemanticVersion(String), +} + +#[cfg(feature = "slang_napi_interfaces")] +impl From for napi::Error { + fn from(value: Error) -> Self { + napi::Error::from_reason(value.to_string()) + } +} + +impl Language { + pub const SUPPORTED_VERSIONS: &'static [Version] = &[]; + + pub fn new(version: Version) -> std::result::Result { + if Self::SUPPORTED_VERSIONS.binary_search(&version).is_ok() { + Ok(Self { version }) + } else { + Err(Error::UnsupportedLanguageVersion(version)) + } + } + + pub fn version(&self) -> &Version { + &self.version + } + + pub fn parse(&self, kind: RuleKind, input: &str) -> ParseOutput { + unreachable!("Attempting to parse in stubs: {kind}: {input}") + } +} + +impl Lexer for Language { + fn leading_trivia(&self, input: &mut ParserContext<'_>) -> ParserResult { + unreachable!("Invoking leading_trivia in stubs: {input:#?}") + } + + fn trailing_trivia(&self, input: &mut ParserContext<'_>) -> ParserResult { + unreachable!("Invoking trailing_trivia in stubs: {input:#?}") + } + + fn delimiters() -> &'static [(TokenKind, TokenKind)] { + unreachable!("Invoking delimiters in stubs.") + } + + fn next_token( + &self, + input: &mut ParserContext<'_>, + ) -> Option { + unreachable!("Invoking next_token in stubs: {input:#?}") + } +} + +#[cfg(feature = "slang_napi_interfaces")] +// NAPI-exposed functions have to accept owned values. +#[allow(clippy::needless_pass_by_value)] +#[napi(namespace = "language")] +impl Language { + #[napi(constructor, catch_unwind)] + pub fn new_napi(version: String) -> std::result::Result { + let version = + Version::parse(&version).map_err(|_| Error::InvalidSemanticVersion(version))?; + Self::new(version).map_err(|e| e.into()) + } + + #[napi(getter, js_name = "version", catch_unwind)] + pub fn version_napi(&self) -> String { + self.version.to_string() + } + + #[napi(js_name = "supportedVersions", catch_unwind)] + pub fn supported_versions_napi() -> Vec { + return Self::SUPPORTED_VERSIONS + .iter() + .map(|v| v.to_string()) + .collect(); + } + + #[napi( + js_name = "parse", + ts_return_type = "parse_output.ParseOutput", + catch_unwind + )] + pub fn parse_napi( + &self, + #[napi(ts_arg_type = "kinds.RuleKind")] kind: RuleKind, + input: String, + ) -> NAPIParseOutput { + self.parse(kind, input.as_str()).into() + } +} diff --git a/crates/codegen/parser/runtime/src/templates/kinds.rs.jinja2 b/crates/codegen/runtime/cargo/src/runtime/kinds.rs.jinja2 similarity index 53% rename from crates/codegen/parser/runtime/src/templates/kinds.rs.jinja2 rename to crates/codegen/runtime/cargo/src/runtime/kinds.rs.jinja2 index be0fe421b2..c89e627052 100644 --- a/crates/codegen/parser/runtime/src/templates/kinds.rs.jinja2 +++ b/crates/codegen/runtime/cargo/src/runtime/kinds.rs.jinja2 @@ -16,10 +16,16 @@ use napi_derive::napi; #[cfg_attr(feature = "slang_napi_interfaces", /* derives `Clone` and `Copy` */ napi(string_enum, namespace = "kinds"))] #[cfg_attr(not(feature = "slang_napi_interfaces"), derive(Clone, Copy))] pub enum RuleKind { - {%- for variant in generator.rule_kinds -%} - {# variant.documentation | indent(prefix = "/// ", first = true, blank = true) #} - {{ variant }}, - {%- endfor -%} + {%- if rendering_in_stubs -%} + Stub1, + Stub2, + Stub3, + {%- else -%} + {%- for variant in model.rule_kinds -%} + {# variant.documentation | indent(prefix = "/// ", first = true, blank = true) #} + {{ variant }}, + {%- endfor -%} + {%- endif -%} } #[derive( @@ -38,7 +44,7 @@ pub enum RuleKind { #[cfg_attr(feature = "slang_napi_interfaces", /* derives `Clone` and `Copy` */ napi(string_enum, namespace = "kinds"))] #[cfg_attr(not(feature = "slang_napi_interfaces"), derive(Clone, Copy))] pub enum NodeLabel { - // Built-in labels + // Built-in: {# _SLANG_INTERNAL_RESERVED_NODE_LABELS_ (keep in sync) #} Item, Variant, @@ -48,10 +54,17 @@ pub enum NodeLabel { RightOperand, LeadingTrivia, TrailingTrivia, - // Generated - {% for variant in generator.labels -%} - {{ variant | pascal_case }}, - {%- endfor -%} + + // Generated: + {% if rendering_in_stubs -%} + Stub1, + Stub2, + Stub3, + {%- else -%} + {% for variant in model.labels -%} + {{ variant | pascal_case }}, + {%- endfor -%} + {%- endif -%} } #[derive( @@ -69,31 +82,49 @@ pub enum NodeLabel { #[cfg_attr(feature = "slang_napi_interfaces", /* derives `Clone` and `Copy` */ napi(string_enum, namespace = "kinds"))] #[cfg_attr(not(feature = "slang_napi_interfaces"), derive(Clone, Copy))] pub enum TokenKind { + // Built-in: SKIPPED, - {%- for variant in generator.token_kinds -%} - {# variant.documentation | indent(prefix = "/// ", first = true, blank = true) #} - {{ variant }}, - {%- endfor -%} + + // Generated: + {% if rendering_in_stubs -%} + Stub1, + Stub2, + Stub3, + {%- else -%} + {%- for variant in model.token_kinds -%} + {# variant.documentation | indent(prefix = "/// ", first = true, blank = true) #} + {{ variant }}, + {%- endfor -%} + {%- endif -%} } impl TokenKind { pub fn is_trivia(&self) -> bool { - #[allow(clippy::match_like_matches_macro)] - match self { - {%- for variant in generator.trivia_scanner_names -%} - Self::{{ variant }} => true, + {%- if rendering_in_stubs -%} + false + {%- else -%} + matches!( + self, + {%- for variant in model.trivia_scanner_names -%} + | Self::{{ variant }} {%- endfor -%} - _ => false, - } + ) + {%- endif -%} } } /// The lexical context of the scanner. #[derive(strum_macros::FromRepr, Clone, Copy)] pub(crate) enum LexicalContext { - {%- for context_name, _context in generator.scanner_contexts %} - {{ context_name }}, - {%- endfor %} + {%- if rendering_in_stubs -%} + Stub1, + Stub2, + Stub3, + {%- else -%} + {%- for context_name, _ in model.scanner_contexts %} + {{ context_name }}, + {%- endfor %} + {%- endif -%} } /// Marker trait for type-level [`LexicalContext`] variants. @@ -104,14 +135,15 @@ pub(crate) trait IsLexicalContext { #[allow(non_snake_case)] pub(crate) mod LexicalContextType { - use super::{IsLexicalContext, LexicalContext}; + {%- if not rendering_in_stubs -%} + {%- for context_name, _ in model.scanner_contexts %} + pub struct {{ context_name }}; - {%- for context_name, _ in generator.scanner_contexts %} - pub struct {{ context_name }} {} - impl IsLexicalContext for {{ context_name }} { - fn value() -> LexicalContext { - LexicalContext::{{ context_name }} + impl super::IsLexicalContext for {{ context_name }} { + fn value() -> super::LexicalContext { + super::LexicalContext::{{ context_name }} + } } - } - {%- endfor %} + {%- endfor %} + {%- endif -%} } diff --git a/crates/codegen/runtime/cargo/src/runtime/language.rs.jinja2 b/crates/codegen/runtime/cargo/src/runtime/language.rs.jinja2 new file mode 100644 index 0000000000..ef0248d1ab --- /dev/null +++ b/crates/codegen/runtime/cargo/src/runtime/language.rs.jinja2 @@ -0,0 +1,303 @@ +// This file is generated; we can't reasonably satisfy some of these lints. +#![allow( + clippy::if_not_else, + clippy::too_many_lines, + clippy::unused_self, + clippy::struct_excessive_bools, + clippy::similar_names, + unused_imports +)] + +use semver::Version; +#[cfg(feature = "slang_napi_interfaces")] +use napi_derive::napi; + +use crate::cst; +use crate::kinds::{ + NodeLabel, IsLexicalContext, LexicalContext, LexicalContextType, RuleKind, TokenKind, +}; +use crate::lexer::{KeywordScan, Lexer, ScannedToken}; +#[cfg(feature = "slang_napi_interfaces")] +use crate::napi_interface::parse_output::ParseOutput as NAPIParseOutput; +use crate::parse_output::ParseOutput; +use crate::parser_support::{ + ChoiceHelper, OneOrMoreHelper, OptionalHelper, ParserContext, ParserFunction, ParserResult, + PrecedenceHelper, SeparatedHelper, SequenceHelper, TokenAcceptanceThreshold, ZeroOrMoreHelper, +}; + +#[derive(Debug)] +#[cfg_attr(feature = "slang_napi_interfaces", napi(namespace = "language"))] +pub struct Language { + {%- if not rendering_in_stubs -%} + {%- for version in model.referenced_versions -%} + pub(crate) version_is_at_least_{{ version | replace(from=".", to="_") }}: bool, + {%- endfor -%} + {%- endif -%} + + pub(crate) version: Version, +} + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("Unsupported language version '{0}'.")] + UnsupportedLanguageVersion(Version), + + #[cfg(feature = "slang_napi_interfaces")] + #[error("Invalid semantic version '{0}'.")] + InvalidSemanticVersion(String), +} + +#[cfg(feature = "slang_napi_interfaces")] +impl From for napi::Error { + fn from(value: Error) -> Self { + napi::Error::from_reason(value.to_string()) + } +} + +impl Language { + pub const SUPPORTED_VERSIONS: &'static [Version] = &[ + {%- if not rendering_in_stubs -%} + {% for version in model.all_versions %} + Version::new({{ version | split(pat=".") | join(sep=", ") }}), + {% endfor %} + {%- endif -%} + ]; + + pub fn new(version: Version) -> std::result::Result { + if Self::SUPPORTED_VERSIONS.binary_search(&version).is_ok() { + Ok(Self { + {%- if not rendering_in_stubs -%} + {%- for version in model.referenced_versions %} + version_is_at_least_{{ version | replace(from=".", to="_") }}: Version::new({{ version | split(pat=".") | join(sep=", ") }}) <= version, + {%- endfor -%} + {%- endif -%} + + version, + }) + } else { + Err(Error::UnsupportedLanguageVersion(version)) + } + } + + pub fn version(&self) -> &Version { + &self.version + } + + {%- if not rendering_in_stubs -%} + + /******************************************** + * Parser Functions + ********************************************/ + + {% for parser_name, parser_code in model.parser_functions %} + #[allow(unused_assignments, unused_parens)] + fn {{ parser_name | snake_case }}(&self, input: &mut ParserContext<'_>) -> ParserResult { {{ parser_code }} } + {% endfor %} + + {% for parser_name, parser_code in model.trivia_parser_functions %} + #[allow(unused_assignments, unused_parens)] + fn {{ parser_name | snake_case }}(&self, input: &mut ParserContext<'_>) -> ParserResult { {{ parser_code }} } + {% endfor %} + + /******************************************** + * Scanner Functions + ********************************************/ + + {% for scanner_name, scanner_code in model.scanner_functions %} + #[allow(unused_assignments, unused_parens)] + fn {{ scanner_name | snake_case }}(&self, input: &mut ParserContext<'_>) -> bool { {{ scanner_code }} } + {% endfor %} + + {%- for keyword_name, keyword_code in model.keyword_compound_scanners %} + #[inline] + fn {{ keyword_name | snake_case }}(&self, input: &mut ParserContext<'_>, ident: &str) -> KeywordScan { {{ keyword_code }} } + {% endfor %} + + {% endif %} + + pub fn parse(&self, kind: RuleKind, input: &str) -> ParseOutput { + {%- if rendering_in_stubs -%} + unreachable!("Attempting to parse in stubs: {kind}: {input}") + {%- else -%} + match kind { + {%- for parser_name, _ in model.parser_functions -%} + RuleKind::{{ parser_name }} => Self::{{ parser_name | snake_case }}.parse(self, input), + {%- endfor -%} + } + {%- endif -%} + } +} + +impl Lexer for Language { + fn leading_trivia(&self, input: &mut ParserContext<'_>) -> ParserResult { + {%- if rendering_in_stubs -%} + unreachable!("Invoking leading_trivia in stubs: {input:#?}") + {%- else -%} + Language::leading_trivia(self, input) + {%- endif -%} + } + + fn trailing_trivia(&self, input: &mut ParserContext<'_>) -> ParserResult { + {%- if rendering_in_stubs -%} + unreachable!("Invoking trailing_trivia in stubs: {input:#?}") + {%- else -%} + Language::trailing_trivia(self, input) + {%- endif -%} + } + + fn delimiters() -> &'static [(TokenKind, TokenKind)] { + {%- if rendering_in_stubs -%} + unreachable!("Invoking delimiters in stubs.") + {%- else -%} + match LexCtx::value() { + {%- for context_name, context in model.scanner_contexts %} + LexicalContext::{{ context_name }} => &[ + {%- for open, close in context.delimiters %} + (TokenKind::{{ open }}, TokenKind::{{ close }}), + {%- endfor %} + ], + {%- endfor %} + } + {%- endif -%} + } + + fn next_token(&self, input: &mut ParserContext<'_>) -> Option { + {%- if rendering_in_stubs -%} + unreachable!("Invoking next_token in stubs: {input:#?}") + {%- else -%} + let save = input.position(); + let mut furthest_position = input.position(); + let mut longest_token = None; + + macro_rules! longest_match { + ($( { $kind:ident = $function:ident } )*) => { + $( + if self.$function(input) && input.position() > furthest_position { + furthest_position = input.position(); + + longest_token = Some(TokenKind::$kind); + } + input.set_position(save); + )* + }; + } + + match LexCtx::value() { + {%- for context_name, context in model.scanner_contexts %} + LexicalContext::{{ context_name }} => { + if let Some(kind) = {{ context.literal_scanner }} { + furthest_position = input.position(); + longest_token = Some(kind); + } + input.set_position(save); + + longest_match! { + {%- for name in context.compound_scanner_names %} + {%- if name not in context.promotable_identifier_scanners %} + { {{name }} = {{ name | snake_case }} } + {%- endif -%} + {%- endfor %} + } + // Make sure promotable identifiers are last so they don't grab other things + longest_match! { + {%- for name in context.promotable_identifier_scanners %} + { {{ name }} = {{ name | snake_case }} } + {%- endfor %} + } + + // We have an identifier; we need to check if it's a keyword + if let Some(identifier) = longest_token.filter(|tok| + [ + {% for name in context.promotable_identifier_scanners %} + TokenKind::{{ name }}, + {% endfor %} + ] + .contains(tok) + ) { + let kw_scan = {{ context.keyword_trie_scanner }}; + let kw_scan = match kw_scan { + // Strict prefix; we need to match the whole identifier to promote + _ if input.position() < furthest_position => KeywordScan::Absent, + value => value, + }; + + {% if context.keyword_compound_scanners | length > 0 %} + // Perf: only scan for a compound keyword if we didn't already find one + let mut kw_scan = kw_scan; + if kw_scan == KeywordScan::Absent { + input.set_position(save); + + // TODO(#638): Don't allocate a string here + let ident_value = input.content(save.utf8..furthest_position.utf8); + + for keyword_compound_scanner in [ + {%- for keyword_name, _ in context.keyword_compound_scanners %} + Self::{{ keyword_name | snake_case }}, + {%- endfor %} + ] { + match keyword_compound_scanner(self, input, &ident_value) { + _ if input.position() < furthest_position => {/* Strict prefix */}, + KeywordScan::Absent => {}, + value => kw_scan = value, + } + input.set_position(save); + } + } + {% endif %} + + input.set_position(furthest_position); + return Some(ScannedToken::IdentifierOrKeyword { identifier, kw: kw_scan }); + } + }, + {%- endfor %} + } + + match longest_token { + Some(token) => { + input.set_position(furthest_position); + Some(ScannedToken::Single(token)) + }, + // Skip a character if possible and if we didn't recognize a token + None if input.peek().is_some() => { + let _ = input.next(); + Some(ScannedToken::Single(TokenKind::SKIPPED)) + }, + None => None, + } + + {%- endif -%} + } +} + +#[cfg(feature = "slang_napi_interfaces")] +// NAPI-exposed functions have to accept owned values. +#[allow(clippy::needless_pass_by_value)] +#[napi(namespace = "language")] +impl Language { + + #[napi(constructor, catch_unwind)] + pub fn new_napi(version: String) -> std::result::Result { + let version = Version::parse(&version).map_err(|_| Error::InvalidSemanticVersion(version))?; + Self::new(version).map_err(|e| e.into()) + } + + #[napi(getter, js_name = "version", catch_unwind)] + pub fn version_napi(&self) -> String { + self.version.to_string() + } + + #[napi(js_name = "supportedVersions", catch_unwind)] + pub fn supported_versions_napi() -> Vec { + return Self::SUPPORTED_VERSIONS.iter().map(|v| v.to_string()).collect(); + } + + #[napi(js_name = "parse", ts_return_type = "parse_output.ParseOutput", catch_unwind)] + pub fn parse_napi( + &self, + #[napi(ts_arg_type = "kinds.RuleKind")] kind: RuleKind, + input: String + ) -> NAPIParseOutput { + self.parse(kind, input.as_str()).into() + } + +} diff --git a/crates/codegen/parser/runtime/src/lexer.rs b/crates/codegen/runtime/cargo/src/runtime/lexer.rs similarity index 100% rename from crates/codegen/parser/runtime/src/lexer.rs rename to crates/codegen/runtime/cargo/src/runtime/lexer.rs diff --git a/crates/codegen/parser/runtime/src/mod_for_destination.rs b/crates/codegen/runtime/cargo/src/runtime/mod.rs similarity index 71% rename from crates/codegen/parser/runtime/src/mod_for_destination.rs rename to crates/codegen/runtime/cargo/src/runtime/mod.rs index a2b258a41c..eab63a3f88 100644 --- a/crates/codegen/parser/runtime/src/mod_for_destination.rs +++ b/crates/codegen/runtime/cargo/src/runtime/mod.rs @@ -1,15 +1,18 @@ #[macro_use] -pub mod parser_support; +pub(crate) mod parser_support; +pub(crate) mod lexer; pub mod cst; pub mod cursor; -pub mod kinds; -pub mod language; -pub(crate) mod lexer; pub mod parse_error; pub mod parse_output; pub mod query; pub mod text_index; +#[path = "generated/kinds.rs"] +pub mod kinds; +#[path = "generated/language.rs"] +pub mod language; + #[cfg(feature = "slang_napi_interfaces")] pub mod napi_interface; diff --git a/crates/codegen/runtime/cargo/src/runtime/napi_interface/ast_selectors.rs.jinja2 b/crates/codegen/runtime/cargo/src/runtime/napi_interface/ast_selectors.rs.jinja2 new file mode 100644 index 0000000000..4add655bdf --- /dev/null +++ b/crates/codegen/runtime/cargo/src/runtime/napi_interface/ast_selectors.rs.jinja2 @@ -0,0 +1,356 @@ +#![allow(clippy::too_many_lines)] + +use std::rc::Rc; + +use napi::Either; +use napi_derive::napi; + +use crate::napi_interface::cst::{RuleNode, TokenNode}; +use crate::napi_interface::{RuleKind, RustLabeledNode, RustNode, RustRuleNode, TokenKind}; + +// +// Sequences: +// + +#[napi( + namespace = "ast_internal", + ts_return_type = "Array", + catch_unwind, +)] +pub fn select_sequence( + #[napi(ts_arg_type = "cst.RuleNode")] node: &RuleNode, +) -> Result>>> { + {%- if rendering_in_stubs -%} + unreachable!("Invoking AST selectors in stubs: {node:#?}") + {%- else -%} + let mut selector = Selector::new(node); + + let result = match node.kind() { + {%- for sequence in model.ast.sequences -%} + RuleKind::{{ sequence.name }} => { + selector.{{ sequence.name | snake_case }}()? + }, + {%- endfor -%} + _ => { + return Error::UnexpectedParent(node.kind()).into(); + } + }; + + selector.finalize()?; + Ok(result) + {%- endif -%} +} + +{%- if not rendering_in_stubs -%} + {% for sequence in model.ast.sequences %} + impl Selector { + fn {{ sequence.name | snake_case }}(&mut self) -> Result>>> { + Ok(vec![ + {%- for field in sequence.fields -%} + {%- if field.is_optional -%} + + self.try_select(|node| { + {%- if field.is_terminal -%} + node.is_token_with_kind(TokenKind::{{ field.reference }}) + {%- else -%} + node.is_rule_with_kind(RuleKind::{{ field.reference }}) + {%- endif -%} + })?, + + {%- else -%} + + Some(self.select(|node| { + {%- if field.is_terminal -%} + node.is_token_with_kind(TokenKind::{{ field.reference }}) + {%- else -%} + node.is_rule_with_kind(RuleKind::{{ field.reference }}) + {%- endif -%} + })?), + + {%- endif -%} + {%- endfor -%} + ]) + } + } + {% endfor %} +{%- endif -%} + +// +// Choices: +// + +#[napi( + namespace = "ast_internal", + ts_return_type = "cst.Node", + catch_unwind, +)] +pub fn select_choice( + #[napi(ts_arg_type = "cst.RuleNode")] node: &RuleNode, +) -> Result> { + {%- if rendering_in_stubs -%} + unreachable!("Invoking AST selectors in stubs: {node:#?}") + {%- else -%} + let mut selector = Selector::new(node); + + let result = match node.kind() { + {%- for choice in model.ast.choices -%} + RuleKind::{{ choice.name }} => { + selector.{{ choice.name | snake_case }}()? + }, + {%- endfor -%} + _ => { + return Error::UnexpectedParent(node.kind()).into(); + } + }; + + selector.finalize()?; + Ok(result) + {%- endif -%} +} + +{% if not rendering_in_stubs %} + {% for choice in model.ast.choices %} + impl Selector { + fn {{ choice.name | snake_case }}(&mut self) -> Result> { + self.select(|node| { + {%- set non_terminals_len = choice.non_terminals | length -%} + {%- set terminals_len = choice.terminals | length -%} + + {%- if non_terminals_len == 1 -%} + node.is_rule_with_kind(RuleKind::{{ choice.non_terminals[0] }}) + {%- elif non_terminals_len > 1 -%} + node.is_rule_with_kinds(&[ + {%- for non_terminal in choice.non_terminals -%} + RuleKind::{{ non_terminal }}, + {%- endfor -%} + ]) + {%- endif -%} + + {%- if non_terminals_len > 0 and terminals_len > 0 -%} + || + {%- endif -%} + + {%- if terminals_len == 1 -%} + node.is_token_with_kind(TokenKind::{{ choice.terminals[0] }}) + {%- elif terminals_len > 1 -%} + node.is_token_with_kinds(&[ + {%- for terminal in choice.terminals -%} + TokenKind::{{ terminal }}, + {%- endfor -%} + ]) + {%- endif -%} + }) + } + } + {% endfor %} +{% endif %} + +// +// Repeated: +// + +#[napi( + namespace = "ast_internal", + ts_return_type = "Array", + catch_unwind, +)] +pub fn select_repeated( + #[napi(ts_arg_type = "cst.RuleNode")] node: &RuleNode, +) -> Result>> { + {%- if rendering_in_stubs -%} + unreachable!("Invoking AST selectors in stubs: {node:#?}") + {%- else -%} + let mut selector = Selector::new(node); + + let result = match node.kind() { + {%- for repeated in model.ast.repeated -%} + RuleKind::{{ repeated.name }} => { + selector.{{ repeated.name | snake_case }}()? + }, + {%- endfor -%} + _ => { + return Error::UnexpectedParent(node.kind()).into(); + } + }; + + selector.finalize()?; + Ok(result) + {%- endif -%} +} + +{% if not rendering_in_stubs %} + {% for repeated in model.ast.repeated %} + impl Selector { + fn {{ repeated.name | snake_case }}(&mut self) -> Result>> { + let mut items = vec![]; + + while let Some(item) = self.try_select(|node| { + {%- if repeated.is_terminal -%} + node.is_token_with_kind(TokenKind::{{ repeated.reference }}) + {%- else -%} + node.is_rule_with_kind(RuleKind::{{ repeated.reference }}) + {%- endif -%} + })? { + items.push(item); + } + + Ok(items) + } + } + {% endfor %} +{% endif %} + +// +// Separated: +// + +#[napi( + namespace = "ast_internal", + ts_return_type = "[Array, Array]", + catch_unwind, +)] +pub fn select_separated( + #[napi(ts_arg_type = "cst.RuleNode")] node: &RuleNode, +) -> Result>>> { + {%- if rendering_in_stubs -%} + unreachable!("Invoking AST selectors in stubs: {node:#?}") + {%- else -%} + let mut selector = Selector::new(node); + + let result = match node.kind() { + {%- for separated in model.ast.separated -%} + RuleKind::{{ separated.name }} => { + selector.{{ separated.name | snake_case }}()? + }, + {%- endfor -%} + _ => { + return Error::UnexpectedParent(node.kind()).into(); + } + }; + + selector.finalize()?; + Ok(result) + {%- endif -%} +} + +{% if not rendering_in_stubs %} + {% for separated in model.ast.separated %} + impl Selector { + fn {{ separated.name | snake_case }}(&mut self) -> Result>>> { + let mut separated = vec![]; + let mut separators = vec![]; + + if let Some(first) = self.try_select(|node| { + {%- if separated.is_terminal -%} + node.is_token_with_kind(TokenKind::{{ separated.reference }}) + {%- else -%} + node.is_rule_with_kind(RuleKind::{{ separated.reference }}) + {%- endif -%} + })? { + separated.push(first); + + while let Some(separator) = self.try_select(|node| node.is_token_with_kind(TokenKind::{{ separated.separator }}))? { + separators.push(separator); + + separated.push(self.select(|node| { + {%- if separated.is_terminal -%} + node.is_token_with_kind(TokenKind::{{ separated.reference }}) + {%- else -%} + node.is_rule_with_kind(RuleKind::{{ separated.reference }}) + {%- endif -%} + })?); + } + } + + Ok(vec![separated, separators]) + } + } + {% endfor %} +{% endif %} + +// +// Common: +// + +struct Selector { + node: Rc, + index: usize, +} + +impl Selector { + fn new(node: &RuleNode) -> Self { + Self { + node: Rc::clone(&node.0), + index: 0, + } + } + + fn select(&mut self, filter: impl FnOnce(&RustNode) -> bool) -> Result> { + match self.try_select(filter)? { + Some(node) => Ok(node), + None => Error::MissingChild(self.index).into(), + } + } + + fn try_select(&mut self, filter: impl FnOnce(&RustNode) -> bool) -> Result>> { + while let Some(child) = self.node.children.get(self.index) { + match child { + node if node.is_trivia() => { + // skip trivia, since it's not part of the AST + self.index += 1; + continue; + } + RustLabeledNode { + label: _, + node: RustNode::Token(token), + } if matches!(token.kind, TokenKind::SKIPPED) => { + return Error::SkippedToken(self.index).into(); + } + labeled if filter(labeled) => { + self.index += 1; + return Ok(Some(labeled.node.clone().into_js_either_node())); + } + _ => { + break; + }, + } + } + + Ok(None) + } + + fn finalize(mut self) -> Result<()> { + if self.try_select(|_| true)?.is_some() { + return Error::UnexpectedTrailing(self.index - 1).into(); + } + + Ok(()) + } +} + +type Result = std::result::Result; + +#[derive(Debug, thiserror::Error)] +enum Error { + // Should not theoretically happen, since we're only called from our own generated AST types. + #[error("Unexpected parent node with RuleKind '{0}'.")] + UnexpectedParent(RuleKind), + + // Should not theoretically happen, since we're only called from our own generated AST types. + #[error("Unexpected trailing children at index '{0}'.")] + UnexpectedTrailing(usize), + + // Should not theoretically happen, unless AST error recovery was changed. + #[error("Missing child node at index '{0}'.")] + MissingChild(usize), + + // Can happen if the user decided to use an incorrect/incomplete CST node. + #[error("Unexpected SKIPPED token at index '{0}'. Creating AST types from incorrect/incomplete CST nodes is not supported yet.")] + SkippedToken(usize), +} + +impl From for Result { + fn from(error: Error) -> Self { + Err(napi::Error::from_reason(error.to_string())) + } +} diff --git a/crates/codegen/parser/runtime/src/napi_interface/cst.rs b/crates/codegen/runtime/cargo/src/runtime/napi_interface/cst.rs similarity index 99% rename from crates/codegen/parser/runtime/src/napi_interface/cst.rs rename to crates/codegen/runtime/cargo/src/runtime/napi_interface/cst.rs index 97ff8cc421..7f1e75d315 100644 --- a/crates/codegen/parser/runtime/src/napi_interface/cst.rs +++ b/crates/codegen/runtime/cargo/src/runtime/napi_interface/cst.rs @@ -31,9 +31,11 @@ impl From for Either { } } +#[derive(Debug)] #[napi(namespace = "cst")] pub struct RuleNode(pub(crate) Rc); +#[derive(Debug)] #[napi(namespace = "cst")] pub struct TokenNode(pub(crate) Rc); diff --git a/crates/codegen/parser/runtime/src/napi_interface/cursor.rs b/crates/codegen/runtime/cargo/src/runtime/napi_interface/cursor.rs similarity index 100% rename from crates/codegen/parser/runtime/src/napi_interface/cursor.rs rename to crates/codegen/runtime/cargo/src/runtime/napi_interface/cursor.rs diff --git a/crates/codegen/runtime/cargo/src/runtime/napi_interface/generated/ast_selectors.rs b/crates/codegen/runtime/cargo/src/runtime/napi_interface/generated/ast_selectors.rs new file mode 100644 index 0000000000..0d49c77389 --- /dev/null +++ b/crates/codegen/runtime/cargo/src/runtime/napi_interface/generated/ast_selectors.rs @@ -0,0 +1,158 @@ +// This file is generated automatically by infrastructure scripts. Please don't edit by hand. + +#![allow(clippy::too_many_lines)] + +use std::rc::Rc; + +use napi::Either; +use napi_derive::napi; + +use crate::napi_interface::cst::{RuleNode, TokenNode}; +use crate::napi_interface::{RuleKind, RustLabeledNode, RustNode, RustRuleNode, TokenKind}; + +// +// Sequences: +// + +#[napi( + namespace = "ast_internal", + ts_return_type = "Array", + catch_unwind +)] +pub fn select_sequence( + #[napi(ts_arg_type = "cst.RuleNode")] node: &RuleNode, +) -> Result>>> { + unreachable!("Invoking AST selectors in stubs: {node:#?}") +} // + // Choices: + // + +#[napi(namespace = "ast_internal", ts_return_type = "cst.Node", catch_unwind)] +pub fn select_choice( + #[napi(ts_arg_type = "cst.RuleNode")] node: &RuleNode, +) -> Result> { + unreachable!("Invoking AST selectors in stubs: {node:#?}") +} + +// +// Repeated: +// + +#[napi( + namespace = "ast_internal", + ts_return_type = "Array", + catch_unwind +)] +pub fn select_repeated( + #[napi(ts_arg_type = "cst.RuleNode")] node: &RuleNode, +) -> Result>> { + unreachable!("Invoking AST selectors in stubs: {node:#?}") +} + +// +// Separated: +// + +#[napi( + namespace = "ast_internal", + ts_return_type = "[Array, Array]", + catch_unwind +)] +pub fn select_separated( + #[napi(ts_arg_type = "cst.RuleNode")] node: &RuleNode, +) -> Result>>> { + unreachable!("Invoking AST selectors in stubs: {node:#?}") +} + +// +// Common: +// + +struct Selector { + node: Rc, + index: usize, +} + +impl Selector { + fn new(node: &RuleNode) -> Self { + Self { + node: Rc::clone(&node.0), + index: 0, + } + } + + fn select( + &mut self, + filter: impl FnOnce(&RustNode) -> bool, + ) -> Result> { + match self.try_select(filter)? { + Some(node) => Ok(node), + None => Error::MissingChild(self.index).into(), + } + } + + fn try_select( + &mut self, + filter: impl FnOnce(&RustNode) -> bool, + ) -> Result>> { + while let Some(child) = self.node.children.get(self.index) { + match child { + node if node.is_trivia() => { + // skip trivia, since it's not part of the AST + self.index += 1; + continue; + } + RustLabeledNode { + label: _, + node: RustNode::Token(token), + } if matches!(token.kind, TokenKind::SKIPPED) => { + return Error::SkippedToken(self.index).into(); + } + labeled if filter(labeled) => { + self.index += 1; + return Ok(Some(labeled.node.clone().into_js_either_node())); + } + _ => { + break; + } + } + } + + Ok(None) + } + + fn finalize(mut self) -> Result<()> { + if self.try_select(|_| true)?.is_some() { + return Error::UnexpectedTrailing(self.index - 1).into(); + } + + Ok(()) + } +} + +type Result = std::result::Result; + +#[derive(Debug, thiserror::Error)] +enum Error { + // Should not theoretically happen, since we're only called from our own generated AST types. + #[error("Unexpected parent node with RuleKind '{0}'.")] + UnexpectedParent(RuleKind), + + // Should not theoretically happen, since we're only called from our own generated AST types. + #[error("Unexpected trailing children at index '{0}'.")] + UnexpectedTrailing(usize), + + // Should not theoretically happen, unless AST error recovery was changed. + #[error("Missing child node at index '{0}'.")] + MissingChild(usize), + + // Can happen if the user decided to use an incorrect/incomplete CST node. + #[error("Unexpected SKIPPED token at index '{0}'. Creating AST types from incorrect/incomplete CST nodes is not supported yet.")] + SkippedToken(usize), +} + +impl From for Result { + fn from(error: Error) -> Self { + Err(napi::Error::from_reason(error.to_string())) + } +} diff --git a/crates/codegen/parser/runtime/src/napi_interface/mod.rs b/crates/codegen/runtime/cargo/src/runtime/napi_interface/mod.rs similarity index 95% rename from crates/codegen/parser/runtime/src/napi_interface/mod.rs rename to crates/codegen/runtime/cargo/src/runtime/napi_interface/mod.rs index 2a494b9869..2c6d569d03 100644 --- a/crates/codegen/parser/runtime/src/napi_interface/mod.rs +++ b/crates/codegen/runtime/cargo/src/runtime/napi_interface/mod.rs @@ -1,4 +1,3 @@ -pub mod ast_selectors; pub mod cst; pub mod cursor; pub mod parse_error; @@ -6,6 +5,9 @@ pub mod parse_output; pub mod query; pub mod text_index; +#[path = "generated/ast_selectors.rs"] +pub mod ast_selectors; + type RustCursor = crate::cursor::Cursor; type RustLabeledNode = crate::cst::LabeledNode; type RustNode = crate::cst::Node; diff --git a/crates/codegen/parser/runtime/src/napi_interface/parse_error.rs b/crates/codegen/runtime/cargo/src/runtime/napi_interface/parse_error.rs similarity index 100% rename from crates/codegen/parser/runtime/src/napi_interface/parse_error.rs rename to crates/codegen/runtime/cargo/src/runtime/napi_interface/parse_error.rs diff --git a/crates/codegen/parser/runtime/src/napi_interface/parse_output.rs b/crates/codegen/runtime/cargo/src/runtime/napi_interface/parse_output.rs similarity index 100% rename from crates/codegen/parser/runtime/src/napi_interface/parse_output.rs rename to crates/codegen/runtime/cargo/src/runtime/napi_interface/parse_output.rs diff --git a/crates/codegen/parser/runtime/src/napi_interface/query.rs b/crates/codegen/runtime/cargo/src/runtime/napi_interface/query.rs similarity index 100% rename from crates/codegen/parser/runtime/src/napi_interface/query.rs rename to crates/codegen/runtime/cargo/src/runtime/napi_interface/query.rs diff --git a/crates/codegen/parser/runtime/src/napi_interface/text_index.rs b/crates/codegen/runtime/cargo/src/runtime/napi_interface/text_index.rs similarity index 100% rename from crates/codegen/parser/runtime/src/napi_interface/text_index.rs rename to crates/codegen/runtime/cargo/src/runtime/napi_interface/text_index.rs diff --git a/crates/codegen/parser/runtime/src/parse_error.rs b/crates/codegen/runtime/cargo/src/runtime/parse_error.rs similarity index 100% rename from crates/codegen/parser/runtime/src/parse_error.rs rename to crates/codegen/runtime/cargo/src/runtime/parse_error.rs diff --git a/crates/codegen/parser/runtime/src/parse_output.rs b/crates/codegen/runtime/cargo/src/runtime/parse_output.rs similarity index 100% rename from crates/codegen/parser/runtime/src/parse_output.rs rename to crates/codegen/runtime/cargo/src/runtime/parse_output.rs diff --git a/crates/codegen/parser/runtime/src/parser_support/choice_helper.rs b/crates/codegen/runtime/cargo/src/runtime/parser_support/choice_helper.rs similarity index 100% rename from crates/codegen/parser/runtime/src/parser_support/choice_helper.rs rename to crates/codegen/runtime/cargo/src/runtime/parser_support/choice_helper.rs diff --git a/crates/codegen/parser/runtime/src/parser_support/context.rs b/crates/codegen/runtime/cargo/src/runtime/parser_support/context.rs similarity index 99% rename from crates/codegen/parser/runtime/src/parser_support/context.rs rename to crates/codegen/runtime/cargo/src/runtime/parser_support/context.rs index 50c1f65aea..bfcb0556d3 100644 --- a/crates/codegen/parser/runtime/src/parser_support/context.rs +++ b/crates/codegen/runtime/cargo/src/runtime/parser_support/context.rs @@ -5,6 +5,7 @@ use crate::kinds::TokenKind; use crate::parse_error::ParseError; use crate::text_index::TextIndex; +#[derive(Debug)] pub struct ParserContext<'s> { source: &'s str, position: TextIndex, diff --git a/crates/codegen/parser/runtime/src/parser_support/mod.rs b/crates/codegen/runtime/cargo/src/runtime/parser_support/mod.rs similarity index 100% rename from crates/codegen/parser/runtime/src/parser_support/mod.rs rename to crates/codegen/runtime/cargo/src/runtime/parser_support/mod.rs diff --git a/crates/codegen/parser/runtime/src/parser_support/optional_helper.rs b/crates/codegen/runtime/cargo/src/runtime/parser_support/optional_helper.rs similarity index 100% rename from crates/codegen/parser/runtime/src/parser_support/optional_helper.rs rename to crates/codegen/runtime/cargo/src/runtime/parser_support/optional_helper.rs diff --git a/crates/codegen/parser/runtime/src/parser_support/parser_function.rs b/crates/codegen/runtime/cargo/src/runtime/parser_support/parser_function.rs similarity index 100% rename from crates/codegen/parser/runtime/src/parser_support/parser_function.rs rename to crates/codegen/runtime/cargo/src/runtime/parser_support/parser_function.rs diff --git a/crates/codegen/parser/runtime/src/parser_support/parser_result.rs b/crates/codegen/runtime/cargo/src/runtime/parser_support/parser_result.rs similarity index 100% rename from crates/codegen/parser/runtime/src/parser_support/parser_result.rs rename to crates/codegen/runtime/cargo/src/runtime/parser_support/parser_result.rs diff --git a/crates/codegen/parser/runtime/src/parser_support/precedence_helper.rs b/crates/codegen/runtime/cargo/src/runtime/parser_support/precedence_helper.rs similarity index 100% rename from crates/codegen/parser/runtime/src/parser_support/precedence_helper.rs rename to crates/codegen/runtime/cargo/src/runtime/parser_support/precedence_helper.rs diff --git a/crates/codegen/parser/runtime/src/parser_support/recovery.rs b/crates/codegen/runtime/cargo/src/runtime/parser_support/recovery.rs similarity index 100% rename from crates/codegen/parser/runtime/src/parser_support/recovery.rs rename to crates/codegen/runtime/cargo/src/runtime/parser_support/recovery.rs diff --git a/crates/codegen/parser/runtime/src/parser_support/repetition_helper.rs b/crates/codegen/runtime/cargo/src/runtime/parser_support/repetition_helper.rs similarity index 100% rename from crates/codegen/parser/runtime/src/parser_support/repetition_helper.rs rename to crates/codegen/runtime/cargo/src/runtime/parser_support/repetition_helper.rs diff --git a/crates/codegen/parser/runtime/src/parser_support/scanner_macros.rs b/crates/codegen/runtime/cargo/src/runtime/parser_support/scanner_macros.rs similarity index 100% rename from crates/codegen/parser/runtime/src/parser_support/scanner_macros.rs rename to crates/codegen/runtime/cargo/src/runtime/parser_support/scanner_macros.rs diff --git a/crates/codegen/parser/runtime/src/parser_support/separated_helper.rs b/crates/codegen/runtime/cargo/src/runtime/parser_support/separated_helper.rs similarity index 100% rename from crates/codegen/parser/runtime/src/parser_support/separated_helper.rs rename to crates/codegen/runtime/cargo/src/runtime/parser_support/separated_helper.rs diff --git a/crates/codegen/parser/runtime/src/parser_support/sequence_helper.rs b/crates/codegen/runtime/cargo/src/runtime/parser_support/sequence_helper.rs similarity index 100% rename from crates/codegen/parser/runtime/src/parser_support/sequence_helper.rs rename to crates/codegen/runtime/cargo/src/runtime/parser_support/sequence_helper.rs diff --git a/crates/codegen/parser/runtime/src/query/engine.rs b/crates/codegen/runtime/cargo/src/runtime/query/engine.rs similarity index 100% rename from crates/codegen/parser/runtime/src/query/engine.rs rename to crates/codegen/runtime/cargo/src/runtime/query/engine.rs diff --git a/crates/codegen/runtime/cargo/src/runtime/query/generated/user_defined_queries.rs b/crates/codegen/runtime/cargo/src/runtime/query/generated/user_defined_queries.rs new file mode 100644 index 0000000000..be909afcf2 --- /dev/null +++ b/crates/codegen/runtime/cargo/src/runtime/query/generated/user_defined_queries.rs @@ -0,0 +1,3 @@ +// This file is generated automatically by infrastructure scripts. Please don't edit by hand. + +pub(crate) trait UserDefinedQueries {} diff --git a/crates/codegen/parser/runtime/src/query/mod.rs b/crates/codegen/runtime/cargo/src/runtime/query/mod.rs similarity index 80% rename from crates/codegen/parser/runtime/src/query/mod.rs rename to crates/codegen/runtime/cargo/src/runtime/query/mod.rs index 0d5aa77993..2abfca13d5 100644 --- a/crates/codegen/parser/runtime/src/query/mod.rs +++ b/crates/codegen/runtime/cargo/src/runtime/query/mod.rs @@ -1,6 +1,8 @@ mod engine; mod model; mod parser; + +#[path = "generated/user_defined_queries.rs"] mod user_defined_queries; pub use engine::{QueryResult, QueryResultIterator}; diff --git a/crates/codegen/parser/runtime/src/query/model.rs b/crates/codegen/runtime/cargo/src/runtime/query/model.rs similarity index 100% rename from crates/codegen/parser/runtime/src/query/model.rs rename to crates/codegen/runtime/cargo/src/runtime/query/model.rs diff --git a/crates/codegen/parser/runtime/src/query/parser.rs b/crates/codegen/runtime/cargo/src/runtime/query/parser.rs similarity index 100% rename from crates/codegen/parser/runtime/src/query/parser.rs rename to crates/codegen/runtime/cargo/src/runtime/query/parser.rs diff --git a/crates/codegen/runtime/cargo/src/runtime/query/user_defined_queries.rs.jinja2 b/crates/codegen/runtime/cargo/src/runtime/query/user_defined_queries.rs.jinja2 new file mode 100644 index 0000000000..e725cf0b9b --- /dev/null +++ b/crates/codegen/runtime/cargo/src/runtime/query/user_defined_queries.rs.jinja2 @@ -0,0 +1,16 @@ +pub(crate) trait UserDefinedQueries { + {% if not rendering_in_stubs %} + {%- for query, captures in model.queries -%} + {# + # TODO(#554): not sure if this type is expected to be side-effect free implementation, + # or is it going to carry an incremental state. We can add other callbacks as needed. + #} + + fn {{ query }}( + {%- for capture in captures -%} + {{ capture }}: &crate::cursor::Cursor, + {%- endfor -%} + ); + {%- endfor -%} + {% endif %} +} diff --git a/crates/codegen/parser/runtime/src/text_index.rs b/crates/codegen/runtime/cargo/src/runtime/text_index.rs similarity index 100% rename from crates/codegen/parser/runtime/src/text_index.rs rename to crates/codegen/runtime/cargo/src/runtime/text_index.rs diff --git a/crates/codegen/runtime/cargo/src/user_defined/mod.rs b/crates/codegen/runtime/cargo/src/user_defined/mod.rs new file mode 100644 index 0000000000..67350db2a2 --- /dev/null +++ b/crates/codegen/runtime/cargo/src/user_defined/mod.rs @@ -0,0 +1 @@ +pub mod query; diff --git a/crates/codegen/runtime/cargo/src/user_defined/query.rs b/crates/codegen/runtime/cargo/src/user_defined/query.rs new file mode 100644 index 0000000000..c2bf9fb672 --- /dev/null +++ b/crates/codegen/runtime/cargo/src/user_defined/query.rs @@ -0,0 +1,7 @@ +use crate::query::UserDefinedQueries; + +pub struct UserDefinedQueriesImpl; + +impl UserDefinedQueries for UserDefinedQueriesImpl { + // Empty Stub +} diff --git a/crates/codegen/parser/generator/Cargo.toml b/crates/codegen/runtime/generator/Cargo.toml similarity index 93% rename from crates/codegen/parser/generator/Cargo.toml rename to crates/codegen/runtime/generator/Cargo.toml index 4ec238c00c..72ea0e22da 100644 --- a/crates/codegen/parser/generator/Cargo.toml +++ b/crates/codegen/runtime/generator/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "codegen_parser_generator" +name = "codegen_runtime_generator" version.workspace = true rust-version.workspace = true edition.workspace = true @@ -10,8 +10,8 @@ anyhow = { workspace = true } codegen_grammar = { workspace = true } codegen_language_definition = { workspace = true } indexmap = { workspace = true } -infra_utils = { workspace = true } Inflector = { workspace = true } +infra_utils = { workspace = true } itertools = { workspace = true } proc-macro2 = { workspace = true } quote = { workspace = true } @@ -20,4 +20,3 @@ serde = { workspace = true } [lints] workspace = true - diff --git a/crates/codegen/parser/generator/src/ast_model.rs b/crates/codegen/runtime/generator/src/ast.rs similarity index 100% rename from crates/codegen/parser/generator/src/ast_model.rs rename to crates/codegen/runtime/generator/src/ast.rs diff --git a/crates/codegen/parser/generator/src/keyword_scanner_definition.rs b/crates/codegen/runtime/generator/src/keyword_scanner_definition.rs similarity index 100% rename from crates/codegen/parser/generator/src/keyword_scanner_definition.rs rename to crates/codegen/runtime/generator/src/keyword_scanner_definition.rs diff --git a/crates/codegen/runtime/generator/src/lib.rs b/crates/codegen/runtime/generator/src/lib.rs new file mode 100644 index 0000000000..c4f43dd95f --- /dev/null +++ b/crates/codegen/runtime/generator/src/lib.rs @@ -0,0 +1,67 @@ +#![allow(clippy::too_many_lines)] + +use std::path::{Path, PathBuf}; +use std::rc::Rc; + +use anyhow::Result; +use codegen_language_definition::model::Language; +use infra_utils::cargo::CargoWorkspace; +use infra_utils::codegen::CodegenTemplates; +use serde::Serialize; + +use crate::model::RuntimeModel; + +mod ast; +mod keyword_scanner_definition; +mod model; +mod parser_definition; +mod precedence_parser_definition; +mod scanner_definition; +mod trie; + +pub enum OutputLanguage { + Cargo, + Npm, +} + +/// A utility wrapper to make it easier to write conditional code in templates +/// by checking `rendering_in_stubs`. Additionally, it makes sure that all +/// model properties are prefixed with `model.` to make it easier to read/refactor. +#[derive(Serialize)] +struct ModelWrapper { + rendering_in_stubs: bool, + model: Option, +} + +impl OutputLanguage { + pub fn generate_runtime(&self, language: &Rc, output_dir: &Path) -> Result<()> { + let model = ModelWrapper { + rendering_in_stubs: false, + model: Some(RuntimeModel::from_language(language)), + }; + + let mut templates = CodegenTemplates::new(self.source_dir()?)?; + + templates.render_directory(model, output_dir) + } + + pub fn generate_stubs(&self) -> Result<()> { + let model = ModelWrapper { + rendering_in_stubs: true, + model: None, + }; + + let mut templates = CodegenTemplates::new(self.source_dir()?)?; + + templates.render_stubs(&model) + } + + fn source_dir(&self) -> Result { + let crate_name = match self { + Self::Cargo => "codegen_runtime_cargo", + Self::Npm => "codegen_runtime_npm", + }; + + Ok(CargoWorkspace::locate_source_crate(crate_name)?.join("src/runtime")) + } +} diff --git a/crates/codegen/parser/generator/src/rust_generator.rs b/crates/codegen/runtime/generator/src/model.rs similarity index 73% rename from crates/codegen/parser/generator/src/rust_generator.rs rename to crates/codegen/runtime/generator/src/model.rs index 0bbd0bcaa5..379f9070a6 100644 --- a/crates/codegen/parser/generator/src/rust_generator.rs +++ b/crates/codegen/runtime/generator/src/model.rs @@ -1,11 +1,8 @@ -// TODO(#863): this is getting replaced by runtime templates: #![allow(clippy::too_many_lines)] use std::collections::{BTreeMap, BTreeSet}; -use std::path::Path; use std::rc::Rc; -use anyhow::Result; use codegen_grammar::{ Grammar, GrammarConstructorDslV2, GrammarVisitor, KeywordScannerAtomic, KeywordScannerDefinitionRef, ParserDefinitionNode, ParserDefinitionRef, @@ -14,13 +11,11 @@ use codegen_grammar::{ }; use codegen_language_definition::model::Language; use indexmap::IndexMap; -use infra_utils::cargo::CargoWorkspace; -use infra_utils::codegen::Codegen; use quote::{format_ident, quote}; use semver::Version; use serde::Serialize; -use crate::ast_model::AstModel; +use crate::ast::AstModel; use crate::keyword_scanner_definition::KeywordScannerDefinitionExtensions; use crate::parser_definition::ParserDefinitionExtensions; use crate::precedence_parser_definition::PrecedenceParserDefinitionExtensions; @@ -28,7 +23,8 @@ use crate::scanner_definition::ScannerDefinitionExtensions; use crate::trie::Trie; #[derive(Default, Serialize)] -pub struct RustGenerator { +pub struct RuntimeModel { + all_versions: BTreeSet, referenced_versions: BTreeSet, rule_kinds: BTreeSet<&'static str>, @@ -43,6 +39,10 @@ pub struct RustGenerator { parser_functions: BTreeMap<&'static str, String>, // (name of parser, code) trivia_parser_functions: BTreeMap<&'static str, String>, // (name of parser, code) + ast: AstModel, + + queries: IndexMap>, + #[serde(skip)] top_level_scanner_names: BTreeSet<&'static str>, #[serde(skip)] @@ -65,132 +65,27 @@ struct ScannerContext { delimiters: BTreeMap<&'static str, &'static str>, } -impl RustGenerator { - pub fn generate(language: &Language, output_dir: &Path) -> Result<()> { +impl RuntimeModel { + pub fn from_language(language: &Rc) -> Self { let grammar = Grammar::from_dsl_v2(language); - let generator = &mut Self::default(); - grammar.accept_visitor(generator); - - let runtime_dir = - CargoWorkspace::locate_source_crate("codegen_parser_runtime")?.join("src"); - - let mut codegen = Codegen::read_write(&runtime_dir)?; - - { - #[derive(Serialize)] - struct Context { - ast_model: AstModel, - } - codegen.render( - Context { - ast_model: AstModel::create(language), - }, - runtime_dir.join("napi_interface/templates/ast_selectors.rs.jinja2"), - output_dir.join("napi_interface/ast_selectors.rs"), - )?; - } - - { - #[derive(Serialize)] - struct Context<'a> { - generator: &'a RustGenerator, - } - codegen.render( - Context { generator }, - runtime_dir.join("templates/kinds.rs.jinja2"), - output_dir.join("kinds.rs"), - )?; - } - - { - #[derive(Serialize)] - struct Context { - queries: IndexMap>, - } - - let queries = language - .queries - .keys() - .enumerate() - .map(|(index, key)| { - // TODO(#554): parse the query and extract the real captures: - ( - key.to_string(), - ["foo", "bar", "baz"].into_iter().take(index + 1).collect(), - ) - }) - .collect(); - - codegen.render( - Context { queries }, - runtime_dir.join("templates/user_defined_queries.rs.jinja2"), - output_dir.join("query/user_defined_queries.rs"), - )?; - } - - { - #[derive(Serialize)] - struct Context<'a> { - generator: &'a RustGenerator, - language_name: String, - versions: BTreeSet, - } - codegen.render( - Context { - generator, - language_name: grammar.name.clone(), - versions: grammar.versions.clone(), - }, - runtime_dir.join("templates/language.rs.jinja2"), - output_dir.join("language.rs"), - )?; - } - - #[allow(clippy::single_element_loop)] - // kept in case there is once again more than one of these - for (src_file, destination_file) in &[("mod_for_destination.rs", "mod.rs")] { - codegen.copy_file( - runtime_dir.join(src_file), - output_dir.join(destination_file), - )?; - } - - for file in &[ - "cst.rs", - "cursor.rs", - "lexer.rs", - "napi_interface/cst.rs", - "napi_interface/cursor.rs", - "napi_interface/mod.rs", - "napi_interface/parse_error.rs", - "napi_interface/parse_output.rs", - "napi_interface/query.rs", - "napi_interface/text_index.rs", - "parse_error.rs", - "parse_output.rs", - "parser_support/choice_helper.rs", - "parser_support/context.rs", - "parser_support/mod.rs", - "parser_support/optional_helper.rs", - "parser_support/parser_function.rs", - "parser_support/parser_result.rs", - "parser_support/precedence_helper.rs", - "parser_support/recovery.rs", - "parser_support/repetition_helper.rs", - "parser_support/scanner_macros.rs", - "parser_support/separated_helper.rs", - "parser_support/sequence_helper.rs", - "query/engine.rs", - "query/mod.rs", - "query/model.rs", - "query/parser.rs", - "text_index.rs", - ] { - codegen.copy_file(runtime_dir.join(file), output_dir.join(file))?; - } - - Ok(()) + let mut model = Self::default(); + grammar.accept_visitor(&mut model); + + // TODO(#638): Absorb the relevant fields into the model tree after migration is complete: + model.all_versions = language.versions.iter().cloned().collect(); + model.ast = AstModel::create(language); + model + .queries + .extend(language.queries.keys().enumerate().map(|(index, key)| { + // TODO(#554): parse the query and extract the real captures: + ( + key.to_string(), + ["foo", "bar", "baz"].into_iter().take(index + 1).collect(), + ) + })); + + model } fn set_current_context(&mut self, name: &'static str) { @@ -205,7 +100,7 @@ impl RustGenerator { } } -impl GrammarVisitor for RustGenerator { +impl GrammarVisitor for RuntimeModel { fn grammar_leave(&mut self, _grammar: &Grammar) { // Expose the scanner functions that... self.scanner_functions = self diff --git a/crates/codegen/parser/generator/src/parser_definition.rs b/crates/codegen/runtime/generator/src/parser_definition.rs similarity index 100% rename from crates/codegen/parser/generator/src/parser_definition.rs rename to crates/codegen/runtime/generator/src/parser_definition.rs diff --git a/crates/codegen/parser/generator/src/precedence_parser_definition.rs b/crates/codegen/runtime/generator/src/precedence_parser_definition.rs similarity index 100% rename from crates/codegen/parser/generator/src/precedence_parser_definition.rs rename to crates/codegen/runtime/generator/src/precedence_parser_definition.rs diff --git a/crates/codegen/parser/generator/src/scanner_definition.rs b/crates/codegen/runtime/generator/src/scanner_definition.rs similarity index 100% rename from crates/codegen/parser/generator/src/scanner_definition.rs rename to crates/codegen/runtime/generator/src/scanner_definition.rs diff --git a/crates/codegen/parser/generator/src/trie.rs b/crates/codegen/runtime/generator/src/trie.rs similarity index 100% rename from crates/codegen/parser/generator/src/trie.rs rename to crates/codegen/runtime/generator/src/trie.rs diff --git a/crates/codegen/runtime/node_addon/Cargo.toml b/crates/codegen/runtime/node_addon/Cargo.toml new file mode 100644 index 0000000000..16b5731253 --- /dev/null +++ b/crates/codegen/runtime/node_addon/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "codegen_runtime_node_addon" +version.workspace = true +rust-version.workspace = true +edition.workspace = true +publish = false + +# At the time of writing, building cdylib/plugins with Cargo has limited support +# (see for the general tracking issue). +# To workaround some of the issues, we create a synthetic crate that shares the same code as the original Rust crate +# but only exposes a `cdylib` library. +# There are a couple of things at play here: +# - To build a Node addon, we need to build a standalone cdylib, that's set up to expect host `napi_*` functions at runtime +# - `napi build` does that for us but only supports `cdylib` for the `lib` targets (e.g. not bin/examples) +# - expansion of #[napi] macros is enabled behind a feature flag; it expands the code that expects the host functions +# - while `napi/noop` feature exists that disables that, we test in workspace with `--all-features` and Cargo features are additive +# - thus, they get conflated and so the regular `rlib` library target gets built with the host functions expectation +# - ...which reasonably fails to link when `rlib` library is used as a regular Rust library (i.e. not in Node environment). +# Another workaround is to use an example with `cdylib` crate-type (wouldn't conflate the features in a downstream usable +# library), but napi doesn't support that yet. +# Also see for a similar issue with PyO3. +[lib] +path = "../cargo/src/lib.rs" +crate-type = ["cdylib"] + +[features] +default = ["slang_napi_interfaces"] +slang_napi_interfaces = [ + # This enables '#[napi]' attributes on the Rust types imported via [lib.path] above. + "dep:serde_json", +] + +[build-dependencies] +napi-build = { workspace = true } + +[dependencies] +ariadne = { workspace = true } +napi = { workspace = true } +napi-derive = { workspace = true } +nom = { workspace = true } +semver = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true, optional = true } +strum = { workspace = true } +strum_macros = { workspace = true } +thiserror = { workspace = true } + +[lints] +workspace = true diff --git a/crates/codegen/runtime/node_addon/build.rs b/crates/codegen/runtime/node_addon/build.rs new file mode 100644 index 0000000000..27fcbe05cf --- /dev/null +++ b/crates/codegen/runtime/node_addon/build.rs @@ -0,0 +1,6 @@ +#[cfg(not(feature = "slang_napi_interfaces"))] +compile_error!("The whole point is to enable slang_napi_interfaces for this cdylib crate!"); + +fn main() { + napi_build::setup(); +} diff --git a/crates/codegen/runtime/npm/CHANGELOG.md b/crates/codegen/runtime/npm/CHANGELOG.md new file mode 100644 index 0000000000..e26ad44d42 --- /dev/null +++ b/crates/codegen/runtime/npm/CHANGELOG.md @@ -0,0 +1,318 @@ +# changelog + +## 0.14.2 + +### Patch Changes + +- [#948](https://github.com/NomicFoundation/slang/pull/948) [`ce88cb7`](https://github.com/NomicFoundation/slang/commit/ce88cb7a6fd945b59ccc967cfd20f423dadc36fc) Thanks [@Xanewok](https://github.com/Xanewok)! - Restrict the grammar to correctly only allow an identifier in Yul variable declaration + +- [#945](https://github.com/NomicFoundation/slang/pull/945) [`e8f80d8`](https://github.com/NomicFoundation/slang/commit/e8f80d867b4b9d02413f42a8ece2630a43bc7494) Thanks [@Xanewok](https://github.com/Xanewok)! - Support `.address` built-in access in Yul paths + +## 0.14.1 + +### Patch Changes + +- [#943](https://github.com/NomicFoundation/slang/pull/943) [`a561fb1`](https://github.com/NomicFoundation/slang/commit/a561fb161eb7c18c838c85f71d132764d1d04050) Thanks [@Xanewok](https://github.com/Xanewok)! - Support Solidity 0.8.25 + +## 0.14.0 + +### Minor Changes + +- [#753](https://github.com/NomicFoundation/slang/pull/753) [`b35c763`](https://github.com/NomicFoundation/slang/commit/b35c7630ab7240304e67a43734700cf359acde0b) Thanks [@AntonyBlakey](https://github.com/AntonyBlakey)! - Add tree query implementation as `Query::parse` and `Cursor::query` + +- [#755](https://github.com/NomicFoundation/slang/pull/755) [`8c260fc`](https://github.com/NomicFoundation/slang/commit/8c260fcb7e3111191cd33dd527817fb51119eac4) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - support parsing NatSpec comments + +- [#908](https://github.com/NomicFoundation/slang/pull/908) [`ab3688b`](https://github.com/NomicFoundation/slang/commit/ab3688bb99a60862c506566ac6122cd9c1155c57) Thanks [@Xanewok](https://github.com/Xanewok)! - Changed the cst.NodeType in TS to use more descriptive string values rather than 0/1 integers + +- [#886](https://github.com/NomicFoundation/slang/pull/886) [`0125717`](https://github.com/NomicFoundation/slang/commit/0125717fb0b48a5342a8452f18080db13e68fb6b) Thanks [@Xanewok](https://github.com/Xanewok)! - Add `TokenKind::is_trivia` + +- [#887](https://github.com/NomicFoundation/slang/pull/887) [`dff1201`](https://github.com/NomicFoundation/slang/commit/dff12011c549d68b20ecd54251af764643fb72db) Thanks [@Xanewok](https://github.com/Xanewok)! - Add support for constant function modifier removed in 0.5.0 + +- [#885](https://github.com/NomicFoundation/slang/pull/885) [`a9bd8da`](https://github.com/NomicFoundation/slang/commit/a9bd8da018469739832f71e38437caa83087baf0) Thanks [@Xanewok](https://github.com/Xanewok)! - Flatten the trivia syntax nodes into sibling tokens + +- [#908](https://github.com/NomicFoundation/slang/pull/908) [`ab3688b`](https://github.com/NomicFoundation/slang/commit/ab3688bb99a60862c506566ac6122cd9c1155c57) Thanks [@Xanewok](https://github.com/Xanewok)! - Add RuleNode/TokenNode::toJSON() in the TypeScript API + +### Patch Changes + +- [#801](https://github.com/NomicFoundation/slang/pull/801) [`ecbba49`](https://github.com/NomicFoundation/slang/commit/ecbba49c7ac25e37b8d317fb60fab7340c0628a5) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - unreserve pragma keywords in all versions + +- [#869](https://github.com/NomicFoundation/slang/pull/869) [`951b58d`](https://github.com/NomicFoundation/slang/commit/951b58ddb3eaea600ddf44427a82649761c6b651) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - support dots in yul identifiers from `0.5.8` till `0.7.0` + +- [#890](https://github.com/NomicFoundation/slang/pull/890) [`1ff8599`](https://github.com/NomicFoundation/slang/commit/1ff85993f25d92b38d0a500baa6ee48669a1b62a) Thanks [@Xanewok](https://github.com/Xanewok)! - Mark `override` as being a valid attribute only after 0.6.0 + +- [#800](https://github.com/NomicFoundation/slang/pull/800) [`d1827ff`](https://github.com/NomicFoundation/slang/commit/d1827ff7e1010493ff5487532a5ee0c77d355aa2) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - support unicode characters in string literals up to `0.7.0` + +- [#797](https://github.com/NomicFoundation/slang/pull/797) [`86f36d7`](https://github.com/NomicFoundation/slang/commit/86f36d71e60a44261ec114339e931dd3d24cd4a4) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - fix source locations for unicode characters in error reports + +- [#854](https://github.com/NomicFoundation/slang/pull/854) [`4b8970b`](https://github.com/NomicFoundation/slang/commit/4b8970b47ef7a2d1d51339cf5020a3e0f168b9aa) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - parse line breaks without newlines + +- [#844](https://github.com/NomicFoundation/slang/pull/844) [`f62de9e`](https://github.com/NomicFoundation/slang/commit/f62de9ea3fc2049ee11e5dbeff3dc51eb1ca984e) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - fix parsing empty `/**/` comments + +- [#799](https://github.com/NomicFoundation/slang/pull/799) [`303dda9`](https://github.com/NomicFoundation/slang/commit/303dda95c08b20450d03116765c210ece64a0864) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - prevent parsing multiple literals under `StringExpression` before `0.5.14` + +- [#847](https://github.com/NomicFoundation/slang/pull/847) [`6b6f260`](https://github.com/NomicFoundation/slang/commit/6b6f2603e3ba07c0a7dede0f96082369dc1df940) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - prioritize parsing `MultiLineComment` over `MultiLineNatSpecComment` + +- [#796](https://github.com/NomicFoundation/slang/pull/796) [`59e1e53`](https://github.com/NomicFoundation/slang/commit/59e1e53e7efa52355c273d7cef1a3974de13d88d) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - add `public` and `internal` to `UnnamedFunctionAttribute` till `0.5.0` + +- [#756](https://github.com/NomicFoundation/slang/pull/756) [`e839817`](https://github.com/NomicFoundation/slang/commit/e8398173f62d48596669628afc7c8b3572a15291) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - fix parsing `payable` primary expressions + +- [#851](https://github.com/NomicFoundation/slang/pull/851) [`67dfde8`](https://github.com/NomicFoundation/slang/commit/67dfde81a6d00101a9ed133104f15da5d46662b6) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - fix selection order of prefix/postfix AST fields + +- [#857](https://github.com/NomicFoundation/slang/pull/857) [`f677d5e`](https://github.com/NomicFoundation/slang/commit/f677d5eff40c4bfcf1db2fc4e63cdf37457fe467) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - rename `FieldName` to `NodeLabel` + +- [#852](https://github.com/NomicFoundation/slang/pull/852) [`ca79eca`](https://github.com/NomicFoundation/slang/commit/ca79ecaa522e531420b42ffba67da192c1e5fdb2) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - allow parsing `ColonEqual` as two separate tokens before `0.5.5` + +- [#889](https://github.com/NomicFoundation/slang/pull/889) [`ce5050f`](https://github.com/NomicFoundation/slang/commit/ce5050f95195fdd018a38a0351d8525f7d62073a) Thanks [@Xanewok](https://github.com/Xanewok)! - Support `delete` as an expression rather than a statement + +- [#923](https://github.com/NomicFoundation/slang/pull/923) [`bb30fc1`](https://github.com/NomicFoundation/slang/commit/bb30fc1e28a0fe806f8954a0d2779d903f3f4da7) Thanks [@Xanewok](https://github.com/Xanewok)! - Support arbitrary ASCII escape sequences in string literals until 0.4.25 + +- [#887](https://github.com/NomicFoundation/slang/pull/887) [`dff1201`](https://github.com/NomicFoundation/slang/commit/dff12011c549d68b20ecd54251af764643fb72db) Thanks [@Xanewok](https://github.com/Xanewok)! - Support view and pure function modifiers only from 0.4.16 + +- [#800](https://github.com/NomicFoundation/slang/pull/800) [`d1827ff`](https://github.com/NomicFoundation/slang/commit/d1827ff7e1010493ff5487532a5ee0c77d355aa2) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - rename `AsciiStringLiteral` to `StringLiteral` + +- [#838](https://github.com/NomicFoundation/slang/pull/838) [`ad98d1c`](https://github.com/NomicFoundation/slang/commit/ad98d1c7d9f9f7cb12b4b6184c04c9b680e6d70a) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - upgrade to rust `1.76.0` + +- [#849](https://github.com/NomicFoundation/slang/pull/849) [`5c42e0e`](https://github.com/NomicFoundation/slang/commit/5c42e0ef5f3afe0355614967cb6d2daa31518ccf) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - add `override` and `virtual` to `ConstructorAttribute` + +- [#862](https://github.com/NomicFoundation/slang/pull/862) [`5e37ea0`](https://github.com/NomicFoundation/slang/commit/5e37ea0c40e929e0888b6297fa6dd92952d9cd73) Thanks [@Xanewok](https://github.com/Xanewok)! - allow call options as a post-fix expression + +- [#786](https://github.com/NomicFoundation/slang/pull/786) [`0bfa6b7`](https://github.com/NomicFoundation/slang/commit/0bfa6b7397cd25aca713b30628c6d06e761b416a) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - support Yul label statements before `0.5.0` + +- [#839](https://github.com/NomicFoundation/slang/pull/839) [`2d698eb`](https://github.com/NomicFoundation/slang/commit/2d698ebe469110b85f539d6e0c75b503cd4ce57e) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - support string literals in version pragmas + +- [#891](https://github.com/NomicFoundation/slang/pull/891) [`70c9d7d`](https://github.com/NomicFoundation/slang/commit/70c9d7deebddb0f22114b7b05ddc85da6dcceaaf) Thanks [@Xanewok](https://github.com/Xanewok)! - Fix parsing `.member` member access expression + +- [#842](https://github.com/NomicFoundation/slang/pull/842) [`2069126`](https://github.com/NomicFoundation/slang/commit/20691263fb6967195bee30fba92abdfb06daa6fa) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - add `private` to `UnnamedFunctionAttribute` till `0.5.0` + +- [#840](https://github.com/NomicFoundation/slang/pull/840) [`7fb0d20`](https://github.com/NomicFoundation/slang/commit/7fb0d20655024daf71c872a6ef95aa30277a1366) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - allow `var` in `TupleDeconstructionStatement` before `0.5.0` + +## 0.13.1 + +### Patch Changes + +- [#748](https://github.com/NomicFoundation/slang/pull/748) [`c289cbf7`](https://github.com/NomicFoundation/slang/commit/c289cbf7e22118881818b82d0ffc5933a424a7aa) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - Properly parse EVM built-ins up till Paris/Solidity 0.8.18 + +## 0.13.0 + +### Minor Changes + +- [#710](https://github.com/NomicFoundation/slang/pull/710) [`2025b6cb`](https://github.com/NomicFoundation/slang/commit/2025b6cb23dc320b413b482ed1fe8455229b7d84) Thanks [@Xanewok](https://github.com/Xanewok)! - CST children nodes are now named + +- [#723](https://github.com/NomicFoundation/slang/pull/723) [`b3dc6bcd`](https://github.com/NomicFoundation/slang/commit/b3dc6bcdc1834d266a87d483927894617bf8e817) Thanks [@Xanewok](https://github.com/Xanewok)! - Properly parse unreserved keywords in an identifier position, i.e. `from`, `emit`, `global` etc. + +- [#728](https://github.com/NomicFoundation/slang/pull/728) [`662a672c`](https://github.com/NomicFoundation/slang/commit/662a672cd661b9f1bf4c18587acf68111fd1f2e8) Thanks [@Xanewok](https://github.com/Xanewok)! - Remove Language#scan API; use the parser API instead + +- [#719](https://github.com/NomicFoundation/slang/pull/719) [`1ad6bb37`](https://github.com/NomicFoundation/slang/commit/1ad6bb37337aa28d9344380c5c9eb1945e908271) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - introduce strong types for all Solidity non terminals in the TypeScript API. + +### Patch Changes + +- [#719](https://github.com/NomicFoundation/slang/pull/719) [`1ad6bb37`](https://github.com/NomicFoundation/slang/commit/1ad6bb37337aa28d9344380c5c9eb1945e908271) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - unify Rust/TypeScript node helpers: `*_with_kind()`, `*_with_kinds()`, `*_is_kind()`), ... + +- [#731](https://github.com/NomicFoundation/slang/pull/731) [`3deaea2e`](https://github.com/NomicFoundation/slang/commit/3deaea2eb82ce33dbccc54d1a79b9cf5657385ac) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - add `RuleNode.unparse()` to the TypeScript API + +## 0.12.0 + +### Minor Changes + +- [#699](https://github.com/NomicFoundation/slang/pull/699) [`ddfebfe9`](https://github.com/NomicFoundation/slang/commit/ddfebfe988345136007431f8ea2efac19fd7e887) Thanks [@Xanewok](https://github.com/Xanewok)! - Remove `ProductionKind` in favor of `RuleKind` + +- [#699](https://github.com/NomicFoundation/slang/pull/699) [`ddfebfe9`](https://github.com/NomicFoundation/slang/commit/ddfebfe988345136007431f8ea2efac19fd7e887) Thanks [@Xanewok](https://github.com/Xanewok)! - Allow parsing individual precedence expressions, like `ShiftExpression` + +- [#665](https://github.com/NomicFoundation/slang/pull/665) [`4b5f8b46`](https://github.com/NomicFoundation/slang/commit/4b5f8b467d4cbab72cf27a539bb5ca8c71090dd6) Thanks [@Xanewok](https://github.com/Xanewok)! - Remove the CST Visitor API in favor of the Cursor API + +- [#666](https://github.com/NomicFoundation/slang/pull/666) [`0434b68c`](https://github.com/NomicFoundation/slang/commit/0434b68c9ef9cd1d1dcc07d7ed50e6d63645319b) Thanks [@Xanewok](https://github.com/Xanewok)! - Add `Node::unparse()` that allows to reconstruct the source code from the CST node + +- [#675](https://github.com/NomicFoundation/slang/pull/675) [`daea4b7f`](https://github.com/NomicFoundation/slang/commit/daea4b7f954ff1e918b9191aff40ee95c10a4db2) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - rename `Cursor`'s `pathRuleNodes()` to `ancestors()` in the NodeJS API. + +- [#676](https://github.com/NomicFoundation/slang/pull/676) [`b496d361`](https://github.com/NomicFoundation/slang/commit/b496d36120700347bcbcc25b948eb46814fd5412) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - Fix NAPI `cursor` types and expose `cursor.depth`. + +### Patch Changes + +- [#685](https://github.com/NomicFoundation/slang/pull/685) [`b5fca94a`](https://github.com/NomicFoundation/slang/commit/b5fca94af917a2f0418c224b3101885c02e5cb9c) Thanks [@Xanewok](https://github.com/Xanewok)! - `bytes` is now properly recognized as a reserved word + +- [#660](https://github.com/NomicFoundation/slang/pull/660) [`97028991`](https://github.com/NomicFoundation/slang/commit/9702899164f0540a49f2e0f7f19d82fbd04b1d1b) Thanks [@Xanewok](https://github.com/Xanewok)! - Drop List suffix from collection grammar rule names + +## 0.11.0 + +### Minor Changes + +- [#625](https://github.com/NomicFoundation/slang/pull/625) [`7bb650b`](https://github.com/NomicFoundation/slang/commit/7bb650b12ae793a318dc5b7839fb93915c88828e) Thanks [@Xanewok](https://github.com/Xanewok)! - The CST Cursor now implements the Iterator trait as part of the Rust API + +- [#647](https://github.com/NomicFoundation/slang/pull/647) [`b1dced3`](https://github.com/NomicFoundation/slang/commit/b1dced355ce83f3bd858c02837d67665f7ef281d) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - Require specifying an initial offset when creating a CST cursor. + +### Patch Changes + +- [#648](https://github.com/NomicFoundation/slang/pull/648) [`2327bf5`](https://github.com/NomicFoundation/slang/commit/2327bf5d8c40d85edd0cc80fe9e36d367a1a3336) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - Support Solidity v0.8.22. + +- [#623](https://github.com/NomicFoundation/slang/pull/623) [`80114a8`](https://github.com/NomicFoundation/slang/commit/80114a833dc8249447c382bf457215b1a4d9e5ae) Thanks [@AntonyBlakey](https://github.com/AntonyBlakey)! - Correct the types in the TS api by adding the correct namespaces to type references + +## 0.10.1 + +### Patch Changes + +- [#615](https://github.com/NomicFoundation/slang/pull/615) [`06cbbe8`](https://github.com/NomicFoundation/slang/commit/06cbbe88bc68928ad44046a96c31ad6e53fbf76c) Thanks [@AntonyBlakey](https://github.com/AntonyBlakey)! - `cursor` method is now exposed in Typescript API + +## 0.10.0 + +### Minor Changes + +- [#595](https://github.com/NomicFoundation/slang/pull/595) [`1a258c4`](https://github.com/NomicFoundation/slang/commit/1a258c49432eac06dac7055bc427e68af1fa3875) Thanks [@Xanewok](https://github.com/Xanewok)! - Attempt error recovery when parsing incomplete lists + +- [#564](https://github.com/NomicFoundation/slang/pull/564) [`e326a06`](https://github.com/NomicFoundation/slang/commit/e326a064da559c974fbb7a199090e9e5a313abb8) Thanks [@AntonyBlakey](https://github.com/AntonyBlakey)! - Parsing operators with missing operands should no longer panic + +- [#564](https://github.com/NomicFoundation/slang/pull/564) [`e326a06`](https://github.com/NomicFoundation/slang/commit/e326a064da559c974fbb7a199090e9e5a313abb8) Thanks [@AntonyBlakey](https://github.com/AntonyBlakey)! - Inline parse rules are no longer exposed to the API. + +- [#564](https://github.com/NomicFoundation/slang/pull/564) [`e326a06`](https://github.com/NomicFoundation/slang/commit/e326a064da559c974fbb7a199090e9e5a313abb8) Thanks [@AntonyBlakey](https://github.com/AntonyBlakey)! - Scanners are no longer available as methods - use next_token instead + +- [#564](https://github.com/NomicFoundation/slang/pull/564) [`e326a06`](https://github.com/NomicFoundation/slang/commit/e326a064da559c974fbb7a199090e9e5a313abb8) Thanks [@AntonyBlakey](https://github.com/AntonyBlakey)! - Scanners are now grouped into contexts to deal with contextual scanning + +### Patch Changes + +- [#601](https://github.com/NomicFoundation/slang/pull/601) [`cbd2a79`](https://github.com/NomicFoundation/slang/commit/cbd2a79658849c0029bb6a5ccc0b086564c28fe0) Thanks [@Xanewok](https://github.com/Xanewok)! - Attempt parser error recovery between bracket delimiters + +- [#599](https://github.com/NomicFoundation/slang/pull/599) [`4bbad48`](https://github.com/NomicFoundation/slang/commit/4bbad48d45ae7bde8a22198b33f790b7c792b319) Thanks [@Xanewok](https://github.com/Xanewok)! - Use correct versions for the `revert` and `global` keywords + +- [#561](https://github.com/NomicFoundation/slang/pull/561) [`cb6a138`](https://github.com/NomicFoundation/slang/commit/cb6a1384cb6f04926def3e4c1fe7a0b12a67143c) Thanks [@Xanewok](https://github.com/Xanewok)! - Add preliminary documentation for the `solidity_language` Rust package + +- [#603](https://github.com/NomicFoundation/slang/pull/603) [`be59a10`](https://github.com/NomicFoundation/slang/commit/be59a10c937542f0413a34fd84d84ec4d4400f6d) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - upgrade to rust 1.72.0 + +## 0.9.0 + +### Minor Changes + +- [#540](https://github.com/NomicFoundation/slang/pull/540) [`0d04f95`](https://github.com/NomicFoundation/slang/commit/0d04f959bf1f5831c912d5109de3d933cfaa6266) Thanks [@AntonyBlakey](https://github.com/AntonyBlakey)! - Add a Rust Cursor API and refactor the Rust Visitor API to run on top of it. + +- [#540](https://github.com/NomicFoundation/slang/pull/540) [`0d04f95`](https://github.com/NomicFoundation/slang/commit/0d04f959bf1f5831c912d5109de3d933cfaa6266) Thanks [@AntonyBlakey](https://github.com/AntonyBlakey)! - Move Visitor et al to node:: namespace, which is where Cursor is. + +- [#540](https://github.com/NomicFoundation/slang/pull/540) [`0d04f95`](https://github.com/NomicFoundation/slang/commit/0d04f959bf1f5831c912d5109de3d933cfaa6266) Thanks [@AntonyBlakey](https://github.com/AntonyBlakey)! - Rename `range` functions that return a TextRange to `text_range` + +### Patch Changes + +- [#543](https://github.com/NomicFoundation/slang/pull/543) [`7a34599`](https://github.com/NomicFoundation/slang/commit/7a34599f6b237b03a0f8ba92755cae6107589e37) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - Move `syntax::parser::ProductionKind` to `syntax::nodes` namespace. + +- [#540](https://github.com/NomicFoundation/slang/pull/540) [`0d04f95`](https://github.com/NomicFoundation/slang/commit/0d04f959bf1f5831c912d5109de3d933cfaa6266) Thanks [@AntonyBlakey](https://github.com/AntonyBlakey)! - Add TokenNode.text to the TS API. + +- [#540](https://github.com/NomicFoundation/slang/pull/540) [`0d04f95`](https://github.com/NomicFoundation/slang/commit/0d04f959bf1f5831c912d5109de3d933cfaa6266) Thanks [@AntonyBlakey](https://github.com/AntonyBlakey)! - Add first pass of Typescript binding to the Cursor API, but no TS Visitor yet. + +- [#545](https://github.com/NomicFoundation/slang/pull/545) [`e73658a`](https://github.com/NomicFoundation/slang/commit/e73658ae4e777e78a01e213f213e2a5dc13e5cba) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - render EBNF grammar on top of each `ProductionKind`, `RuleKind`, and `TokenKind`. + +- [#558](https://github.com/NomicFoundation/slang/pull/558) [`95bbc50`](https://github.com/NomicFoundation/slang/commit/95bbc5025fbf63b8d4e07f7652a70a7f66363db6) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - Correct versioning for `SourceUnitMember` and `ContractMember` children. + +## 0.8.0 + +### Minor Changes + +- [#513](https://github.com/NomicFoundation/slang/pull/513) [`7e01250`](https://github.com/NomicFoundation/slang/commit/7e012501c04e639b54cd150e3736683ee2c2606f) Thanks [@AntonyBlakey](https://github.com/AntonyBlakey)! - Typescript API now has TextIndex and TextRange types that are returned from the appropriate methods rather than tuples. + +### Patch Changes + +- [#527](https://github.com/NomicFoundation/slang/pull/527) [`7ccca87`](https://github.com/NomicFoundation/slang/commit/7ccca87beaa9cb96ad294d1af8a02f115481b71a) Thanks [@AntonyBlakey](https://github.com/AntonyBlakey)! - Fix pratt parser behavior in the face of error correction +- [#531](https://github.com/NomicFoundation/slang/pull/531) [`e3450be4`](https://github.com/NomicFoundation/slang/commit/e3450be4722845bcfce7a9ec3b3046ba6eb6961d) Thanks [@alcuadrado](https://github.com/alcuadrado)! - Make ESM named imports work in Node.js. + +## 0.7.0 + +### Minor Changes + +- [#502](https://github.com/NomicFoundation/slang/pull/502) [`c383238`](https://github.com/NomicFoundation/slang/commit/c383238c1f51157b37ec63bc99e63fb85c1bc224) Thanks [@AntonyBlakey](https://github.com/AntonyBlakey)! - Added error recovery i.e. a CST is _always_ produced, even if there are errors. The erroneous/skipped text is in the CST as a `TokenKind::SKIPPED` token. + +- [#501](https://github.com/NomicFoundation/slang/pull/501) [`cb221fe`](https://github.com/NomicFoundation/slang/commit/cb221fed784e8a2eb59f17907412149c7b415ed8) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - generate typescript string enums for CST kinds + +- [#517](https://github.com/NomicFoundation/slang/pull/517) [`8bd5446`](https://github.com/NomicFoundation/slang/commit/8bd544695a6dd4880a00d0cecf8d13ad79b238d3) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - extract inlined and sub-expressions in language grammar + +- [#518](https://github.com/NomicFoundation/slang/pull/518) [`b3b562b`](https://github.com/NomicFoundation/slang/commit/b3b562be6365fab25b97e54746a7500b9e7bd595) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - fill in missing CST node names + +- [#515](https://github.com/NomicFoundation/slang/pull/515) [`f24e873`](https://github.com/NomicFoundation/slang/commit/f24e873a93cbcef53aad1fa5eed1ea9ab1af1c04) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - switch over the NPM package to use CommonJS modules instead of ES modules. + +- [#498](https://github.com/NomicFoundation/slang/pull/498) [`44f1ff7`](https://github.com/NomicFoundation/slang/commit/44f1ff70100d6e2f8afe54c7ff87e24a8479e4b9) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - flatten unnamed CST nodes into parent nodes + +- [#502](https://github.com/NomicFoundation/slang/pull/502) [`c383238`](https://github.com/NomicFoundation/slang/commit/c383238c1f51157b37ec63bc99e63fb85c1bc224) Thanks [@AntonyBlakey](https://github.com/AntonyBlakey)! - Use the Rowan model for the CST i.e. TokenNodes contain the string content, and RuleNodes contain only the combined _length_ of their children's text. + +- [#499](https://github.com/NomicFoundation/slang/pull/499) [`1582d60`](https://github.com/NomicFoundation/slang/commit/1582d60c7ef81a785db0b9e3cb4d074d9cb6d442) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - preserve correct ranges on empty rule nodes + +- [#500](https://github.com/NomicFoundation/slang/pull/500) [`73ddac9`](https://github.com/NomicFoundation/slang/commit/73ddac9ca972f80aa9a0321de7f94c47b505d7a6) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - inlining CST nodes that offer no additional syntactic information + +- [#512](https://github.com/NomicFoundation/slang/pull/512) [`72dc3d3`](https://github.com/NomicFoundation/slang/commit/72dc3d3d90bc6a02d36836cc1fed17f5be5de2fb) Thanks [@AntonyBlakey](https://github.com/AntonyBlakey)! - Expression productions now correctly wrap the recursive 'calls' in a rule node + +## 0.6.0 + +### Minor Changes + +- [#490](https://github.com/NomicFoundation/slang/pull/490) [`ea8e7e7`](https://github.com/NomicFoundation/slang/commit/ea8e7e771fef7fd9195bcc3004b08fc132c8990d) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - provide API to list supported language versions + +- [#489](https://github.com/NomicFoundation/slang/pull/489) [`15c34a7`](https://github.com/NomicFoundation/slang/commit/15c34a7bb0268bf26eaa6535dd637f73349596c8) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - replace panics with JS exceptions in npm package + +### Patch Changes + +- [#488](https://github.com/NomicFoundation/slang/pull/488) [`d7f171c`](https://github.com/NomicFoundation/slang/commit/d7f171cf1e2da375a7ededd034a62fc6b723d44d) Thanks [@DaniPopes](https://github.com/DaniPopes)! - introduce a `cli` Cargo feature to compile the CLI binary + +## 0.5.0 + +### Minor Changes + +- [#475](https://github.com/NomicFoundation/slang/pull/475) [`0cdfe86`](https://github.com/NomicFoundation/slang/commit/0cdfe86037bfe2f1f8be66a69e8e7d7bdbf06364) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - match TypeScript and Rust API namespaces + +- [#477](https://github.com/NomicFoundation/slang/pull/477) [`13c85a2`](https://github.com/NomicFoundation/slang/commit/13c85a2a3e4e97894d9f24a3e2886a08ffe6e569) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - move expression operators into separate nodes + +- [#481](https://github.com/NomicFoundation/slang/pull/481) [`0269f2b`](https://github.com/NomicFoundation/slang/commit/0269f2b9de3f6711055119e1f40c3f036fe3a81f) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - fix grammar versions of individual keywords + +- [#473](https://github.com/NomicFoundation/slang/pull/473) [`11d8cb0`](https://github.com/NomicFoundation/slang/commit/11d8cb0658e01f16b7afd808f31d1da88e967679) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - upgrade to rust 1.69.0 + +## 0.4.0 + +### Minor Changes + +- [#458](https://github.com/NomicFoundation/slang/pull/458) [`c0fc7e9`](https://github.com/NomicFoundation/slang/commit/c0fc7e95b87eb1ddca4f9e0003136fcbe74f5798) Thanks [@AntonyBlakey](https://github.com/AntonyBlakey)! - Record both character and byte offsets for input positions + +- [#463](https://github.com/NomicFoundation/slang/pull/463) [`0958d6b`](https://github.com/NomicFoundation/slang/commit/0958d6baadba3295df9307e421ddd0a41ef3aaa0) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - use `number` and getters in npm public API + +## 0.3.0 + +### Minor Changes + +- [#457](https://github.com/NomicFoundation/slang/pull/457) [`b7aae2a`](https://github.com/NomicFoundation/slang/commit/b7aae2ad891f940ee764365ac12c75fd1cb47687) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - minor grammar fixes + +- [#453](https://github.com/NomicFoundation/slang/pull/453) [`0f2f9ab`](https://github.com/NomicFoundation/slang/commit/0f2f9abec3f2525640d25bf1f288b769917fbc61) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - move Rust's `syntax::Parser::Language` API to root module + +- [#454](https://github.com/NomicFoundation/slang/pull/454) [`85dec01`](https://github.com/NomicFoundation/slang/commit/85dec0196eafa337065233de03c30d36211b03cf) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - moving to Rust version 1.65.0 + +- [#456](https://github.com/NomicFoundation/slang/pull/456) [`c6d1041`](https://github.com/NomicFoundation/slang/commit/c6d10417da440f15e1c074b7d8b5d13d38e95519) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - expose `ParseError` API + +- [#451](https://github.com/NomicFoundation/slang/pull/451) [`78f633c`](https://github.com/NomicFoundation/slang/commit/78f633cb5977d358b4bcc468151359a4301089b2) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - rename `VisitorExitResponse::StepIn` to `VisitorExitResponse::Continue` + +## 0.2.1 + +### Patch Changes + +- [#444](https://github.com/NomicFoundation/slang/pull/444) [`a858e2c`](https://github.com/NomicFoundation/slang/commit/a858e2c842db3b2183821fb92ed26af6b433e823) Thanks [@AntonyBlakey](https://github.com/AntonyBlakey)! - Fix HexLiteral cannot have NumberUnit + +## 0.2.0 + +### Minor Changes + +- [#435](https://github.com/NomicFoundation/slang/pull/435) [`2a5b193`](https://github.com/NomicFoundation/slang/commit/2a5b1930b20024359fbaf06b6e9748585d7423ff) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - support user defined operators + +### Patch Changes + +- [#416](https://github.com/NomicFoundation/slang/pull/416) [`fb977a5`](https://github.com/NomicFoundation/slang/commit/fb977a52b152a1ce8d8ce92db4bb00fcfb5881c1) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - fix primary expressions parser order + +- [#434](https://github.com/NomicFoundation/slang/pull/434) [`beb3708`](https://github.com/NomicFoundation/slang/commit/beb3708218ec797614ba283a13f1854d5f3c7239) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - fix UnicodeStringLiteral versioning + +- [#430](https://github.com/NomicFoundation/slang/pull/430) [`8b7492e`](https://github.com/NomicFoundation/slang/commit/8b7492e65ec7261176e444dca2563a82603b43b0) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - update READMEs with links to packages and user guides. + +- [#425](https://github.com/NomicFoundation/slang/pull/425) [`9b49b3d`](https://github.com/NomicFoundation/slang/commit/9b49b3d827536e707d78a6bc349fc82698237b75) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - add user guides to rust crate and npm packages. + +- [#432](https://github.com/NomicFoundation/slang/pull/432) [`1d1a8bb`](https://github.com/NomicFoundation/slang/commit/1d1a8bb5503c510a470bb99a18632c3e51a587ec) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - fix FunctionCallOptions versioning + +- [#427](https://github.com/NomicFoundation/slang/pull/427) [`1103916`](https://github.com/NomicFoundation/slang/commit/11039163ac3a3b66a74fa85683bde1c380a519f4) Thanks [@AntonyBlakey](https://github.com/AntonyBlakey)! - fix VariableDeclarationStatement versioning + +## 0.1.1 + +### Patch Changes + +- [#412](https://github.com/NomicFoundation/slang/pull/412) [`9cac1a04`](https://github.com/NomicFoundation/slang/commit/9cac1a04670fa870c15cee1bd20e0e78c1d213db) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - publish npm packages + +## 0.1.0 + +### Minor Changes + +- [#396](https://github.com/NomicFoundation/slang/pull/396) [`621b338`](https://github.com/NomicFoundation/slang/commit/621b33838c74415c46ab157205068008e05c5b9b) Thanks [@OmarTawfik](https://github.com/OmarTawfik)! - Initial release. diff --git a/crates/codegen/runtime/npm/Cargo.toml b/crates/codegen/runtime/npm/Cargo.toml new file mode 100644 index 0000000000..90e2e4ddf3 --- /dev/null +++ b/crates/codegen/runtime/npm/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "codegen_runtime_npm" +version.workspace = true +rust-version.workspace = true +edition.workspace = true +publish = false + +description = "TypeScript runtime copied over by codegen" + +[build-dependencies] +anyhow = { workspace = true } +codegen_runtime_generator = { workspace = true } +infra_utils = { workspace = true } diff --git a/crates/codegen/runtime/npm/LICENSE b/crates/codegen/runtime/npm/LICENSE new file mode 100644 index 0000000000..84f9adf0ae --- /dev/null +++ b/crates/codegen/runtime/npm/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Nomic Foundation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/codegen/runtime/npm/README.md b/crates/codegen/runtime/npm/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/codegen/runtime/npm/build.rs b/crates/codegen/runtime/npm/build.rs new file mode 100644 index 0000000000..12cb8a6e02 --- /dev/null +++ b/crates/codegen/runtime/npm/build.rs @@ -0,0 +1,6 @@ +use anyhow::Result; +use codegen_runtime_generator::OutputLanguage; + +fn main() -> Result<()> { + OutputLanguage::Npm.generate_stubs() +} diff --git a/crates/codegen/runtime/npm/package.json b/crates/codegen/runtime/npm/package.json new file mode 100644 index 0000000000..3dc3a1eac7 --- /dev/null +++ b/crates/codegen/runtime/npm/package.json @@ -0,0 +1,17 @@ +{ + "name": "@slang-private/slang-codegen-runtime", + "private": true, + "type": "commonjs", + "devDependencies": { + "@napi-rs/cli": "2.18.3" + }, + "napi": { + "triples": { + "defaults": false, + "additional": [] + } + }, + "engines": { + "node": ">= 10" + } +} diff --git a/crates/codegen/runtime/npm/src/lib.rs b/crates/codegen/runtime/npm/src/lib.rs new file mode 100644 index 0000000000..e9b55221e1 --- /dev/null +++ b/crates/codegen/runtime/npm/src/lib.rs @@ -0,0 +1 @@ +//! Dummy file to allow the crate to be included in the dependency graph: diff --git a/crates/codegen/runtime/npm/src/runtime/ast/ast_types.ts.jinja2 b/crates/codegen/runtime/npm/src/runtime/ast/ast_types.ts.jinja2 new file mode 100644 index 0000000000..413426f8cb --- /dev/null +++ b/crates/codegen/runtime/npm/src/runtime/ast/ast_types.ts.jinja2 @@ -0,0 +1,189 @@ +{%- if rendering_in_stubs -%} + export class StubAst {} +{%- else -%} + import * as assert from "node:assert"; + import { ast_internal } from "../../napi-bindings/generated"; + import { RuleNode, TokenNode } from "../../cst"; + import { RuleKind, TokenKind } from "../../kinds"; + + /* + * Sequences: + */ + + {% for sequence in model.ast.sequences %} + export class {{ sequence.name }} { + private readonly fetch = once(() => { + const [ + {%- for field in sequence.fields %} + ${{ field.name | camel_case }}, + {%- endfor %} + ] = ast_internal.selectSequence(this.cst); + + return { + {%- for field in sequence.fields %} + {{ field.name | camel_case }}: + {%- if field.is_optional -%} + ${{ field.name | camel_case }} === null ? undefined : + {%- endif -%} + {%- if field.is_terminal -%} + ${{ field.name | camel_case }} as TokenNode, + {%- else -%} + new {{ field.reference }}(${{ field.name | camel_case }} as RuleNode), + {%- endif -%} + {% endfor -%} + }; + }); + + public constructor(public readonly cst: RuleNode) { + assertKind(this.cst.kind, RuleKind.{{ sequence.name }}); + } + + {% for field in sequence.fields %} + public get {{ field.name | camel_case }}() : + {%- if field.is_terminal -%} + TokenNode + {%- else -%} + {{ field.reference }} + {%- endif -%} + {%- if field.is_optional -%} + | undefined + {%- endif -%} + { + return this.fetch().{{ field.name | camel_case }}; + } + {% endfor %} + } + {% endfor %} + + /* + * Choices: + */ + + {% for choice in model.ast.choices %} + export class {{ choice.name }} { + {%- set variant_types = choice.non_terminals -%} + {%- if choice.terminals | length > 0 -%} + {%- set variant_types = variant_types | concat(with = "TokenNode") -%} + {%- endif -%} + {%- set variant_types = variant_types | join(sep = " | ") -%} + + private readonly fetch: () => {{ variant_types }} = once(() => { + const variant = ast_internal.selectChoice(this.cst); + + switch (variant.kind) { + {%- for non_terminal in choice.non_terminals %} + case RuleKind.{{ non_terminal }}: + return new {{ non_terminal }}(variant as RuleNode); + {%- endfor %} + + {% if choice.terminals | length > 0 %} + {%- for terminal in choice.terminals %} + case TokenKind.{{ terminal }}: + {%- endfor %} + return variant as TokenNode; + {%- endif %} + + default: + assert.fail(`Unexpected variant: ${variant.kind}`); + } + }); + + public constructor(public readonly cst: RuleNode) { + assertKind(this.cst.kind, RuleKind.{{ choice.name }}); + } + + public get variant(): {{ variant_types }} { + return this.fetch(); + } + } + {% endfor %} + + /* + * Repeated: + */ + + {% for repeated in model.ast.repeated %} + export class {{ repeated.name }} { + private readonly fetch = once(() => { + const items = ast_internal.selectRepeated(this.cst); + + {%- if repeated.is_terminal -%} + return items as TokenNode[]; + {%- else -%} + return items.map((item) => new {{ repeated.reference }}(item as RuleNode)); + {%- endif -%} + }); + + public constructor(public readonly cst: RuleNode) { + assertKind(this.cst.kind, RuleKind.{{ repeated.name }}); + } + + public get items(): + {%- if repeated.is_terminal -%} + readonly TokenNode[] + {%- else -%} + readonly {{ repeated.reference }}[] + {%- endif -%} + { + return this.fetch(); + } + } + {% endfor %} + + /* + * Separated: + */ + + {% for separated in model.ast.separated %} + export class {{ separated.name }} { + private readonly fetch = once(() => { + const [items, separators] = ast_internal.selectSeparated(this.cst); + + return { + {%- if separated.is_terminal -%} + items: items as TokenNode[], + {%- else -%} + items: items.map((item) => new {{ separated.reference }}(item as RuleNode)), + {%- endif -%} + separators: separators as TokenNode[], + }; + }); + + public constructor(public readonly cst: RuleNode) { + assertKind(this.cst.kind, RuleKind.{{ separated.name }}); + } + + public get items(): + {%- if separated.is_terminal -%} + readonly TokenNode[] + {%- else -%} + readonly {{ separated.reference }}[] + {%- endif -%} + { + return this.fetch().items; + } + + public get separators(): readonly TokenNode[] { + return this.fetch().separators; + } + } + {% endfor %} + + /* + * Helpers: + */ + + function once(factory: () => T): () => T { + let value: T | undefined; + return () => { + if (value === undefined) { + value = factory(); + } + return value; + }; + } + + function assertKind(actual: RuleKind, expected: RuleKind): void { + assert.equal(actual, expected, `${expected} can only be initialized with a CST node of the same kind.`); + } +{%- endif %} diff --git a/crates/codegen/runtime/npm/src/runtime/ast/generated/ast_types.ts b/crates/codegen/runtime/npm/src/runtime/ast/generated/ast_types.ts new file mode 100644 index 0000000000..f9b60beb3a --- /dev/null +++ b/crates/codegen/runtime/npm/src/runtime/ast/generated/ast_types.ts @@ -0,0 +1,3 @@ +// This file is generated automatically by infrastructure scripts. Please don't edit by hand. + +export class StubAst {} diff --git a/crates/solidity/outputs/npm/package/src/ast/index.ts b/crates/codegen/runtime/npm/src/runtime/ast/index.ts similarity index 100% rename from crates/solidity/outputs/npm/package/src/ast/index.ts rename to crates/codegen/runtime/npm/src/runtime/ast/index.ts diff --git a/crates/testlang/outputs/npm/package/src/cst/index.ts b/crates/codegen/runtime/npm/src/runtime/cst/index.ts similarity index 85% rename from crates/testlang/outputs/npm/package/src/cst/index.ts rename to crates/codegen/runtime/npm/src/runtime/cst/index.ts index eb7c1bf986..a156422a81 100644 --- a/crates/testlang/outputs/npm/package/src/cst/index.ts +++ b/crates/codegen/runtime/npm/src/runtime/cst/index.ts @@ -1,4 +1,4 @@ -import * as generated from "../generated"; +import * as generated from "../napi-bindings/generated"; export type Node = generated.cst.Node; diff --git a/crates/solidity/outputs/npm/package/src/cursor/index.ts b/crates/codegen/runtime/npm/src/runtime/cursor/index.ts similarity index 62% rename from crates/solidity/outputs/npm/package/src/cursor/index.ts rename to crates/codegen/runtime/npm/src/runtime/cursor/index.ts index e64291609e..60200f5a56 100644 --- a/crates/solidity/outputs/npm/package/src/cursor/index.ts +++ b/crates/codegen/runtime/npm/src/runtime/cursor/index.ts @@ -1,4 +1,4 @@ -import * as generated from "../generated"; +import * as generated from "../napi-bindings/generated"; export const Cursor = generated.cursor.Cursor; export type Cursor = generated.cursor.Cursor; diff --git a/crates/solidity/outputs/npm/package/src/index.ts b/crates/codegen/runtime/npm/src/runtime/index.ts similarity index 100% rename from crates/solidity/outputs/npm/package/src/index.ts rename to crates/codegen/runtime/npm/src/runtime/index.ts diff --git a/crates/testlang/outputs/npm/package/src/kinds/index.ts b/crates/codegen/runtime/npm/src/runtime/kinds/index.ts similarity index 84% rename from crates/testlang/outputs/npm/package/src/kinds/index.ts rename to crates/codegen/runtime/npm/src/runtime/kinds/index.ts index 0c1d84aa75..e4929309eb 100644 --- a/crates/testlang/outputs/npm/package/src/kinds/index.ts +++ b/crates/codegen/runtime/npm/src/runtime/kinds/index.ts @@ -1,4 +1,4 @@ -import * as generated from "../generated"; +import * as generated from "../napi-bindings/generated"; export const RuleKind = generated.kinds.RuleKind; export type RuleKind = generated.kinds.RuleKind; diff --git a/crates/solidity/outputs/npm/package/src/language/index.ts b/crates/codegen/runtime/npm/src/runtime/language/index.ts similarity index 65% rename from crates/solidity/outputs/npm/package/src/language/index.ts rename to crates/codegen/runtime/npm/src/runtime/language/index.ts index 19a20f8fa9..fab1a1d18a 100644 --- a/crates/solidity/outputs/npm/package/src/language/index.ts +++ b/crates/codegen/runtime/npm/src/runtime/language/index.ts @@ -1,4 +1,4 @@ -import * as generated from "../generated"; +import * as generated from "../napi-bindings/generated"; export const Language = generated.language.Language; export type Language = generated.language.Language; diff --git a/crates/codegen/runtime/npm/src/runtime/napi-bindings/generated/index.d.ts b/crates/codegen/runtime/npm/src/runtime/napi-bindings/generated/index.d.ts new file mode 100644 index 0000000000..822c31129f --- /dev/null +++ b/crates/codegen/runtime/npm/src/runtime/napi-bindings/generated/index.d.ts @@ -0,0 +1,152 @@ +// This file is generated automatically by infrastructure scripts. Please don't edit by hand. + +// Slang License: https://github.com/NomicFoundation/slang/blob/main/LICENSE +// NAPI-RS License: https://github.com/napi-rs/napi-rs/blob/main/LICENSE + +/* tslint:disable */ +/* eslint-disable */ + +/* auto-generated by NAPI-RS */ + +export namespace kinds { + export enum RuleKind { + Stub1 = "Stub1", + Stub2 = "Stub2", + Stub3 = "Stub3", + } + export enum NodeLabel { + Item = "Item", + Variant = "Variant", + Separator = "Separator", + Operand = "Operand", + LeftOperand = "LeftOperand", + RightOperand = "RightOperand", + LeadingTrivia = "LeadingTrivia", + TrailingTrivia = "TrailingTrivia", + Stub1 = "Stub1", + Stub2 = "Stub2", + Stub3 = "Stub3", + } + export enum TokenKind { + SKIPPED = "SKIPPED", + Stub1 = "Stub1", + Stub2 = "Stub2", + Stub3 = "Stub3", + } +} +export namespace language { + export class Language { + constructor(version: string); + get version(): string; + static supportedVersions(): Array; + parse(kind: kinds.RuleKind, input: string): parse_output.ParseOutput; + } +} +export namespace cst { + export enum NodeType { + Rule = "Rule", + Token = "Token", + } + export class RuleNode { + get type(): NodeType.Rule; + get kind(): kinds.RuleKind; + get textLength(): text_index.TextIndex; + children(): Array; + createCursor(textOffset: text_index.TextIndex): cursor.Cursor; + /** Serialize the token node to JSON. */ + toJSON(): string; + unparse(): string; + } + export class TokenNode { + get type(): NodeType.Token; + get kind(): kinds.TokenKind; + get textLength(): text_index.TextIndex; + get text(): string; + /** + * Serialize the token node to JSON. + * + * This method is intended for debugging purposes and may not be stable. + */ + toJSON(): string; + createCursor(textOffset: text_index.TextIndex): cursor.Cursor; + } +} +export namespace cursor { + export class Cursor { + reset(): void; + complete(): void; + clone(): Cursor; + spawn(): Cursor; + get isCompleted(): boolean; + node(): cst.Node; + get label(): kinds.NodeLabel; + get textOffset(): text_index.TextIndex; + get textRange(): text_index.TextRange; + get depth(): number; + ancestors(): Array; + goToNext(): boolean; + goToNextNonDescendent(): boolean; + goToPrevious(): boolean; + goToParent(): boolean; + goToFirstChild(): boolean; + goToLastChild(): boolean; + goToNthChild(childNumber: number): boolean; + goToNextSibling(): boolean; + goToPreviousSibling(): boolean; + goToNextToken(): boolean; + goToNextTokenWithKind(kind: kinds.TokenKind): boolean; + goToNextTokenWithKinds(kinds: Array): boolean; + goToNextRule(): boolean; + goToNextRuleWithKind(kind: kinds.RuleKind): boolean; + goToNextRuleWithKinds(kinds: Array): boolean; + query(queries: Array): query.QueryResultIterator; + } +} +export namespace parse_error { + export class ParseError { + get textRange(): text_index.TextRange; + toErrorReport(sourceId: string, source: string, withColor: boolean): string; + } +} +export namespace parse_output { + export class ParseOutput { + tree(): cst.Node; + errors(): Array; + get isValid(): boolean; + /** Creates a cursor that starts at the root of the parse tree. */ + createTreeCursor(): cursor.Cursor; + } +} +export namespace query { + export interface QueryResult { + queryNumber: number; + bindings: { [key: string]: cursor.Cursor[] }; + } + export class Query { + static parse(text: string): Query; + } + export class QueryResultIterator { + next(): QueryResult | null; + } +} +export namespace text_index { + export interface TextIndex { + utf8: number; + utf16: number; + char: number; + } + export interface TextRange { + start: TextIndex; + end: TextIndex; + } +} +export namespace ast_internal { + export function selectSequence(node: cst.RuleNode): Array; + export function selectChoice(node: cst.RuleNode): cst.Node; + export function selectRepeated(node: cst.RuleNode): Array; + export function selectSeparated(node: cst.RuleNode): [Array, Array]; +} + +export namespace cst { + export type Node = RuleNode | TokenNode; +} diff --git a/crates/codegen/runtime/npm/src/runtime/napi-bindings/generated/index.js b/crates/codegen/runtime/npm/src/runtime/napi-bindings/generated/index.js new file mode 100644 index 0000000000..abd6ab450c --- /dev/null +++ b/crates/codegen/runtime/npm/src/runtime/napi-bindings/generated/index.js @@ -0,0 +1,304 @@ +// This file is generated automatically by infrastructure scripts. Please don't edit by hand. + +// Slang License: https://github.com/NomicFoundation/slang/blob/main/LICENSE +// NAPI-RS License: https://github.com/napi-rs/napi-rs/blob/main/LICENSE + +// @ts-nocheck + +/* tslint:disable */ +/* eslint-disable */ +/* prettier-ignore */ + +/* auto-generated by NAPI-RS */ + +const { existsSync, readFileSync } = require('fs') +const { join } = require("path"); + +const { platform, arch } = process; + +let nativeBinding = null; +let localFileExisted = false; +let loadError = null; + +function isMusl() { + // For Node 10 + if (!process.report || typeof process.report.getReport !== "function") { + try { + const lddPath = require("child_process").execSync("which ldd").toString().trim(); + return readFileSync(lddPath, "utf8").includes("musl"); + } catch (e) { + return true; + } + } else { + const { glibcVersionRuntime } = process.report.getReport().header; + return !glibcVersionRuntime; + } +} + +switch (platform) { + case "android": + switch (arch) { + case "arm64": + localFileExisted = existsSync(join(__dirname, "index.android-arm64.node")); + try { + if (localFileExisted) { + nativeBinding = require("./index.android-arm64.node"); + } else { + nativeBinding = require("@slang-private/slang-codegen-runtime-android-arm64"); + } + } catch (e) { + loadError = e; + } + break; + case "arm": + localFileExisted = existsSync(join(__dirname, "index.android-arm-eabi.node")); + try { + if (localFileExisted) { + nativeBinding = require("./index.android-arm-eabi.node"); + } else { + nativeBinding = require("@slang-private/slang-codegen-runtime-android-arm-eabi"); + } + } catch (e) { + loadError = e; + } + break; + default: + throw new Error(`Unsupported architecture on Android ${arch}`); + } + break; + case "win32": + switch (arch) { + case "x64": + localFileExisted = existsSync(join(__dirname, "index.win32-x64-msvc.node")); + try { + if (localFileExisted) { + nativeBinding = require("./index.win32-x64-msvc.node"); + } else { + nativeBinding = require("@slang-private/slang-codegen-runtime-win32-x64-msvc"); + } + } catch (e) { + loadError = e; + } + break; + case "ia32": + localFileExisted = existsSync(join(__dirname, "index.win32-ia32-msvc.node")); + try { + if (localFileExisted) { + nativeBinding = require("./index.win32-ia32-msvc.node"); + } else { + nativeBinding = require("@slang-private/slang-codegen-runtime-win32-ia32-msvc"); + } + } catch (e) { + loadError = e; + } + break; + case "arm64": + localFileExisted = existsSync(join(__dirname, "index.win32-arm64-msvc.node")); + try { + if (localFileExisted) { + nativeBinding = require("./index.win32-arm64-msvc.node"); + } else { + nativeBinding = require("@slang-private/slang-codegen-runtime-win32-arm64-msvc"); + } + } catch (e) { + loadError = e; + } + break; + default: + throw new Error(`Unsupported architecture on Windows: ${arch}`); + } + break; + case "darwin": + localFileExisted = existsSync(join(__dirname, "index.darwin-universal.node")); + try { + if (localFileExisted) { + nativeBinding = require("./index.darwin-universal.node"); + } else { + nativeBinding = require("@slang-private/slang-codegen-runtime-darwin-universal"); + } + break; + } catch {} + switch (arch) { + case "x64": + localFileExisted = existsSync(join(__dirname, "index.darwin-x64.node")); + try { + if (localFileExisted) { + nativeBinding = require("./index.darwin-x64.node"); + } else { + nativeBinding = require("@slang-private/slang-codegen-runtime-darwin-x64"); + } + } catch (e) { + loadError = e; + } + break; + case "arm64": + localFileExisted = existsSync(join(__dirname, "index.darwin-arm64.node")); + try { + if (localFileExisted) { + nativeBinding = require("./index.darwin-arm64.node"); + } else { + nativeBinding = require("@slang-private/slang-codegen-runtime-darwin-arm64"); + } + } catch (e) { + loadError = e; + } + break; + default: + throw new Error(`Unsupported architecture on macOS: ${arch}`); + } + break; + case "freebsd": + if (arch !== "x64") { + throw new Error(`Unsupported architecture on FreeBSD: ${arch}`); + } + localFileExisted = existsSync(join(__dirname, "index.freebsd-x64.node")); + try { + if (localFileExisted) { + nativeBinding = require("./index.freebsd-x64.node"); + } else { + nativeBinding = require("@slang-private/slang-codegen-runtime-freebsd-x64"); + } + } catch (e) { + loadError = e; + } + break; + case "linux": + switch (arch) { + case "x64": + if (isMusl()) { + localFileExisted = existsSync(join(__dirname, "index.linux-x64-musl.node")); + try { + if (localFileExisted) { + nativeBinding = require("./index.linux-x64-musl.node"); + } else { + nativeBinding = require("@slang-private/slang-codegen-runtime-linux-x64-musl"); + } + } catch (e) { + loadError = e; + } + } else { + localFileExisted = existsSync(join(__dirname, "index.linux-x64-gnu.node")); + try { + if (localFileExisted) { + nativeBinding = require("./index.linux-x64-gnu.node"); + } else { + nativeBinding = require("@slang-private/slang-codegen-runtime-linux-x64-gnu"); + } + } catch (e) { + loadError = e; + } + } + break; + case "arm64": + if (isMusl()) { + localFileExisted = existsSync(join(__dirname, "index.linux-arm64-musl.node")); + try { + if (localFileExisted) { + nativeBinding = require("./index.linux-arm64-musl.node"); + } else { + nativeBinding = require("@slang-private/slang-codegen-runtime-linux-arm64-musl"); + } + } catch (e) { + loadError = e; + } + } else { + localFileExisted = existsSync(join(__dirname, "index.linux-arm64-gnu.node")); + try { + if (localFileExisted) { + nativeBinding = require("./index.linux-arm64-gnu.node"); + } else { + nativeBinding = require("@slang-private/slang-codegen-runtime-linux-arm64-gnu"); + } + } catch (e) { + loadError = e; + } + } + break; + case "arm": + if (isMusl()) { + localFileExisted = existsSync(join(__dirname, "index.linux-arm-musleabihf.node")); + try { + if (localFileExisted) { + nativeBinding = require("./index.linux-arm-musleabihf.node"); + } else { + nativeBinding = require("@slang-private/slang-codegen-runtime-linux-arm-musleabihf"); + } + } catch (e) { + loadError = e; + } + } else { + localFileExisted = existsSync(join(__dirname, "index.linux-arm-gnueabihf.node")); + try { + if (localFileExisted) { + nativeBinding = require("./index.linux-arm-gnueabihf.node"); + } else { + nativeBinding = require("@slang-private/slang-codegen-runtime-linux-arm-gnueabihf"); + } + } catch (e) { + loadError = e; + } + } + break; + case "riscv64": + if (isMusl()) { + localFileExisted = existsSync(join(__dirname, "index.linux-riscv64-musl.node")); + try { + if (localFileExisted) { + nativeBinding = require("./index.linux-riscv64-musl.node"); + } else { + nativeBinding = require("@slang-private/slang-codegen-runtime-linux-riscv64-musl"); + } + } catch (e) { + loadError = e; + } + } else { + localFileExisted = existsSync(join(__dirname, "index.linux-riscv64-gnu.node")); + try { + if (localFileExisted) { + nativeBinding = require("./index.linux-riscv64-gnu.node"); + } else { + nativeBinding = require("@slang-private/slang-codegen-runtime-linux-riscv64-gnu"); + } + } catch (e) { + loadError = e; + } + } + break; + case "s390x": + localFileExisted = existsSync(join(__dirname, "index.linux-s390x-gnu.node")); + try { + if (localFileExisted) { + nativeBinding = require("./index.linux-s390x-gnu.node"); + } else { + nativeBinding = require("@slang-private/slang-codegen-runtime-linux-s390x-gnu"); + } + } catch (e) { + loadError = e; + } + break; + default: + throw new Error(`Unsupported architecture on Linux: ${arch}`); + } + break; + default: + throw new Error(`Unsupported OS: ${platform}, architecture: ${arch}`); +} + +if (!nativeBinding) { + if (loadError) { + throw loadError; + } + throw new Error(`Failed to load native binding`); +} + +const { kinds, language, cst, cursor, parse_error, parse_output, query, text_index, ast_internal } = nativeBinding; + +module.exports.kinds = kinds; +module.exports.language = language; +module.exports.cst = cst; +module.exports.cursor = cursor; +module.exports.parse_error = parse_error; +module.exports.parse_output = parse_output; +module.exports.query = query; +module.exports.text_index = text_index; +module.exports.ast_internal = ast_internal; diff --git a/crates/testlang/outputs/npm/package/src/parse_error/index.ts b/crates/codegen/runtime/npm/src/runtime/parse_error/index.ts similarity index 67% rename from crates/testlang/outputs/npm/package/src/parse_error/index.ts rename to crates/codegen/runtime/npm/src/runtime/parse_error/index.ts index 3374ff4655..a391e4bdbb 100644 --- a/crates/testlang/outputs/npm/package/src/parse_error/index.ts +++ b/crates/codegen/runtime/npm/src/runtime/parse_error/index.ts @@ -1,4 +1,4 @@ -import * as generated from "../generated"; +import * as generated from "../napi-bindings/generated"; export const ParseError = generated.parse_error.ParseError; export type ParseError = generated.parse_error.ParseError; diff --git a/crates/testlang/outputs/npm/package/src/parse_output/index.ts b/crates/codegen/runtime/npm/src/runtime/parse_output/index.ts similarity index 68% rename from crates/testlang/outputs/npm/package/src/parse_output/index.ts rename to crates/codegen/runtime/npm/src/runtime/parse_output/index.ts index 90d25489d9..9f5e3d4428 100644 --- a/crates/testlang/outputs/npm/package/src/parse_output/index.ts +++ b/crates/codegen/runtime/npm/src/runtime/parse_output/index.ts @@ -1,4 +1,4 @@ -import * as generated from "../generated"; +import * as generated from "../napi-bindings/generated"; export const ParseOutput = generated.parse_output.ParseOutput; export type ParseOutput = generated.parse_output.ParseOutput; diff --git a/crates/codegen/runtime/npm/src/runtime/query/index.ts b/crates/codegen/runtime/npm/src/runtime/query/index.ts new file mode 100644 index 0000000000..1e6b4e3834 --- /dev/null +++ b/crates/codegen/runtime/npm/src/runtime/query/index.ts @@ -0,0 +1,7 @@ +import * as generated from "../napi-bindings/generated"; + +export const Query = generated.query.Query; +export type Query = generated.query.Query; + +export const QueryResultIterator = generated.query.QueryResultIterator; +export type QueryResultIterator = generated.query.QueryResultIterator; diff --git a/crates/testlang/outputs/npm/package/src/text_index/index.ts b/crates/codegen/runtime/npm/src/runtime/text_index/index.ts similarity index 66% rename from crates/testlang/outputs/npm/package/src/text_index/index.ts rename to crates/codegen/runtime/npm/src/runtime/text_index/index.ts index 6a981827fa..1c8ee2dfd9 100644 --- a/crates/testlang/outputs/npm/package/src/text_index/index.ts +++ b/crates/codegen/runtime/npm/src/runtime/text_index/index.ts @@ -1,4 +1,4 @@ -import * as generated from "../generated"; +import * as generated from "../napi-bindings/generated"; export type TextIndex = generated.text_index.TextIndex; diff --git a/crates/codegen/runtime/npm/tsconfig.json b/crates/codegen/runtime/npm/tsconfig.json new file mode 100644 index 0000000000..2ad4db51a5 --- /dev/null +++ b/crates/codegen/runtime/npm/tsconfig.json @@ -0,0 +1,20 @@ +{ + "extends": "../../../../tsconfig.json", + + "include": ["./src/**/*.ts"], + + "compilerOptions": { + /* Modules */ + "module": "CommonJS", + "moduleResolution": "Node", + "rootDir": "./src/runtime", + + /* Language and Environment */ + "target": "ES2016", + + /* Projects */ + "composite": true, + "incremental": true, + "tsBuildInfoFile": "target/tsc/.tsbuildinfo" + } +} diff --git a/crates/codegen/spec/src/generators/navigation.rs b/crates/codegen/spec/src/generators/navigation.rs index ef941fccbc..f7dd4d2b9c 100644 --- a/crates/codegen/spec/src/generators/navigation.rs +++ b/crates/codegen/spec/src/generators/navigation.rs @@ -2,7 +2,7 @@ use std::fmt::Write; use std::path::Path; use anyhow::Result; -use infra_utils::codegen::CodegenWriteOnly; +use infra_utils::codegen::CodegenFileSystem; pub enum SpecEntry { Dir(SpecDir), @@ -32,7 +32,7 @@ impl SpecDir { self.entries.push(SpecEntry::Page(page)); } - pub fn write_to_disk(&self, codegen: &mut CodegenWriteOnly, parent_dir: &Path) -> Result<()> { + pub fn write_to_disk(&self, fs: &mut CodegenFileSystem, parent_dir: &Path) -> Result<()> { let SpecDir { title, slug, @@ -60,7 +60,7 @@ impl SpecDir { writeln!(index, "- [{child_title}](./{child_slug}/index.md)")?; writeln!(navigation, "- [{child_title}](./{child_slug}/)")?; - child.write_to_disk(codegen, ¤t_dir)?; + child.write_to_disk(fs, ¤t_dir)?; } SpecEntry::Page(SpecPage { title: child_title, @@ -70,13 +70,13 @@ impl SpecDir { writeln!(index, "- [{child_title}](./{child_slug}.md)")?; writeln!(navigation, "- [{child_title}](./{child_slug}.md)")?; - codegen.write_file(current_dir.join(format!("{child_slug}.md")), contents)?; + fs.write_file(current_dir.join(format!("{child_slug}.md")), contents)?; } } } - codegen.write_file(current_dir.join("index.md"), index)?; - codegen.write_file(current_dir.join(".navigation.md"), navigation)?; + fs.write_file(current_dir.join("index.md"), index)?; + fs.write_file(current_dir.join(".navigation.md"), navigation)?; Ok(()) } } diff --git a/crates/codegen/spec/src/lib.rs b/crates/codegen/spec/src/lib.rs index 611d6109f6..85ca47da4d 100644 --- a/crates/codegen/spec/src/lib.rs +++ b/crates/codegen/spec/src/lib.rs @@ -14,7 +14,7 @@ use std::rc::Rc; use anyhow::Result; use codegen_language_definition::model::Language; -use infra_utils::codegen::CodegenWriteOnly; +use infra_utils::codegen::CodegenFileSystem; use crate::generators::grammar_ebnf::generate_grammar_ebnf; use crate::generators::navigation::{SpecDir, SpecPage}; @@ -26,14 +26,14 @@ pub struct Spec; impl Spec { pub fn generate(language: Rc, output_dir: &Path) -> Result<()> { - let model = SpecModel::build(language); - - let mut codegen = CodegenWriteOnly::new()?; + let mut fs = CodegenFileSystem::new(&language.documentation_dir)?; + let model = SpecModel::build(language); let public_dir = Self::generate_public_dir(&model)?; - public_dir.write_to_disk(&mut codegen, output_dir)?; - codegen.write_file( + public_dir.write_to_disk(&mut fs, output_dir)?; + + fs.write_file( output_dir.join("grammar.ebnf"), generate_grammar_ebnf(&model)?, )?; diff --git a/crates/codegen/testing/src/cst_output.rs b/crates/codegen/testing/src/cst_output.rs index e52767acbd..6fa0c88eff 100644 --- a/crates/codegen/testing/src/cst_output.rs +++ b/crates/codegen/testing/src/cst_output.rs @@ -5,7 +5,7 @@ use std::path::Path; use anyhow::{bail, Result}; use codegen_language_definition::model::Language; use inflector::Inflector; -use infra_utils::codegen::{Codegen, CodegenReadWrite}; +use infra_utils::codegen::CodegenFileSystem; use infra_utils::paths::{FileWalker, PathExtensions}; pub fn generate_cst_output_tests( @@ -15,18 +15,13 @@ pub fn generate_cst_output_tests( ) -> Result<()> { let parser_tests = collect_parser_tests(data_dir)?; - let mut codegen = Codegen::read_write(data_dir)?; + let mut fs = CodegenFileSystem::new(data_dir)?; - generate_mod_file( - language, - &mut codegen, - &output_dir.join("mod.rs"), - &parser_tests, - )?; + generate_mod_file(language, &mut fs, &output_dir.join("mod.rs"), &parser_tests)?; for (parser_name, test_names) in &parser_tests { generate_unit_test_file( - &mut codegen, + &mut fs, parser_name, test_names, &output_dir.join(format!("{0}.rs", parser_name.to_snake_case())), @@ -77,7 +72,7 @@ fn collect_parser_tests(data_dir: &Path) -> Result>, ) -> Result<()> { @@ -114,11 +109,11 @@ fn generate_mod_file( ", ); - codegen.write_file(mod_file_path, contents) + fs.write_file(mod_file_path, contents) } fn generate_unit_test_file( - codegen: &mut CodegenReadWrite, + fs: &mut CodegenFileSystem, parser_name: &str, test_names: &BTreeSet, unit_test_file_path: &Path, @@ -149,5 +144,5 @@ fn generate_unit_test_file( " ); - codegen.write_file(unit_test_file_path, contents) + fs.write_file(unit_test_file_path, contents) } diff --git a/crates/infra/cli/Cargo.toml b/crates/infra/cli/Cargo.toml index 747e110f1d..c5579d6a44 100644 --- a/crates/infra/cli/Cargo.toml +++ b/crates/infra/cli/Cargo.toml @@ -18,6 +18,8 @@ regex = { workspace = true } semver = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +strum = { workspace = true } +strum_macros = { workspace = true } tempfile = { workspace = true } toml = { workspace = true } diff --git a/crates/infra/cli/src/commands/check/mod.rs b/crates/infra/cli/src/commands/check/mod.rs index e79c201c56..bd99d0d7d6 100644 --- a/crates/infra/cli/src/commands/check/mod.rs +++ b/crates/infra/cli/src/commands/check/mod.rs @@ -2,6 +2,7 @@ use anyhow::Result; use clap::{Parser, ValueEnum}; use infra_utils::cargo::CargoWorkspace; use infra_utils::terminal::Terminal; +use strum::IntoEnumIterator; use crate::toolchains::mkdocs::Mkdocs; use crate::toolchains::napi::{NapiCompiler, NapiProfile, NapiResolver}; @@ -62,8 +63,10 @@ fn check_rustdoc() -> Result<()> { } fn check_npm() -> Result<()> { - NapiCompiler::run(&NapiResolver::testlang(), NapiProfile::Debug)?; - NapiCompiler::run(&NapiResolver::solidity(), NapiProfile::Debug)?; + for resolver in NapiResolver::iter() { + NapiCompiler::run(resolver, NapiProfile::Debug)?; + } + Ok(()) } diff --git a/crates/infra/cli/src/commands/lint/mod.rs b/crates/infra/cli/src/commands/lint/mod.rs index 5ab383b9cb..9d3ee7ee08 100644 --- a/crates/infra/cli/src/commands/lint/mod.rs +++ b/crates/infra/cli/src/commands/lint/mod.rs @@ -87,13 +87,10 @@ fn run_markdown_link_check() -> Result<()> { } fn run_markdown_lint() -> Result<()> { - let markdown_files = FileWalker::from_repo_root() - .find(["**/*.md"])? - .into_iter() - .map(|path| { - println!("{}", path.display()); - path - }); + let markdown_files = FileWalker::from_repo_root().find(["**/*.md"])?.map(|path| { + println!("{}", path.display()); + path + }); let mut command = Command::new("markdownlint").flag("--dot"); @@ -120,7 +117,6 @@ fn run_rustfmt() -> Result<()> { fn run_shellcheck() -> Result<()> { let bash_files = FileWalker::from_repo_root() .find(["scripts/**"])? - .into_iter() .map(|path| { println!("{}", path.display()); path @@ -142,7 +138,6 @@ fn run_yamllint() -> Result<()> { let yaml_files = FileWalker::from_repo_root() .find(["**/*.yml"])? - .into_iter() .map(|path| { println!("{}", path.display()); path diff --git a/crates/infra/cli/src/commands/publish/changesets/mod.rs b/crates/infra/cli/src/commands/publish/changesets/mod.rs index a498401497..9337b1d780 100644 --- a/crates/infra/cli/src/commands/publish/changesets/mod.rs +++ b/crates/infra/cli/src/commands/publish/changesets/mod.rs @@ -15,7 +15,7 @@ use crate::toolchains::napi::{NapiCli, NapiConfig, NapiResolver}; /// However, NAPI platform-specific packages cannot be added to the workspace, because other platforms will fail "npm install". /// So we have to bump the versions over ourselves anyways. pub fn publish_changesets() -> Result<()> { - let resolver = NapiResolver::solidity(); + let resolver = NapiResolver::Solidity; let package_dir = resolver.main_package_dir(); let package_version = NapiConfig::local_version(&package_dir)?; @@ -46,7 +46,7 @@ pub fn publish_changesets() -> Result<()> { // Update platform-specific packages: - NapiCli::prepublish(&resolver)?; + NapiCli::prepublish(resolver)?; // Format the updated package files: diff --git a/crates/infra/cli/src/commands/publish/npm/mod.rs b/crates/infra/cli/src/commands/publish/npm/mod.rs index 76581fbc3f..6c52c523b4 100644 --- a/crates/infra/cli/src/commands/publish/npm/mod.rs +++ b/crates/infra/cli/src/commands/publish/npm/mod.rs @@ -11,16 +11,16 @@ use crate::toolchains::napi::{ }; pub fn publish_npm(dry_run: DryRun) -> Result<()> { - let resolver = NapiResolver::solidity(); + let resolver = NapiResolver::Solidity; - NapiCompiler::run(&resolver, NapiProfile::Release)?; + NapiCompiler::run(resolver, NapiProfile::Release)?; // Publish platform-specific packages first, as the main package now depends on their latest version: for platform_dir in resolver.platforms_dir().collect_children()? { let platform = platform_dir.unwrap_name().to_owned(); publish_package( - &resolver, + resolver, &platform_dir, &NapiPackageKind::Platform(platform), dry_run, @@ -30,11 +30,11 @@ pub fn publish_npm(dry_run: DryRun) -> Result<()> { // Then publish the main package, that depends on the previously published platform-specific packages: let package_dir = resolver.main_package_dir(); - publish_package(&resolver, &package_dir, &NapiPackageKind::Main, dry_run) + publish_package(resolver, &package_dir, &NapiPackageKind::Main, dry_run) } fn publish_package( - resolver: &NapiResolver, + resolver: NapiResolver, package_dir: &Path, kind: &NapiPackageKind, dry_run: DryRun, diff --git a/crates/solidity/outputs/npm/package/templates/index.d.ts.jinja2 b/crates/infra/cli/src/toolchains/napi/bindings/index.d.ts.jinja2 similarity index 100% rename from crates/solidity/outputs/npm/package/templates/index.d.ts.jinja2 rename to crates/infra/cli/src/toolchains/napi/bindings/index.d.ts.jinja2 diff --git a/crates/testlang/outputs/npm/package/templates/index.d.ts.jinja2 b/crates/infra/cli/src/toolchains/napi/bindings/index.js.jinja2 similarity index 77% rename from crates/testlang/outputs/npm/package/templates/index.d.ts.jinja2 rename to crates/infra/cli/src/toolchains/napi/bindings/index.js.jinja2 index 570cb75214..5e8c6b9b36 100644 --- a/crates/testlang/outputs/npm/package/templates/index.d.ts.jinja2 +++ b/crates/infra/cli/src/toolchains/napi/bindings/index.js.jinja2 @@ -1,12 +1,12 @@ // Slang License: https://github.com/NomicFoundation/slang/blob/main/LICENSE // NAPI-RS License: https://github.com/napi-rs/napi-rs/blob/main/LICENSE +// @ts-nocheck + {# Automatically generated by NAPI: #} {{ contents }} {# Manual Additions: #} -export namespace cst { - export type Node = RuleNode | TokenNode; -} +{# Any manual additions should go here... #} diff --git a/crates/infra/cli/src/toolchains/napi/cli.rs b/crates/infra/cli/src/toolchains/napi/cli.rs index 9c9fdb3a2d..79db67f443 100644 --- a/crates/infra/cli/src/toolchains/napi/cli.rs +++ b/crates/infra/cli/src/toolchains/napi/cli.rs @@ -23,13 +23,13 @@ pub struct NapiCli; impl NapiCli { pub fn build( - resolver: &NapiResolver, + resolver: NapiResolver, output_dir: impl AsRef, target: &BuildTarget, ) -> Result { let output_dir = output_dir.as_ref(); let package_dir = resolver.main_package_dir(); - let crate_dir = resolver.crate_dir(); + let crate_dir = resolver.rust_crate_dir(); let mut command = Command::new("napi"); @@ -100,7 +100,7 @@ impl NapiCli { }) } - pub fn prepublish(resolver: &NapiResolver) -> Result<()> { + pub fn prepublish(resolver: NapiResolver) -> Result<()> { let package_dir = resolver.main_package_dir(); let platforms_dir = resolver.platforms_dir(); diff --git a/crates/infra/cli/src/toolchains/napi/compiler.rs b/crates/infra/cli/src/toolchains/napi/compiler.rs index c892790bad..aba0167d37 100644 --- a/crates/infra/cli/src/toolchains/napi/compiler.rs +++ b/crates/infra/cli/src/toolchains/napi/compiler.rs @@ -2,7 +2,7 @@ use std::path::{Path, PathBuf}; use anyhow::Result; use infra_utils::cargo::CargoWorkspace; -use infra_utils::codegen::Codegen; +use infra_utils::codegen::CodegenTemplates; use infra_utils::commands::Command; use infra_utils::paths::PathExtensions; use serde::Serialize; @@ -22,7 +22,7 @@ pub enum NapiProfile { pub struct NapiCompiler; impl NapiCompiler { - pub fn run(resolver: &NapiResolver, profile: NapiProfile) -> Result<()> { + pub fn run(resolver: NapiResolver, profile: NapiProfile) -> Result<()> { match profile { NapiProfile::Debug => { // Compiles the default target for local development @@ -50,7 +50,7 @@ impl NapiCompiler { } } -fn compile_all_targets(resolver: &NapiResolver) -> Result> { +fn compile_all_targets(resolver: NapiResolver) -> Result> { let targets = NapiConfig::list_all_targets(resolver)?; assert_ne!( @@ -80,7 +80,7 @@ fn compile_all_targets(resolver: &NapiResolver) -> Result> { Ok(node_binaries) } -fn compile_target(resolver: &NapiResolver, target: &BuildTarget) -> Result { +fn compile_target(resolver: NapiResolver, target: &BuildTarget) -> Result { let output_dir = resolver.napi_output_dir(target); std::fs::create_dir_all(&output_dir)?; @@ -89,25 +89,26 @@ fn compile_target(resolver: &NapiResolver, target: &BuildTarget) -> Result Result<()> { - let templates_dir = resolver.templates_dir(); +fn process_generated_files(resolver: NapiResolver, napi_output: &NapiCliOutput) -> Result<()> { + let templates_dir = + CargoWorkspace::locate_source_crate("infra_cli")?.join("src/toolchains/napi/bindings"); - let mut codegen = Codegen::read_write(&templates_dir)?; + let mut templates = CodegenTemplates::new(&templates_dir)?; for source in &napi_output.source_files { let file_name = source.unwrap_name(); let contents = source.read_to_string()?; - let destination_path = resolver.generated_dir().join(file_name); + let destination_path = resolver.bindings_dir().join(file_name); let template_path = templates_dir.join(format!("{file_name}.jinja2")); - codegen.render( - LicenseHeaderTemplate { contents }, + templates.render_single( &template_path, + LicenseHeaderModel { contents }, destination_path, )?; } @@ -115,7 +116,7 @@ fn process_generated_files(resolver: &NapiResolver, napi_output: &NapiCliOutput) Ok(()) } -fn compile_root_package(resolver: &NapiResolver, node_binary: Option<&Path>) -> Result<()> { +fn compile_root_package(resolver: NapiResolver, node_binary: Option<&Path>) -> Result<()> { let package_dir = resolver.main_package_dir(); let output_dir = resolver.npm_output_dir(&NapiPackageKind::Main); @@ -135,19 +136,19 @@ fn compile_root_package(resolver: &NapiResolver, node_binary: Option<&Path>) -> std::fs::copy(source, destination)?; } - let generated_dir = resolver.generated_dir(); - let generated_output_dir = resolver.generated_output_dir(); + let bindings_dir = resolver.bindings_dir(); + let bindings_output_dir = resolver.bindings_output_dir(); - std::fs::create_dir_all(&generated_output_dir)?; + std::fs::create_dir_all(&bindings_output_dir)?; - for child in generated_dir.collect_children()? { - let destination = generated_output_dir.join(child.unwrap_name()); + for child in bindings_dir.collect_children()? { + let destination = bindings_output_dir.join(child.unwrap_name()); std::fs::copy(child, destination)?; } if let Some(node_binary) = node_binary { - let destination = generated_output_dir.join(node_binary.unwrap_name()); + let destination = bindings_output_dir.join(node_binary.unwrap_name()); std::fs::copy(node_binary, destination)?; } @@ -155,7 +156,7 @@ fn compile_root_package(resolver: &NapiResolver, node_binary: Option<&Path>) -> Ok(()) } -fn compile_platform_packages(resolver: &NapiResolver, node_binaries: &[PathBuf]) -> Result<()> { +fn compile_platform_packages(resolver: NapiResolver, node_binaries: &[PathBuf]) -> Result<()> { for platform_dir in resolver.platforms_dir().collect_children()? { let platform = platform_dir.unwrap_name(); let package_kind = NapiPackageKind::Platform(platform.to_owned()); diff --git a/crates/infra/cli/src/toolchains/napi/config.rs b/crates/infra/cli/src/toolchains/napi/config.rs index 654522220c..e76ae581a2 100644 --- a/crates/infra/cli/src/toolchains/napi/config.rs +++ b/crates/infra/cli/src/toolchains/napi/config.rs @@ -54,7 +54,7 @@ impl NapiConfig { Ok(Version::parse(version.trim())?) } - pub fn list_all_targets(resolver: &NapiResolver) -> Result> { + pub fn list_all_targets(resolver: NapiResolver) -> Result> { let package = load_package(&resolver.main_package_dir())?; let triples = package @@ -71,7 +71,7 @@ impl NapiConfig { } /// Returns the target glibc version for the GNU targets. - pub fn target_glibc(resolver: &NapiResolver) -> Result { + pub fn target_glibc(resolver: NapiResolver) -> Result { let package = load_package(&resolver.main_package_dir())?; Ok(package diff --git a/crates/infra/cli/src/toolchains/napi/glibc.rs b/crates/infra/cli/src/toolchains/napi/glibc.rs index 7fbd26ee34..aff31260df 100644 --- a/crates/infra/cli/src/toolchains/napi/glibc.rs +++ b/crates/infra/cli/src/toolchains/napi/glibc.rs @@ -70,7 +70,7 @@ impl PartialEq for ZigGlibcVersion { /// This is necessary to retain extension compatibility with as many systems as possible: /// . pub fn ensure_correct_glibc_for_vscode( - resolver: &NapiResolver, + resolver: NapiResolver, output_dir: &Path, target: &BuildTarget, ) -> Result<()> { diff --git a/crates/infra/cli/src/toolchains/napi/resolver.rs b/crates/infra/cli/src/toolchains/napi/resolver.rs index 7772a13534..c29805b44c 100644 --- a/crates/infra/cli/src/toolchains/napi/resolver.rs +++ b/crates/infra/cli/src/toolchains/napi/resolver.rs @@ -1,6 +1,7 @@ use std::path::PathBuf; use infra_utils::cargo::CargoWorkspace; +use strum_macros::EnumIter; use crate::toolchains::napi::cli::BuildTarget; @@ -9,51 +10,57 @@ pub enum NapiPackageKind { Platform(String), } -pub struct NapiResolver { - rust_crate: &'static str, - npm_package: &'static str, +#[derive(Clone, Copy, EnumIter)] +pub enum NapiResolver { + Codegen, + Solidity, + Testlang, } impl NapiResolver { - pub fn testlang() -> Self { - Self { - rust_crate: "slang_testlang_node_addon", - npm_package: "testlang_npm_package", + pub fn rust_crate_name(self) -> &'static str { + match self { + Self::Codegen => "codegen_runtime_node_addon", + Self::Solidity => "slang_solidity_node_addon", + Self::Testlang => "slang_testlang_node_addon", } } - pub fn solidity() -> Self { - Self { - rust_crate: "slang_solidity_node_addon", - npm_package: "solidity_npm_package", + pub fn main_package_name(self) -> &'static str { + match self { + Self::Codegen => "codegen_runtime_npm", + Self::Solidity => "solidity_npm_package", + Self::Testlang => "testlang_npm_package", } } - pub fn rust_crate_name(&self) -> &'static str { - self.rust_crate + pub fn rust_crate_dir(self) -> PathBuf { + CargoWorkspace::locate_source_crate(self.rust_crate_name()).unwrap() } - pub fn crate_dir(&self) -> PathBuf { - CargoWorkspace::locate_source_crate(self.rust_crate).unwrap() + pub fn main_package_dir(self) -> PathBuf { + CargoWorkspace::locate_source_crate(self.main_package_name()).unwrap() } - pub fn main_package_dir(&self) -> PathBuf { - CargoWorkspace::locate_source_crate(self.npm_package).unwrap() - } - - pub fn templates_dir(&self) -> PathBuf { - self.main_package_dir().join("templates") + pub fn platforms_dir(self) -> PathBuf { + self.main_package_dir().join("platforms") } - pub fn generated_dir(&self) -> PathBuf { - self.main_package_dir().join("src/generated") + pub fn bindings_dir(self) -> PathBuf { + self.main_package_dir().join(match self { + // Source templates: + Self::Codegen => "src/runtime/napi-bindings/generated", + // Generated Languages: + Self::Solidity | Self::Testlang => "src/generated/napi-bindings/generated", + }) } - pub fn platforms_dir(&self) -> PathBuf { - self.main_package_dir().join("platforms") + pub fn bindings_output_dir(self) -> PathBuf { + self.npm_output_dir(&NapiPackageKind::Main) + .join("napi-bindings/generated") } - pub fn napi_output_dir(&self, target: &BuildTarget) -> PathBuf { + pub fn napi_output_dir(self, target: &BuildTarget) -> PathBuf { self.main_package_dir() .join("target/napi") .join(match target { @@ -62,7 +69,7 @@ impl NapiResolver { }) } - pub fn npm_output_dir(&self, kind: &NapiPackageKind) -> PathBuf { + pub fn npm_output_dir(self, kind: &NapiPackageKind) -> PathBuf { self.main_package_dir().join("target/npm").join(match kind { NapiPackageKind::Main => { // __SLANG_NPM_PACKAGE_MAIN_OUTPUT_DIR__ (keep in sync) @@ -71,9 +78,4 @@ impl NapiResolver { NapiPackageKind::Platform(platform) => platform, }) } - - pub fn generated_output_dir(&self) -> PathBuf { - self.npm_output_dir(&NapiPackageKind::Main) - .join("generated") - } } diff --git a/crates/infra/utils/Cargo.toml b/crates/infra/utils/Cargo.toml index 659dd764ae..5f93560d52 100644 --- a/crates/infra/utils/Cargo.toml +++ b/crates/infra/utils/Cargo.toml @@ -7,6 +7,7 @@ publish = false [dependencies] anyhow = { workspace = true } +ariadne = { workspace = true } cargo-emit = { workspace = true } console = { workspace = true } ignore = { workspace = true } diff --git a/crates/infra/utils/src/codegen/common/file_system.rs b/crates/infra/utils/src/codegen/common/file_system.rs deleted file mode 100644 index cc8d020dd9..0000000000 --- a/crates/infra/utils/src/codegen/common/file_system.rs +++ /dev/null @@ -1,49 +0,0 @@ -use std::path::Path; - -use anyhow::{bail, Context, Result}; -use cargo_emit::warning; - -use crate::cargo::CargoWorkspace; -use crate::codegen::common::formatting::format_source_file; -use crate::paths::PathExtensions; - -pub fn delete_file(file_path: &Path) -> Result<()> { - std::fs::remove_file(file_path) - .with_context(|| format!("Failed to delete source file: {file_path:?}")) -} - -pub fn write_file(file_path: &Path, contents: &str) -> Result<()> { - std::fs::create_dir_all(file_path.unwrap_parent()) - .with_context(|| format!("Cannot create parent directory of: {file_path:?}"))?; - - let formatted = format_source_file(file_path, contents)?; - - // To respect Cargo incrementability, don't touch the file if it is already up to date. - if file_path.exists() && formatted == file_path.read_to_string()? { - return Ok(()); - } - - if CargoWorkspace::is_running_inside_build_scripts() { - warning!("Updating: {}", file_path.strip_repo_root()?.unwrap_str()); - } - - file_path.write_string(formatted) -} - -pub fn verify_file(file_path: &Path, contents: &str) -> Result<()> { - let formatted = format_source_file(file_path, contents)?; - - if !file_path.exists() { - bail!("Generated file does not exist: {file_path:?}"); - } - - let existing_contents = file_path.read_to_string()?; - - similar_asserts::assert_eq!( - formatted, - existing_contents, - "Generated file is out of date: {file_path:?}" - ); - - Ok(()) -} diff --git a/crates/infra/utils/src/codegen/common/mod.rs b/crates/infra/utils/src/codegen/common/mod.rs deleted file mode 100644 index 8c0f6133e5..0000000000 --- a/crates/infra/utils/src/codegen/common/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod file_system; -pub mod formatting; -pub mod tera; diff --git a/crates/infra/utils/src/codegen/common/tera.rs b/crates/infra/utils/src/codegen/common/tera.rs deleted file mode 100644 index d2e7234ce7..0000000000 --- a/crates/infra/utils/src/codegen/common/tera.rs +++ /dev/null @@ -1,74 +0,0 @@ -#![allow(clippy::unnecessary_wraps)] - -use std::collections::HashMap; -use std::path::Path; - -use anyhow::{anyhow, Result}; -use inflector::Inflector; -use serde_json::Value; -use tera::Tera; - -use crate::paths::PathExtensions; - -pub fn create_tera_instance(templates_glob: &Path) -> Result { - let mut instance = Tera::new(templates_glob.unwrap_str()).map_err(|error| { - // Stringify and wrap with newlines, as the default error serialization - // does not render the error report correctly: - anyhow!("\n{error}\n") - })?; - - instance.register_filter("camel_case", camel_case); - instance.register_filter("snake_case", snake_case); - instance.register_filter("pascal_case", pascal_case); - - instance.register_function("panic", panic); - - instance.autoescape_on(vec![]); // disable autoescaping - - Ok(instance) -} - -fn camel_case(value: &Value, args: &HashMap) -> tera::Result { - assert_eq!(args.len(), 0, "Expected no arguments"); - - Ok(Value::String( - value - .as_str() - .expect("Expected a string value") - .to_camel_case(), - )) -} - -fn snake_case(value: &Value, args: &HashMap) -> tera::Result { - assert_eq!(args.len(), 0, "Expected no arguments"); - - Ok(Value::String( - value - .as_str() - .expect("Expected a string value") - .to_snake_case(), - )) -} - -fn pascal_case(value: &Value, args: &HashMap) -> tera::Result { - assert_eq!(args.len(), 0, "Expected no arguments"); - - Ok(Value::String( - value - .as_str() - .expect("Expected a string value") - .to_pascal_case(), - )) -} - -fn panic(args: &HashMap) -> tera::Result { - assert_eq!(args.len(), 1, "Expected one argument"); - - let message = args - .get("message") - .expect("Expected a 'message' argument") - .as_str() - .expect("Expected a string message"); - - panic!("{message}"); -} diff --git a/crates/infra/utils/src/codegen/file_system.rs b/crates/infra/utils/src/codegen/file_system.rs new file mode 100644 index 0000000000..7946d4f73a --- /dev/null +++ b/crates/infra/utils/src/codegen/file_system.rs @@ -0,0 +1,159 @@ +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +use anyhow::{bail, Context, Result}; +use cargo_emit::{rerun_if_changed, warning}; + +use crate::cargo::CargoWorkspace; +use crate::codegen::formatting::format_source_file; +use crate::github::GitHub; +use crate::paths::{FileWalker, PathExtensions}; + +pub struct CodegenFileSystem { + input_dir: PathBuf, + generated_dirs: HashSet, + generated_files: HashSet, +} + +impl CodegenFileSystem { + pub fn new(input_dir: impl Into) -> Result { + let input_dir = input_dir.into(); + + let input_dir = input_dir + .canonicalize() + .with_context(|| format!("Directory doesn't exist: {input_dir:?}"))?; + + Ok(Self { + input_dir, + generated_dirs: HashSet::new(), + generated_files: HashSet::new(), + }) + } + + pub fn read_file(&mut self, file_path: impl AsRef) -> Result { + let file_path = file_path.as_ref(); + + if !file_path.starts_with(&self.input_dir) { + bail!( + "File path {:?} is not under input directory {:?}", + file_path, + self.input_dir + ); + } + + file_path.read_to_string() + } + + pub fn write_file( + &mut self, + file_path: impl AsRef, + contents: impl AsRef, + ) -> Result<()> { + let file_path = file_path.as_ref(); + + self.mark_generated_file(file_path)?; + + return if GitHub::is_running_in_ci() { + verify_contents(file_path, contents.as_ref()) + } else { + write_contents(file_path, contents.as_ref()) + }; + } + + pub fn mark_generated_file(&mut self, file_path: impl AsRef) -> Result<()> { + let file_path = file_path.as_ref(); + + if file_path.strip_repo_root().is_err() { + bail!("Generated file is outside repository: {file_path:?}"); + } + + if !self.generated_files.insert(file_path.to_owned()) { + bail!("File was generated twice: {file_path:?}") + } + + self.generated_dirs.insert(file_path.generated_dir()?); + + Ok(()) + } + + pub fn copy_file( + &mut self, + source_path: impl AsRef, + destination_path: impl AsRef, + ) -> Result<()> { + // Go through read_file() API, to record the correct metadata for it. + let contents = self.read_file(source_path)?; + + // Go through write_file() API, to only touch/update the file if it changed. + self.write_file(destination_path, contents)?; + + Ok(()) + } +} + +impl Drop for CodegenFileSystem { + fn drop(&mut self) { + if CargoWorkspace::is_running_inside_build_scripts() { + rerun_if_changed!(self.input_dir.unwrap_str()); + + for generated_dir in &self.generated_dirs { + rerun_if_changed!(generated_dir.unwrap_str()); + } + } + + for generated_dir in &self.generated_dirs { + for file_path in FileWalker::from_directory(generated_dir) + .find_all() + .unwrap() + { + if self.generated_files.contains(&file_path) { + continue; + } + + if GitHub::is_running_in_ci() { + panic!("File was not generated in this context: {file_path:?}"); + } else { + std::fs::remove_file(&file_path) + .with_context(|| format!("Failed to delete source file: {file_path:?}")) + .unwrap(); + } + } + } + } +} + +fn write_contents(file_path: &Path, contents: &str) -> Result<()> { + std::fs::create_dir_all(file_path.unwrap_parent()) + .with_context(|| format!("Cannot create parent directory of: {file_path:?}"))?; + + let formatted = format_source_file(file_path, contents)?; + + // To respect Cargo incrementability, don't touch the file if it is already up to date. + if file_path.exists() && formatted == file_path.read_to_string()? { + return Ok(()); + } + + if CargoWorkspace::is_running_inside_build_scripts() { + warning!("Updating: {}", file_path.strip_repo_root()?.unwrap_str()); + } + + file_path.write_string(formatted) +} + +fn verify_contents(file_path: &Path, contents: &str) -> Result<()> { + let formatted = format_source_file(file_path, contents)?; + + if !file_path.exists() { + bail!("Generated file does not exist: {file_path:?}"); + } + + let existing_contents = file_path.read_to_string()?; + + similar_asserts::assert_eq!( + formatted, + existing_contents, + "Generated file is out of date: {file_path:?}" + ); + + Ok(()) +} diff --git a/crates/infra/utils/src/codegen/common/formatting.rs b/crates/infra/utils/src/codegen/formatting.rs similarity index 100% rename from crates/infra/utils/src/codegen/common/formatting.rs rename to crates/infra/utils/src/codegen/formatting.rs diff --git a/crates/infra/utils/src/codegen/mod.rs b/crates/infra/utils/src/codegen/mod.rs index 079d9352a9..66f311e2c4 100644 --- a/crates/infra/utils/src/codegen/mod.rs +++ b/crates/infra/utils/src/codegen/mod.rs @@ -1,21 +1,7 @@ -mod common; -mod read_write; -mod write_only; +mod file_system; +mod formatting; +mod templates; +mod tera; -use std::path::PathBuf; - -use anyhow::Result; -pub use read_write::*; -pub use write_only::*; - -pub struct Codegen; - -impl Codegen { - pub fn write_only() -> Result { - CodegenWriteOnly::new() - } - - pub fn read_write(input_dir: impl Into) -> Result { - CodegenReadWrite::new(input_dir) - } -} +pub use file_system::CodegenFileSystem; +pub use templates::CodegenTemplates; diff --git a/crates/infra/utils/src/codegen/read_write.rs b/crates/infra/utils/src/codegen/read_write.rs deleted file mode 100644 index 8cfff97ee8..0000000000 --- a/crates/infra/utils/src/codegen/read_write.rs +++ /dev/null @@ -1,109 +0,0 @@ -use std::path::{Path, PathBuf}; - -use anyhow::{bail, Context, Result}; -use cargo_emit::rerun_if_changed; -use serde::Serialize; -use tera::Tera; - -use crate::cargo::CargoWorkspace; -use crate::codegen::common::tera::create_tera_instance; -use crate::codegen::write_only::CodegenWriteOnly; -use crate::paths::PathExtensions; - -pub struct CodegenReadWrite { - writer: CodegenWriteOnly, - tera: Tera, - input_dir: PathBuf, -} - -impl CodegenReadWrite { - pub fn new(input_dir: impl Into) -> Result { - let input_dir = input_dir.into(); - - let input_dir = input_dir - .canonicalize() - .with_context(|| format!("Directory doesn't exist: {input_dir:?}"))?; - - let writer = CodegenWriteOnly::new()?; - - let tera = create_tera_instance(&input_dir.join("**/*.jinja2"))?; - - Ok(Self { - writer, - tera, - input_dir, - }) - } - - pub fn write_file( - &mut self, - file_path: impl AsRef, - contents: impl AsRef, - ) -> Result<()> { - self.writer.write_file(file_path, contents) - } - - pub fn read_file(&mut self, file_path: impl AsRef) -> Result { - let file_path = file_path.as_ref(); - - if !file_path.starts_with(&self.input_dir) { - bail!( - "File path {:?} is not under input directory {:?}", - file_path, - self.input_dir - ); - } - - if file_path.generated_dir().is_ok() { - bail!("Cannot read from a generated directory: {file_path:?}"); - } - - file_path.read_to_string() - } - - pub fn copy_file( - &mut self, - source_path: impl AsRef, - destination_path: impl AsRef, - ) -> Result<()> { - // Go through read_file() API, to record the correct metadata for it. - let contents = self.read_file(source_path)?; - - // Go through write_file() API, to only touch/update the file if it changed. - self.write_file(destination_path, contents)?; - - Ok(()) - } - - pub fn render( - &mut self, - context: impl Serialize, - template_path: impl AsRef, - output_path: impl AsRef, - ) -> Result<()> { - let templates_dir = &self.input_dir; - let template_path = template_path.as_ref(); - - // tera expects these paths to be relative: - let template_path = template_path - .strip_prefix(templates_dir) - .with_context(|| { - format!("Template path {template_path:?} must be under templates directory {templates_dir:?}") - })? - .unwrap_str(); - - let context = tera::Context::from_serialize(context)?; - let rendered = self.tera.render(template_path, &context)?; - - self.write_file(output_path, rendered) - } -} - -impl Drop for CodegenReadWrite { - fn drop(&mut self) { - // Don't pollute test output with markers meant for build scripts: - if CargoWorkspace::is_running_inside_build_scripts() { - rerun_if_changed!(self.input_dir.unwrap_str()); - } - } -} diff --git a/crates/infra/utils/src/codegen/templates.rs b/crates/infra/utils/src/codegen/templates.rs new file mode 100644 index 0000000000..8236206755 --- /dev/null +++ b/crates/infra/utils/src/codegen/templates.rs @@ -0,0 +1,187 @@ +//! Here, `input_dir` will have three kinds of files: +//! +//! 1) `input_dir/template.rs.jinja2` -> a template file to render +//! 2) `input_dir/generated/template.rs` -> the stub file rendered by this template in the source crate +//! 3) `input_dir/other.rs` -> other source files to be copied as-is +//! +//! For `render_standalone_template()`: +//! It will render each template (#1) to its stub file (#2) under the same directory. +//! +//! For `render_runtime_output()`: +//! The first pass below will render the template (#1) to `output_dir/generated/template.rs`, and skip the stub (#2). +//! Then the second pass will copy the remaining sources (#3) to `output_dir/other.rs` as-is. + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +use anyhow::{bail, Result}; +use regex::Regex; +use serde::Serialize; +use tera::Tera; + +use crate::codegen::tera::{create_tera_instance, TEMPLATES_GLOB}; +use crate::codegen::CodegenFileSystem; +use crate::paths::{FileWalker, PathExtensions}; + +pub struct CodegenTemplates { + input_dir: PathBuf, + fs: CodegenFileSystem, + tera: Tera, +} + +impl CodegenTemplates { + pub fn new(input_dir: impl Into) -> Result { + let input_dir = input_dir.into(); + let fs = CodegenFileSystem::new(&input_dir)?; + let tera = create_tera_instance(&input_dir)?; + + Ok(Self { + input_dir, + fs, + tera, + }) + } + + pub fn render_stubs(&mut self, model: impl Serialize) -> Result<()> { + let context = tera::Context::from_serialize(model)?; + + for template_path in FileWalker::from_directory(&self.input_dir).find([TEMPLATES_GLOB])? { + let stub_path = Self::get_stub_path(&template_path); + + self.render_aux(&template_path, &context, &stub_path)?; + } + + Ok(()) + } + + pub fn render_directory( + &mut self, + model: impl Serialize, + output_dir: impl AsRef, + ) -> Result<()> { + let context = tera::Context::from_serialize(model)?; + let output_dir = output_dir.as_ref(); + + let mut template_files = HashSet::new(); + + for template_path in FileWalker::from_directory(&self.input_dir).find([TEMPLATES_GLOB])? { + let stub_path = Self::get_stub_path(&template_path); + + let output_path = output_dir.join(stub_path.strip_prefix(&self.input_dir)?); + + self.render_aux(&template_path, &context, &output_path)?; + + assert!(template_files.insert(template_path)); + assert!(template_files.insert(stub_path)); + } + + for source_path in FileWalker::from_directory(&self.input_dir).find_all()? { + // Skip templates already handled in the first pass: + if template_files.contains(&source_path) { + continue; + } + + let output_path = output_dir.join(source_path.strip_prefix(&self.input_dir)?); + + // Preserve the source of otherwise-generated files: + if source_path.generated_dir().is_ok() { + self.fs.mark_generated_file(output_path)?; + } else { + self.fs.copy_file(source_path, output_path)?; + } + } + + Ok(()) + } + + pub fn render_single( + &mut self, + template_path: impl AsRef, + model: impl Serialize, + output_path: impl AsRef, + ) -> Result<()> { + let context = tera::Context::from_serialize(model)?; + + self.render_aux(template_path.as_ref(), &context, output_path.as_ref()) + } + + fn render_aux( + &mut self, + template_path: &Path, + context: &tera::Context, + output_path: &Path, + ) -> Result<()> { + // tera expects the template path to be relative to the input directory: + let template_relative_path = template_path.strip_prefix(&self.input_dir)?.unwrap_str(); + + let rendered = match self.tera.render(template_relative_path, context) { + Ok(rendered) => rendered, + Err(error) => { + self.try_report_error(error)?; + + // Exit process, to skip printing irrelevant backtraces: + #[allow(clippy::exit)] + std::process::exit(1); + } + }; + + self.fs.write_file(output_path, rendered) + } + + fn get_stub_path(template_path: &Path) -> PathBuf { + let without_extension = template_path.with_extension(""); + let template_name = without_extension.unwrap_name(); + + template_path + .unwrap_parent() + .join("generated") + .join(template_name) + } + + fn try_report_error(&self, error: tera::Error) -> Result<()> { + // Unfortunately, tera's error details are hidden in the debug output. + // It should be fixed in v2: https://github.com/Keats/tera/issues/885 + let error_details = format!("{error:?}"); + + let Some(captures) = Regex::new( + r"Variable `(?.+)` not found in context while rendering '(?