From 8fee17db5990c5a54fc99499fd1a1cca1536b12c Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Mon, 20 Apr 2020 10:38:30 +1000 Subject: [PATCH] jl_value_t as parser options for improved flexibility --- base/meta.jl | 28 +++++++++++++--------------- src/ast.c | 24 ++++++++++++++---------- src/frontend.c | 9 ++++----- src/julia.h | 11 ----------- src/julia_internal.h | 10 +++++++--- 5 files changed, 38 insertions(+), 44 deletions(-) diff --git a/base/meta.jl b/base/meta.jl index 8bb02bb7e4509..ca8333ad9f24c 100644 --- a/base/meta.jl +++ b/base/meta.jl @@ -150,19 +150,24 @@ struct ParseError <: Exception end function _jl_parse(text::AbstractString, filename::AbstractString, - pos::Integer, rule::Integer) + pos::Integer; options...) if pos < 1 || pos > ncodeunits(text) + 1 throw(BoundsError(text, pos)) end - # Technically only need pointers to UTF-8 buffers here. For now converting - # to a plain String is the easy way to ensure that. + # Technically only need pointers to UTF-8 buffers here. Converting to a + # plain String is the easy way to ensure that. filename = String(filename) text = String(text) + options = values(options) + if length(options) == 1 && haskey(options, :rule) + # Hack: Pass rule as a symbol for benefit of jl_fl_parse + options = options.rule + end # Call into the parser which can be replaced globally during bootstrap with # jl_set_parser ex,pos = ccall(:jl_parse, Any, - (Ptr{UInt8}, Csize_t, Ptr{UInt8}, Csize_t, Csize_t, Cint), - text, sizeof(text), filename, sizeof(filename), pos-1, rule) + (Ptr{UInt8}, Csize_t, Ptr{UInt8}, Csize_t, Csize_t, Any), + text, sizeof(text), filename, sizeof(filename), pos-1, options) # internally, pos is a zero-based byte offset - convert back. ex, pos+1 end @@ -190,12 +195,7 @@ julia> Meta.parse("x = 3, y = 5", 5) function parse(str::AbstractString, pos::Integer; greedy::Bool=true, raise::Bool=true, depwarn::Bool=true) filename = "none" - JL_PARSE_ATOM = 1 - JL_PARSE_STATEMENT = 2 - rule = greedy ? JL_PARSE_STATEMENT : JL_PARSE_ATOM - # For now, assume all parser warnings are depwarns - # TODO: remove parser-depwarn; parser no longer emits warnings. - ex, pos = _jl_parse(str, "none", pos, rule) + ex, pos = _jl_parse(str, "none", pos; rule=greedy ? :statement : :atom) if raise && isa(ex,Expr) && ex.head === :error throw(ParseError(ex.args[1])) end @@ -240,13 +240,11 @@ function parse(str::AbstractString; raise::Bool=true, depwarn::Bool=true) end function parseatom(text::AbstractString, pos::Integer; filename="none") - JL_PARSE_ATOM = 1 - return _jl_parse(text, filename, pos, JL_PARSE_ATOM) + return _jl_parse(text, filename, pos; rule=:atom) end function parseall(text::AbstractString; filename="none") - JL_PARSE_ALL = 3 - ex,_ = _jl_parse(text, filename, 1, JL_PARSE_ALL) + ex,_ = _jl_parse(text, filename, 1; rule=:all) return ex end diff --git a/src/ast.c b/src/ast.c index a181e45077c03..8d1eca67086d3 100644 --- a/src/ast.c +++ b/src/ast.c @@ -67,6 +67,7 @@ jl_sym_t *gc_preserve_begin_sym; jl_sym_t *gc_preserve_end_sym; jl_sym_t *coverageeffect_sym; jl_sym_t *escape_sym; jl_sym_t *aliasscope_sym; jl_sym_t *popaliasscope_sym; jl_sym_t *optlevel_sym; +jl_sym_t *atom_sym; jl_sym_t *statement_sym; jl_sym_t *all_sym; static uint8_t flisp_system_image[] = { #include @@ -402,6 +403,9 @@ void jl_init_common_symbols(void) coverageeffect_sym = jl_symbol("code_coverage_effect"); aliasscope_sym = jl_symbol("aliasscope"); popaliasscope_sym = jl_symbol("popaliasscope"); + atom_sym = jl_symbol("atom"); + statement_sym = jl_symbol("statement"); + all_sym = jl_symbol("all"); } JL_DLLEXPORT void jl_lisp_prompt(void) @@ -785,7 +789,7 @@ static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v) // content to `filename`. Return an svec of (parse_result, final_pos) JL_DLLEXPORT jl_value_t *jl_fl_parse(const char* text, size_t text_len, const char* filename, size_t filename_len, - size_t offset, int rule) + size_t offset, jl_value_t *options) { JL_TIMING(PARSING); if (offset > text_len) { @@ -793,8 +797,12 @@ JL_DLLEXPORT jl_value_t *jl_fl_parse(const char* text, size_t text_len, JL_GC_PUSH1(&textstr); jl_bounds_error(textstr, jl_box_long(offset)); } - else if (offset != 0 && rule == JL_PARSE_ALL) { - jl_error("Partial parsing not support by top level grammar rule"); + jl_sym_t *rule = (jl_sym_t*)options; + if (rule != atom_sym && rule != statement_sym && rule != all_sym) { + jl_error("jl_fl_parse: unrecognized parse options"); + } + if (offset != 0 && rule == all_sym) { + jl_error("Parse `all`: offset not supported"); } jl_ast_context_t *ctx = jl_ast_ctx_enter(); @@ -803,23 +811,19 @@ JL_DLLEXPORT jl_value_t *jl_fl_parse(const char* text, size_t text_len, value_t fl_filename = cvalue_static_cstrn(fl_ctx, filename, filename_len); value_t fl_expr; size_t pos1 = 0; - if (rule == JL_PARSE_ALL) { + if (rule == all_sym) { value_t e = fl_applyn(fl_ctx, 2, symbol_value(symbol(fl_ctx, "jl-parse-all")), fl_text, fl_filename); fl_expr = e; pos1 = e == fl_ctx->FL_EOF ? text_len : 0; } - else if (rule == JL_PARSE_STATEMENT || rule == JL_PARSE_ATOM) { - value_t greedy = rule == JL_PARSE_STATEMENT ? fl_ctx->T : fl_ctx->F; + else { + value_t greedy = rule == statement_sym ? fl_ctx->T : fl_ctx->F; value_t p = fl_applyn(fl_ctx, 4, symbol_value(symbol(fl_ctx, "jl-parse-one")), fl_text, fl_filename, fixnum(offset), greedy); fl_expr = car_(p); pos1 = tosize(fl_ctx, cdr_(p), "parse"); } - else { - jl_ast_ctx_leave(ctx); - jl_errorf("Unknown parse rule %d", (int)rule); - } // Convert to julia values jl_value_t *expr=NULL, *end_offset=NULL; diff --git a/src/frontend.c b/src/frontend.c index 6f0626529a2f4..7c24a60b42206 100644 --- a/src/frontend.c +++ b/src/frontend.c @@ -20,9 +20,9 @@ JL_DLLEXPORT void jl_set_parser(jl_parse_func_t parser) JL_DLLEXPORT jl_value_t *jl_parse(const char* text, size_t text_len, const char* filename, size_t filename_len, - size_t offset, int rule) + size_t offset, jl_value_t* options) { - return (*jl_current_parser)(text, text_len, filename, filename_len, offset, rule); + return (*jl_current_parser)(text, text_len, filename, filename_len, offset, options); } // C API @@ -31,18 +31,17 @@ JL_DLLEXPORT jl_value_t *jl_parse_all(const char *text, size_t text_len, const char *filename, size_t filename_len) { jl_value_t *p = jl_parse(text, text_len, filename, filename_len, - 0, JL_PARSE_ALL); + 0, (jl_value_t*)all_sym); return jl_svecref(p, 0); } // this is for parsing one expression out of a string, keeping track of // the current position. -// FIXME: Add filename? JL_DLLEXPORT jl_value_t *jl_parse_string(const char *text, size_t text_len, int pos0, int greedy) { return jl_parse(text, text_len, "none", 4, - pos0, greedy ? JL_PARSE_STATEMENT : JL_PARSE_ATOM); + pos0, greedy ? (jl_value_t*)statement_sym : (jl_value_t*)atom_sym); } // deprecated diff --git a/src/julia.h b/src/julia.h index 75cc0043dc21b..ea01d23bd50c7 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1600,18 +1600,7 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist); JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *depmods); JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(const char *buf, size_t sz, jl_array_t *depmods); -// front end interface -typedef enum { - JL_PARSE_ATOM = 1, - JL_PARSE_STATEMENT = 2, - JL_PARSE_ALL = 3, -} jl_parse_rule_t; - // parsing -JL_DLLEXPORT jl_value_t *jl_parse(const char* text, size_t text_len, - const char* filename, size_t filename_len, - size_t offset, int rule); -// Convenince functions JL_DLLEXPORT jl_value_t *jl_parse_all(const char *str, size_t len, const char *filename, size_t filename_len); JL_DLLEXPORT jl_value_t *jl_parse_string(const char *str, size_t len, diff --git a/src/julia_internal.h b/src/julia_internal.h index 93fc0a63d0e6b..e04d5352b7226 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -638,13 +638,16 @@ jl_tupletype_t *arg_type_tuple(jl_value_t *arg1, jl_value_t **args, size_t nargs int jl_has_meta(jl_array_t *body, jl_sym_t *sym); -// Parser replacement -typedef jl_value_t* (*jl_parse_func_t)(const char*, size_t, const char*, size_t, size_t, int); +// Experimental parser replacement API +JL_DLLEXPORT jl_value_t *jl_parse(const char* text, size_t text_len, + const char* filename, size_t filename_len, + size_t offset, jl_value_t *options); +typedef jl_value_t* (*jl_parse_func_t)(const char*, size_t, const char*, size_t, size_t, jl_value_t*); JL_DLLEXPORT void jl_set_parser(jl_parse_func_t parser); // Builtin flisp parser JL_DLLEXPORT jl_value_t *jl_fl_parse(const char* text, size_t text_len, const char* filename, size_t filename_len, - size_t offset, int rule); + size_t offset, jl_value_t *options); //-------------------------------------------------- // Backtraces @@ -1174,6 +1177,7 @@ extern jl_sym_t *gc_preserve_begin_sym; extern jl_sym_t *gc_preserve_end_sym; extern jl_sym_t *failed_sym; extern jl_sym_t *done_sym; extern jl_sym_t *runnable_sym; extern jl_sym_t *coverageeffect_sym; extern jl_sym_t *escape_sym; extern jl_sym_t *optlevel_sym; +extern jl_sym_t *atom_sym; extern jl_sym_t *statement_sym; extern jl_sym_t *all_sym; struct _jl_sysimg_fptrs_t;