introduce intermediate attribute KeywordAttr and add token information

with it.

Test plan: make test

h_program-lang/ast_generic.ml

@@ -544,6 +544,13 @@ and type_ =
 (*****************************************************************************)
 (* a.k.a decorators, annotations *)
 and attribute = 
+  | KeywordAttr of keyword_attribute wrap
+  (* for general @annotations *)
+  | NamedAttr of ident * any list
+
+  | OtherAttribute of other_attribute_operator * any list
+
+  and keyword_attribute =
   | Static | Volatile | Extern
   (* for class fields *)
   | Public | Private | Protected
@@ -560,10 +567,6 @@ and attribute =
   | Getter | Setter
   (* for parameters *)
   | Variadic
-  (* for general @annotations *)
-  | NamedAttr of ident * any list
-
-  | OtherAttribute of other_attribute_operator * any list
 
   and other_attribute_operator = 
     (* Java *)
@@ -891,6 +894,8 @@ let basic_field id typeopt =
   let entity = basic_entity id [] in
   FieldVar (entity, { vinit = None; vtype = typeopt})
 
+let attr kwd tok =
+  KeywordAttr (kwd, tok)
 
 let expr_to_arg e = 
   Arg e

lang_GENERIC/parsing/map_ast.ml

@@ -288,29 +288,9 @@ and map_other_type_operator =
   | OT_Shape -> OT_Shape
   | OT_Variadic -> OT_Variadic
 
-and map_attribute =
-  function
-    | Recursive -> Recursive
-    | MutuallyRecursive -> MutuallyRecursive
-  | Static -> Static
-  | Volatile -> Volatile
-  | Extern -> Extern
-  | Public -> Public
-  | Private -> Private
-  | Protected -> Protected
-  | Abstract -> Abstract
-  | Final -> Final
-  | Var -> Var
-  | Let -> Let
-  | Const -> Const
-  | Mutable -> Mutable
-  | Generator -> Generator
-  | Async -> Async
-  | Ctor -> Ctor
-  | Dtor -> Dtor
-  | Getter -> Getter
-  | Setter -> Setter
-  | Variadic -> Variadic
+and map_attribute = function
+  | KeywordAttr v1 -> let v1 = map_wrap map_keyword_attribute v1 in 
+      KeywordAttr v1
   | NamedAttr ((v1, v2)) ->
       let v1 = map_ident v1
       and v2 = map_of_list map_any v2
@@ -320,6 +300,7 @@ and map_attribute =
       and v2 = map_of_list map_any v2
       in OtherAttribute ((v1, v2))
 
+and map_keyword_attribute x = x
 and map_other_attribute_operator x  = x
 
 and map_stmt x =

lang_GENERIC/parsing/meta_ast.ml

@@ -352,7 +352,7 @@ and vof_other_type_operator =
   | OT_EnumName -> Ocaml.VSum (("OT_EnumName", []))
   | OT_Shape -> Ocaml.VSum (("OT_Shape", []))
   | OT_Variadic -> Ocaml.VSum (("OT_Variadic", []))
-and vof_attribute =
+and vof_keyword_attribute =
   function
   | Static -> Ocaml.VSum (("Static", []))
   | Volatile -> Ocaml.VSum (("Volatile", []))
@@ -375,6 +375,10 @@ and vof_attribute =
   | Getter -> Ocaml.VSum (("Getter", []))
   | Setter -> Ocaml.VSum (("Setter", []))
   | Variadic -> Ocaml.VSum (("Variadic", []))
+
+and vof_attribute = function
+  | KeywordAttr x -> let v1 = vof_wrap vof_keyword_attribute x in
+    Ocaml.VSum (("KeywordAttr", [v1]))
   | NamedAttr ((v1, v2)) ->
       let v1 = vof_ident v1
       and v2 = Ocaml.vof_list vof_any v2

lang_GENERIC/parsing/visitor_ast.ml

@@ -271,32 +271,14 @@ and v_type_parameter_constraint =
 and v_attribute x =
   let k x = 
   match x with
-  | Recursive | MutuallyRecursive -> ()
-  | Static -> ()
-  | Volatile -> ()
-  | Extern -> ()
-  | Public -> ()
-  | Private -> ()
-  | Protected -> ()
-  | Abstract -> ()
-  | Final -> ()
-  | Var -> ()
-  | Let -> ()
-  | Const -> ()
-  | Mutable -> ()
-  | Generator -> ()
-  | Async -> ()
-  | Ctor -> ()
-  | Dtor -> ()
-  | Getter -> ()
-  | Setter -> ()
-  | Variadic -> ()
+  | KeywordAttr v1 -> let v1 = v_wrap v_keyword_attribute v1 in ()
   | NamedAttr ((v1, v2)) -> let v1 = v_ident v1 and v2 = v_list v_any v2 in ()
   | OtherAttribute ((v1, v2)) ->
       let v1 = v_other_attribute_operator v1 and v2 = v_list v_any v2 in ()
   in
   vin.kattr (k, all_functions) x
 
+and v_keyword_attribute _ = ()
 and v_other_attribute_operator _ = ()
 
 and v_stmts xs =

lang_c/analyze/c_to_generic.ml

@@ -40,7 +40,7 @@ let either f g x =
 
 let string = id
 
-let fake_info () = Parse_info.fake_info "FAKE"
+let fake s = Parse_info.fake_info s
 
 let opt_to_ident opt =
   match opt with
@@ -187,7 +187,7 @@ and expr =
   | Sequence ((v1, v2)) -> let v1 = expr v1 and v2 = expr v2 in
       G.Seq [v1;v2]
   | SizeOf v1 -> let v1 = either expr type_ v1 in
-      G.Call (G.IdSpecial (G.Sizeof, fake_info()), 
+      G.Call (G.IdSpecial (G.Sizeof, fake "sizeof"), 
        (match v1 with
        | Left e -> [G.Arg e]
        | Right t -> [G.ArgType t]
@@ -215,7 +215,7 @@ and expr =
           v1
       in G.Record v1
   | GccConstructor ((v1, v2)) -> let v1 = type_ v1 and v2 = expr v2 in
-      G.Call (G.IdSpecial (G.New, fake_info ()), 
+      G.Call (G.IdSpecial (G.New, fake "new"), 
         (G.ArgType v1)::([v2] |> List.map G.expr_to_arg))
 
 and argument v = 
@@ -285,8 +285,8 @@ and
 and initialiser v = expr v
 
 and storage = function 
-  | Extern -> [G.Extern] 
-  | Static -> [G.Static]
+  | Extern -> [G.attr G.Extern (fake "extern")] 
+  | Static -> [G.attr G.Static (fake "static")]
   | DefaultStorage -> []
 
 let func_def {
@@ -298,7 +298,7 @@ let func_def {
   let v1 = name f_name in
   let (ret, params) = function_type f_type in
   let v3 = list stmt f_body in 
-  let v4 = if f_static then [G.Static] else [] in
+  let v4 = if f_static then [G.attr G.Static (fake "static")] else [] in
   let entity = G.basic_entity v1 v4 in
   entity, G.FuncDef { G.
     fparams = params |> List.map (fun x -> G.ParamClassic x);

lang_go/analyze/go_to_generic.ml

@@ -44,8 +44,8 @@ let name_of_qualified_ident = function
   | Left id -> id, G.empty_name_info
   | Right (xs, id) -> id, { G.name_qualifier = Some xs; name_typeargs = None }
 
-let fake_info () = Parse_info.fake_info "FAKE"
-let fake_name s = (s, fake_info ()), G.empty_name_info
+let fake s = Parse_info.fake_info s
+let fake_name s = (s, fake s), G.empty_name_info
 
 let ii_of_any = Lib_parsing_go.ii_of_any
 
@@ -133,13 +133,13 @@ let rec type_ =
   | TStruct v1 -> let v1 = list struct_field v1 in 
       (* could also use StructName *)
       let s = gensym () in
-      let ent = G.basic_entity (s, fake_info ()) [] in
+      let ent = G.basic_entity (s, fake s) [] in
       let def = G.TypeDef { G.tbody = G.AndType v1 } in
       Common.push (ent, def) anon_types;
       G.TyApply (fake_name s, [])
   | TInterface v1 -> let v1 = list interface_field v1 in 
       let s = gensym () in
-      let ent = G.basic_entity (s, fake_info ()) [] in
+      let ent = G.basic_entity (s, fake s) [] in
       let def = G.ClassDef { G.ckind = G.Interface; 
           cextends = []; cimplements = []; 
           cbody = v1; } in
@@ -162,7 +162,7 @@ and parameter { pname = pname; ptype = ptype; pdots = pdots } =
   let arg3 = option tok pdots in 
   { G.pname = arg1; ptype = Some arg2; 
     pdefault = None; 
-    pattrs = (match arg3 with None -> [] | Some _ -> [G.Variadic]);
+    pattrs = (match arg3 with None -> [] | Some tok -> [G.attr G.Variadic tok]);
     pinfo = G.empty_id_info ()
     }
 
@@ -230,7 +230,7 @@ and expr =
       G.Call (G.IdSpecial (G.ArithOp v2, tok), [v1;v3] |> List.map G.expr_to_arg)
   | CompositeLit ((v1, v2)) ->
       let v1 = type_ v1 and v2 = list init v2 in
-      G.Call (G.IdSpecial (G.New, fake_info ()), 
+      G.Call (G.IdSpecial (G.New, fake "new"), 
         (G.ArgType v1)::(v2 |> List.map G.expr_to_arg))
   | Slice ((v1, v2)) ->
       let e = expr v1 in
@@ -241,7 +241,7 @@ and expr =
       in 
       G.SliceAccess (e, v1, v2, v3)
   | TypeAssert ((v1, v2)) -> let v1 = expr v1 and v2 = type_ v2 in
-      G.Call (G.IdSpecial (G.Instanceof, fake_info()),
+      G.Call (G.IdSpecial (G.Instanceof, fake "instanceof"),
         [G.Arg v1; G.ArgType v2])
   | EllipsisTODO v1 -> let v1 = tok v1 in 
       G.Ellipsis v1
@@ -404,7 +404,7 @@ and stmt =
       in 
       (match opt with
       | None -> 
-         let pattern = G.PatUnderscore (fake_info ()) in
+         let pattern = G.PatUnderscore (fake "_") in
          G.For (G.ForEach (pattern, v3), v4)
       | Some (xs, _tokEqOrColonEqTODO) -> 
           let pattern = G.PatTuple (xs |> List.map G.expr_to_pattern) in
@@ -474,14 +474,14 @@ and decl =
       and v2 = option type_ v2
       and v3 = option constant_expr v3
       in 
-      let ent = G.basic_entity v1 [G.Const] in
+      let ent = G.basic_entity v1 [G.attr G.Const (fake "const")] in
       G.DefStmt (ent, G.VarDef { G.vinit = v3; vtype = v2 })
   | DVar ((v1, v2, v3)) ->
       let v1 = ident v1
       and v2 = option type_ v2
       and v3 = option expr v3
       in
-      let ent = G.basic_entity v1 [G.Var] in
+      let ent = G.basic_entity v1 [G.attr G.Var (fake "var")] in
       G.DefStmt (ent, G.VarDef { G.vinit = v3; vtype = v2 })
   | DTypeAlias ((v1, v2, v3)) ->
       let v1 = ident v1 and _v2 = tok v2 and v3 = type_ v3 in 

lang_java/analyze/java_to_generic.ml

@@ -109,25 +109,25 @@ let type_parameter =
       let v1 = ident v1 and v2 = list ref_type v2 in
       v1, (v2 |> List.map (fun t -> G.Extends t))
 
-let rec modifier =
-  function
-  | Variadic -> G.Variadic
-  | Public -> G.Public
-  | Protected -> G.Protected
-  | Private -> G.Private
-  | Abstract -> G.Abstract
-  | Static -> G.Static
-  | Final -> G.Final
+let rec modifier (x, tok) =
+  match x with
+  | Variadic -> G.attr G.Variadic tok
+  | Public -> G.attr G.Public tok
+  | Protected -> G.attr G.Protected tok
+  | Private -> G.attr G.Private tok
+  | Abstract -> G.attr G.Abstract tok
+  | Static -> G.attr G.Static tok
+  | Final -> G.attr G.Final tok
   | StrictFP -> G.OtherAttribute (G.OA_StrictFP, [])
   | Transient -> G.OtherAttribute (G.OA_Transient, [])
-  | Volatile -> G.Volatile
+  | Volatile -> G.attr G.Volatile tok
   | Synchronized -> G.OtherAttribute (G.OA_Synchronized, [])
   | Native -> G.OtherAttribute (G.OA_Native, [])
   | Annotation _v1 -> 
       (* let _v1TODO = annotation v1 in *)
       G.OtherAttribute (G.OA_AnnotJavaOther, [])
 
-and modifiers v = list (wrap modifier) v |> List.map fst
+and modifiers v = list modifier v
 
 (*  
 and annotation (v1, v2) =

lang_js/analyze/ast_js.ml

@@ -235,21 +235,21 @@ and stmt =
 
 and var = { 
   v_name: name;
-  v_kind: var_kind;
+  v_kind: var_kind wrap;
   v_init: expr;
   v_resolved: resolved_name ref;
 }
   and var_kind = Var | Let | Const
 
 and fun_ = {
-  f_props: fun_prop list;
+  f_props: fun_prop wrap list;
   f_params: parameter list;
   f_body: stmt;
 }
   and parameter = {
     p_name: name;
     p_default: expr option;
-    p_dots: bool;
+    p_dots: tok option;
   }
   (* less: could transpile *)
   and fun_prop = 
@@ -267,7 +267,7 @@ and class_ = {
 
   and property = 
     (* expr is a Fun for methods *)
-    | Field of property_name * property_prop list * expr
+    | Field of property_name * property_prop wrap list * expr
     (* less: can unsugar? *)
     | FieldSpread of expr