-
Notifications
You must be signed in to change notification settings - Fork 47
/
Copy pathToken.hs
182 lines (168 loc) · 13 KB
/
Token.hs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
{-# LANGUAGE CPP, DeriveDataTypeable #-}
-----------------------------------------------------------------------------
-- |
-- Module : Language.Python.Common.Token
-- Copyright : (c) 2009 Bernie Pope
-- License : BSD-style
-- Maintainer : bjpop@csse.unimelb.edu.au
-- Stability : experimental
-- Portability : ghc
--
-- Lexical tokens for the Python lexer. Contains the superset of tokens from
-- version 2 and version 3 of Python (they are mostly the same).
-----------------------------------------------------------------------------
module Language.JavaScript.Parser.Token
(
-- * The tokens
Token (..)
, CommentAnnotation (..)
-- * String conversion
, debugTokenString
-- * Classification
-- TokenClass (..),
) where
import Data.Data
import Language.JavaScript.Parser.SrcLocation
data CommentAnnotation
= CommentA TokenPosn String
| WhiteSpace TokenPosn String
| NoComment
deriving (Eq, Show, Typeable, Data, Read)
-- | Lexical tokens.
-- Each may be annotated with any comment occurring between the prior token and this one
data Token
-- Comment
= CommentToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } -- ^ Single line comment.
| WsToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } -- ^ White space, for preservation.
-- Identifiers
| IdentifierToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } -- ^ Identifier.
-- Javascript Literals
| DecimalToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
-- ^ Literal: Decimal
| HexIntegerToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
-- ^ Literal: Hexadecimal Integer
| OctalToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
-- ^ Literal: Octal Integer
| StringToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
-- ^ Literal: string, delimited by either single or double quotes
| RegExToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
-- ^ Literal: Regular Expression
-- Keywords
| AsyncToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| AwaitToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| BreakToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| CaseToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| CatchToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| ClassToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| ConstToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| LetToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| ContinueToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| DebuggerToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| DefaultToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| DeleteToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| DoToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| ElseToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| EnumToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| ExtendsToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| FalseToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| FinallyToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| ForToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| FunctionToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| FromToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| IfToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| InToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| InstanceofToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| NewToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| NullToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| OfToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| ReturnToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| StaticToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| SuperToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| SwitchToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| ThisToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| ThrowToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| TrueToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| TryToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| TypeofToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| VarToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| VoidToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| WhileToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| YieldToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| ImportToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| WithToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| ExportToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
-- Future reserved words
| FutureToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
-- Needed, not sure what they are though.
| GetToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| SetToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
-- MetaProperty
| TargetToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| MetaToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
-- Delimiters
-- Operators
| AutoSemiToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| SemiColonToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| CommaToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| HookToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| ColonToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| OrToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| AndToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| BitwiseOrToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| BitwiseXorToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| BitwiseAndToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| StrictEqToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| EqToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| TimesAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| DivideAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| ModAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| PlusAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| MinusAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| LshAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| RshAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| UrshAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| AndAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| XorAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| OrAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| SimpleAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| StrictNeToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| NeToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| LshToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| LeToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| LtToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| UrshToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| RshToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| GeToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| GtToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| IncrementToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| DecrementToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| PlusToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| MinusToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| MulToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| DivToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| ModToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| NotToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| BitwiseNotToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| ArrowToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| SpreadToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| DotToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| LeftBracketToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| RightBracketToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| LeftCurlyToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| RightCurlyToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| LeftParenToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| RightParenToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
| CondcommentEndToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] }
-- Template literal lexical components
| NoSubstitutionTemplateToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| TemplateHeadToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| TemplateMiddleToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| TemplateTailToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
-- Special cases
| AsToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| TailToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } -- ^ Stuff between last JS and EOF
| EOFToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } -- ^ End of file
deriving (Eq, Show, Typeable)
-- | Produce a string from a token containing detailed information. Mainly intended for debugging.
debugTokenString :: Token -> String
debugTokenString = takeWhile (/= ' ') . show