Tokens.md

Tokens

The table below defines the tokens of the Serena Shell Language. These tokens are generated by the lexer from the source text of a script and are then handed off to the parser for further processing.

NL
    : \x0a | \x0d \0xa
    ;

ELSE:       'else';
FALSE:      'false';
IF:         'if';
INTERNAL:   'internal';
LET:        'let';
PUBLIC:     'public';
TRUE:       'true';
VAR:        'var';
WHILE:      'while';

AMPERSAND:          '&';
ASSIGNMENT:         '=';
ASTERISK:           '*';
BANG:               '!';
BAR:                '|';
CLOSE_BRACE:        '}';
CLOSE_PARA:         ')';
CONJUNCTION:        '&&';
DISJUNCTION:        '||';
EQEQ:               '==';
GREATER:            '>';
GREQ:               '>=';
LEEQ:               '<=';
LESS:               '<';
MINUS:              '-';
NOEQ:               '!=';
OPEN_BRACE:         '{';
OPEN_PARA:          '(';
PERCENT:            '%';
PLUS:               '+';
SEMICOLON:          ';';
SLASH:              '/';

DOUBLE_BACKTICK(default, dbt_mode): '``';
DOUBLE_QUOTE(default, dq_mode):     '"';

STRING_SEGMENT(dq_mode)
    : [^\$"]
    ;

STRING_SEGMENT(dbt_mode)
    : [^\$``]
    ;

ESCAPE_SEQUENCE(dq_mode, dbt_mode)
    : '\' [abefvrn$"'\]
    ;

ESCAPED_EXPRESSION(dq_mode, dbt_mode)
    : '\('
    ;

SINGLE_QUOTED_STRING
    : '''[^']'''
    ;

SINGLE_BACKTICK_STRING
    : '`'[^`]'`'
    ;

INTEGER
    : ('0'[box])? [0-9]+
    ;

VAR_NAME(default, dq_mode, dbt_mode)
    : '$' (('_' | [a-z] | [A-Z] | [0-9])* ':')? ('_' | [a-z] | [A-Z] | [0-9])+
    ;

IDENTIFIER
    : ([^\0x20\0x09\0x0b\0x0c|&+-*/%\;$"`'(){}<>=!] | ('\'?))+
    ;