250 lines
7.9 KiB
Python
250 lines
7.9 KiB
Python
|
"""JsLex: a lexer for JavaScript"""
|
||
|
|
# Originally from https://bitbucket.org/ned/jslex
|
||
|
|
import re
|
||
|
|
|
||
|
|
|
||
|
|
class Tok:
|
||
|
|
"""
|
||
|
|
A specification for a token class.
|
||
|
|
"""
|
||
|
|
|
||
|
|
num = 0
|
||
|
|
|
||
|
|
def __init__(self, name, regex, next=None):
|
||
|
|
self.id = Tok.num
|
||
|
|
Tok.num += 1
|
||
|
|
self.name = name
|
||
|
|
self.regex = regex
|
||
|
|
self.next = next
|
||
|
|
|
||
|
|
|
||
|
|
def literals(choices, prefix="", suffix=""):
|
||
|
|
"""
|
||
|
|
Create a regex from a space-separated list of literal `choices`.
|
||
|
|
|
||
|
|
If provided, `prefix` and `suffix` will be attached to each choice
|
||
|
|
individually.
|
||
|
|
"""
|
||
|
|
return "|".join(prefix + re.escape(c) + suffix for c in choices.split())
|
||
|
|
|
||
|
|
|
||
|
|
class Lexer:
|
||
|
|
"""
|
||
|
|
A generic multi-state regex-based lexer.
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self, states, first):
|
||
|
|
self.regexes = {}
|
||
|
|
self.toks = {}
|
||
|
|
|
||
|
|
for state, rules in states.items():
|
||
|
|
parts = []
|
||
|
|
for tok in rules:
|
||
|
|
groupid = "t%d" % tok.id
|
||
|
|
self.toks[groupid] = tok
|
||
|
|
parts.append("(?P<%s>%s)" % (groupid, tok.regex))
|
||
|
|
self.regexes[state] = re.compile("|".join(parts), re.MULTILINE | re.VERBOSE)
|
||
|
|
|
||
|
|
self.state = first
|
||
|
|
|
||
|
|
def lex(self, text):
|
||
|
|
"""
|
||
|
|
Lexically analyze `text`.
|
||
|
|
|
||
|
|
Yield pairs (`name`, `tokentext`).
|
||
|
|
"""
|
||
|
|
end = len(text)
|
||
|
|
state = self.state
|
||
|
|
regexes = self.regexes
|
||
|
|
toks = self.toks
|
||
|
|
start = 0
|
||
|
|
|
||
|
|
while start < end:
|
||
|
|
for match in regexes[state].finditer(text, start):
|
||
|
|
name = match.lastgroup
|
||
|
|
tok = toks[name]
|
||
|
|
toktext = match[name]
|
||
|
|
start += len(toktext)
|
||
|
|
yield (tok.name, toktext)
|
||
|
|
|
||
|
|
if tok.next:
|
||
|
|
state = tok.next
|
||
|
|
break
|
||
|
|
|
||
|
|
self.state = state
|
||
|
|
|
||
|
|
|
||
|
|
class JsLexer(Lexer):
|
||
|
|
"""
|
||
|
|
A JavaScript lexer
|
||
|
|
|
||
|
|
>>> lexer = JsLexer()
|
||
|
|
>>> list(lexer.lex("a = 1"))
|
||
|
|
[('id', 'a'), ('ws', ' '), ('punct', '='), ('ws', ' '), ('dnum', '1')]
|
||
|
|
|
||
|
|
This doesn't properly handle non-ASCII characters in the JavaScript source.
|
||
|
|
"""
|
||
|
|
|
||
|
|
# Because these tokens are matched as alternatives in a regex, longer
|
||
|
|
# possibilities must appear in the list before shorter ones, for example,
|
||
|
|
# '>>' before '>'.
|
||
|
|
#
|
||
|
|
# Note that we don't have to detect malformed JavaScript, only properly
|
||
|
|
# lex correct JavaScript, so much of this is simplified.
|
||
|
|
|
||
|
|
# Details of JavaScript lexical structure are taken from
|
||
|
|
# https://www.ecma-international.org/publications-and-standards/standards/ecma-262/
|
||
|
|
|
||
|
|
# A useful explanation of automatic semicolon insertion is at
|
||
|
|
# http://inimino.org/~inimino/blog/javascript_semicolons
|
||
|
|
|
||
|
|
both_before = [
|
||
|
|
Tok("comment", r"/\*(.|\n)*?\*/"),
|
||
|
|
Tok("linecomment", r"//.*?$"),
|
||
|
|
Tok("ws", r"\s+"),
|
||
|
|
Tok(
|
||
|
|
"keyword",
|
||
|
|
literals(
|
||
|
|
"""
|
||
|
|
break case catch class const continue debugger
|
||
|
|
default delete do else enum export extends
|
||
|
|
finally for function if import in instanceof
|
||
|
|
new return super switch this throw try typeof
|
||
|
|
var void while with
|
||
|
|
""",
|
||
|
|
suffix=r"\b",
|
||
|
|
),
|
||
|
|
next="reg",
|
||
|
|
),
|
||
|
|
Tok("reserved", literals("null true false", suffix=r"\b"), next="div"),
|
||
|
|
Tok(
|
||
|
|
"id",
|
||
|
|
r"""
|
||
|
|
([a-zA-Z_$ ]|\\u[0-9a-fA-Z]{4}) # first char
|
||
|
|
([a-zA-Z_$0-9]|\\u[0-9a-fA-F]{4})* # rest chars
|
||
|
|
""",
|
||
|
|
next="div",
|
||
|
|
),
|
||
|
|
Tok("hnum", r"0[xX][0-9a-fA-F]+", next="div"),
|
||
|
|
Tok("onum", r"0[0-7]+"),
|
||
|
|
Tok(
|
||
|
|
"dnum",
|
||
|
|
r"""
|
||
|
|
( (0|[1-9][0-9]*) # DecimalIntegerLiteral
|
||
|
|
\. # dot
|
||
|
|
[0-9]* # DecimalDigits-opt
|
||
|
|
([eE][-+]?[0-9]+)? # ExponentPart-opt
|
||
|
|
|
|
||
|
|
\. # dot
|
||
|
|
[0-9]+ # DecimalDigits
|
||
|
|
([eE][-+]?[0-9]+)? # ExponentPart-opt
|
||
|
|
|
|
||
|
|
(0|[1-9][0-9]*) # DecimalIntegerLiteral
|
||
|
|
([eE][-+]?[0-9]+)? # ExponentPart-opt
|
||
|
|
)
|
||
|
|
""",
|
||
|
|
next="div",
|
||
|
|
),
|
||
|
|
Tok(
|
||
|
|
"punct",
|
||
|
|
literals(
|
||
|
|
"""
|
||
|
|
>>>= === !== >>> <<= >>= <= >= == != << >> &&
|
||
|
|
|| += -= *= %= &= |= ^=
|
||
|
|
"""
|
||
|
|
),
|
||
|
|
next="reg",
|
||
|
|
),
|
||
|
|
Tok("punct", literals("++ -- ) ]"), next="div"),
|
||
|
|
Tok("punct", literals("{ } ( [ . ; , < > + - * % & | ^ ! ~ ? : ="), next="reg"),
|
||
|
|
Tok("string", r'"([^"\\]|(\\(.|\n)))*?"', next="div"),
|
||
|
|
Tok("string", r"'([^'\\]|(\\(.|\n)))*?'", next="div"),
|
||
|
|
]
|
||
|
|
|
||
|
|
both_after = [
|
||
|
|
Tok("other", r"."),
|
||
|
|
]
|
||
|
|
|
||
|
|
states = {
|
||
|
|
# slash will mean division
|
||
|
|
"div": both_before
|
||
|
|
+ [
|
||
|
|
Tok("punct", literals("/= /"), next="reg"),
|
||
|
|
]
|
||
|
|
+ both_after,
|
||
|
|
# slash will mean regex
|
||
|
|
"reg": both_before
|
||
|
|
+ [
|
||
|
|
Tok(
|
||
|
|
"regex",
|
||
|
|
r"""
|
||
|
|
/ # opening slash
|
||
|
|
# First character is..
|
||
|
|
( [^*\\/[] # anything but * \ / or [
|
||
|
|
| \\. # or an escape sequence
|
||
|
|
| \[ # or a class, which has
|
||
|
|
( [^\]\\] # anything but \ or ]
|
||
|
|
| \\. # or an escape sequence
|
||
|
|
)* # many times
|
||
|
|
\]
|
||
|
|
)
|
||
|
|
# Following characters are same, except for excluding a star
|
||
|
|
( [^\\/[] # anything but \ / or [
|
||
|
|
| \\. # or an escape sequence
|
||
|
|
| \[ # or a class, which has
|
||
|
|
( [^\]\\] # anything but \ or ]
|
||
|
|
| \\. # or an escape sequence
|
||
|
|
)* # many times
|
||
|
|
\]
|
||
|
|
)* # many times
|
||
|
|
/ # closing slash
|
||
|
|
[a-zA-Z0-9]* # trailing flags
|
||
|
|
""",
|
||
|
|
next="div",
|
||
|
|
),
|
||
|
|
]
|
||
|
|
+ both_after,
|
||
|
|
}
|
||
|
|
|
||
|
|
def __init__(self):
|
||
|
|
super().__init__(self.states, "reg")
|
||
|
|
|
||
|
|
|
||
|
|
def prepare_js_for_gettext(js):
|
||
|
|
"""
|
||
|
|
Convert the JavaScript source `js` into something resembling C for
|
||
|
|
xgettext.
|
||
|
|
|
||
|
|
What actually happens is that all the regex literals are replaced with
|
||
|
|
"REGEX".
|
||
|
|
"""
|
||
|
|
|
||
|
|
def escape_quotes(m):
|
||
|
|
"""Used in a regex to properly escape double quotes."""
|
||
|
|
s = m[0]
|
||
|
|
if s == '"':
|
||
|
|
return r"\""
|
||
|
|
else:
|
||
|
|
return s
|
||
|
|
|
||
|
|
lexer = JsLexer()
|
||
|
|
c = []
|
||
|
|
for name, tok in lexer.lex(js):
|
||
|
|
if name == "regex":
|
||
|
|
# C doesn't grok regexes, and they aren't needed for gettext,
|
||
|
|
# so just output a string instead.
|
||
|
|
tok = '"REGEX"'
|
||
|
|
elif name == "string":
|
||
|
|
# C doesn't have single-quoted strings, so make all strings
|
||
|
|
# double-quoted.
|
||
|
|
if tok.startswith("'"):
|
||
|
|
guts = re.sub(r"\\.|.", escape_quotes, tok[1:-1])
|
||
|
|
tok = '"' + guts + '"'
|
||
|
|
elif name == "id":
|
||
|
|
# C can't deal with Unicode escapes in identifiers. We don't
|
||
|
|
# need them for gettext anyway, so replace them with something
|
||
|
|
# innocuous
|
||
|
|
tok = tok.replace("\\", "U")
|
||
|
|
c.append(tok)
|
||
|
|
return "".join(c)
|