blob: 78b3ddc788122a3e458104406849d383874387a1 [file] [log] [blame]
import re, os
global_keywords = {
# Values.
"null": 1,
"false": 1,
"true": 1,
# Operators.
"in": 1,
"of": 0,
"typeof": 1,
"instanceof": 1,
"void": 1,
"new": 1,
"delete": 1,
"yield": 1,
# Statements.
"var": 1,
"if": 1,
"else": 1,
"while": 1,
"do": 1,
"for": 1,
"break": 1,
"continue": 1,
"switch": 1,
"case": 1,
"default": 1,
"function": 1,
"return": 1,
"with": 1,
"try": 1,
"catch": 1,
"finally": 1,
"throw": 1,
# Module.
"import": 1,
"export": 1,
# Reserved words.
"meta": 0,
"from": 0,
"this": 1,
"arguments": 0,
"eval": 0,
"target": 0,
"await": 1,
"async": 0,
"class": 1,
"const": 1,
"debugger": 1,
"enum": 1,
"extends": 1,
"implements": 0,
"interface": 0,
"let": 0,
"package": 0,
"private": 0,
"protected": 0,
"public": 0,
"static": 0,
"super": 1
}
class Table:
def __init__(self, header):
self.buffer = []
self.header = header
def add(self, data):
self.buffer.append(data)
def create(self, newlines = False):
result = []
data = self.buffer
result.append("static const {}[{}] =\n{{\n".format(self.header,
len(data)))
last = len(data) - 1
for idx in range(len(data)):
result.append(" {},\n{}".format(data[idx],
("\n" if newlines and idx != last else "")))
result.append("};")
return result
class SHS:
def __init__(self, data):
self.data = data
def test(self, idx_from, idx_to):
stat = []
for i in range(idx_from, idx_to):
mx = 0
used = 0
result = {}
for entry in self.data:
idx = self.make_id(entry['key'], i)
if idx not in result:
used += 1
result[idx] = 0
result[idx] += 1
if result[idx] > mx:
mx = result[idx]
stat.append([mx, used, i])
stat.sort(key = lambda entr: entr[0])
best = stat[0]
print("Max deep {}; Used {} of {}".format(*best))
return best[2]
def make_id(self, key, table_size):
key = key.lower()
return (((ord(key[0]) * ord(key[-1])) + len(key)) % table_size) + 1
def make(self, best_size):
self.table_size = best_size
self.table = [[] for _ in range(self.table_size + 1)]
for e in self.data:
idx = self.make_id(e['key'], self.table_size)
self.table[idx].append(e)
self.table[idx].sort(key = lambda entr: len(entr['key']))
def build(self):
result = {}
unused = []
result[0] = [None, "NULL", self.table_size, 0, True]
for key in range(1, self.table_size + 1):
if not self.table[key]:
unused.append(key)
continue
e = self.table[key].pop(0)
result[key] = [e["key"], e["value"], len(e["key"]), 0, True]
self.idx = self.table_size
self.unused = unused
self.unused_pos = 0
for key in range(1, self.table_size + 1):
if not self.table[key]:
continue
last_entry = result[key]
for e in self.table[key]:
last_entry[3] = self.next_free_pos()
new_entry = [e["key"], e["value"], len(e["key"]), 0, False]
result[last_entry[3]] = new_entry
last_entry = new_entry
return result
def next_free_pos(self):
if len(self.unused) > self.unused_pos:
idx = self.unused[self.unused_pos]
self.unused_pos += 1
return idx
self.idx += 1
return self.idx
if __name__ == "__main__":
data_name = "njs_lexer_kws"
def enum(name):
name = re.sub(r"[^a-zA-Z0-9_]", "_", name)
return "NJS_TOKEN_" + name.upper()
def kw_create():
t = Table("njs_keyword_t " + data_name)
for k, v in sorted(global_keywords.items()):
t.add("{{\n"
" .entry = {{ njs_str(\"{}\") }},\n"
" .type = {},\n"
" .reserved = {}\n"
" }}"
.format(k, enum(k), v))
return t.create(True)
def entries_create():
shs = SHS([{ "key": kw,
"value": "&{}[{}]".format(data_name, i) }
for i, kw in enumerate(sorted(global_keywords.keys()))])
best_size = shs.test(5, 128)
shs.make(best_size)
lst = shs.build()
t = Table("njs_lexer_keyword_entry_t njs_lexer_keyword_entries")
for kw in range(shs.idx + 1):
if kw not in lst:
t.add("{ NULL, NULL, 0, 0 }")
continue
key_val = "\"{}\"".format(lst[kw][0]) if lst[kw][0] else "NULL"
t.add("{{ {}, {}, {}, {} }}".format(key_val, *lst[kw][1:]))
return t.create()
content = ["""
/*
* Copyright (C) Nginx, Inc.
*
* Do not edit, generated by: utils/lexer_keyword.py.
*/
#ifndef _NJS_LEXER_TABLES_H_INCLUDED_
#define _NJS_LEXER_TABLES_H_INCLUDED_
""",
"".join(kw_create()),
"\n\n\n",
"".join(entries_create()),
"\n\n\n#endif /* _NJS_LEXER_TABLES_H_INCLUDED_ */\n"]
out = "".join(content)
print(out)
fn = os.path.join(os.path.dirname(__file__), "../src/njs_lexer_tables.h")
with open(fn, 'w') as fh:
fh.write(out)