Skip to content

Commit acf9184

Browse files
gvanrossumgvanrossum
authoredDec 3, 2022
GH-98831: Support cache effects in super- and macro instructions (#99601)
·
v3.14.0a6v3.12.0a3
1 parent 0547a98 commit acf9184

File tree

4 files changed

+482
-272
lines changed

4 files changed

+482
-272
lines changed
 

‎Python/generated_cases.c.h

+2-2
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Tools/cases_generator/generate_cases.py

+372-222
Large diffs are not rendered by default.

‎Tools/cases_generator/lexer.py

+6-1
Original file line numberDiff line numberDiff line change
@@ -240,7 +240,12 @@ def to_text(tkns: list[Token], dedent: int = 0) -> str:
240240
res.append('\n')
241241
col = 1+dedent
242242
res.append(' '*(c-col))
243-
res.append(tkn.text)
243+
text = tkn.text
244+
if dedent != 0 and tkn.kind == 'COMMENT' and '\n' in text:
245+
if dedent < 0:
246+
text = text.replace('\n', '\n' + ' '*-dedent)
247+
# TODO: dedent > 0
248+
res.append(text)
244249
line, col = tkn.end
245250
return ''.join(res)
246251

‎Tools/cases_generator/parser.py

+102-47
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,12 @@
99

1010
P = TypeVar("P", bound="Parser")
1111
N = TypeVar("N", bound="Node")
12-
def contextual(func: Callable[[P], N|None]) -> Callable[[P], N|None]:
12+
13+
14+
def contextual(func: Callable[[P], N | None]) -> Callable[[P], N | None]:
1315
# Decorator to wrap grammar methods.
1416
# Resets position if `func` returns None.
15-
def contextual_wrapper(self: P) -> N|None:
17+
def contextual_wrapper(self: P) -> N | None:
1618
begin = self.getpos()
1719
res = func(self)
1820
if res is None:
@@ -21,6 +23,7 @@ def contextual_wrapper(self: P) -> N|None:
2123
end = self.getpos()
2224
res.context = Context(begin, end, self)
2325
return res
26+
2427
return contextual_wrapper
2528

2629

@@ -35,7 +38,7 @@ def __repr__(self):
3538

3639
@dataclass
3740
class Node:
38-
context: Context|None = field(init=False, default=None)
41+
context: Context | None = field(init=False, default=None)
3942

4043
@property
4144
def text(self) -> str:
@@ -68,8 +71,14 @@ class CacheEffect(Node):
6871
size: int
6972

7073

74+
@dataclass
75+
class OpName(Node):
76+
name: str
77+
78+
7179
InputEffect = StackEffect | CacheEffect
7280
OutputEffect = StackEffect
81+
UOp = OpName | CacheEffect
7382

7483

7584
@dataclass
@@ -82,32 +91,23 @@ class InstHeader(Node):
8291

8392
@dataclass
8493
class InstDef(Node):
85-
# TODO: Merge InstHeader and InstDef
86-
header: InstHeader
94+
kind: Literal["inst", "op"]
95+
name: str
96+
inputs: list[InputEffect]
97+
outputs: list[OutputEffect]
8798
block: Block
8899

89-
@property
90-
def kind(self) -> str:
91-
return self.header.kind
92-
93-
@property
94-
def name(self) -> str:
95-
return self.header.name
96100

97-
@property
98-
def inputs(self) -> list[InputEffect]:
99-
return self.header.inputs
100-
101-
@property
102-
def outputs(self) -> list[OutputEffect]:
103-
return self.header.outputs
101+
@dataclass
102+
class Super(Node):
103+
name: str
104+
ops: list[OpName]
104105

105106

106107
@dataclass
107-
class Super(Node):
108-
kind: Literal["macro", "super"]
108+
class Macro(Node):
109109
name: str
110-
ops: list[str]
110+
uops: list[UOp]
111111

112112

113113
@dataclass
@@ -118,12 +118,22 @@ class Family(Node):
118118

119119

120120
class Parser(PLexer):
121+
@contextual
122+
def definition(self) -> InstDef | Super | Macro | Family | None:
123+
if inst := self.inst_def():
124+
return inst
125+
if super := self.super_def():
126+
return super
127+
if macro := self.macro_def():
128+
return macro
129+
if family := self.family_def():
130+
return family
121131

122132
@contextual
123133
def inst_def(self) -> InstDef | None:
124-
if header := self.inst_header():
134+
if hdr := self.inst_header():
125135
if block := self.block():
126-
return InstDef(header, block)
136+
return InstDef(hdr.kind, hdr.name, hdr.inputs, hdr.outputs, block)
127137
raise self.make_syntax_error("Expected block")
128138
return None
129139

@@ -132,17 +142,14 @@ def inst_header(self) -> InstHeader | None:
132142
# inst(NAME)
133143
# | inst(NAME, (inputs -- outputs))
134144
# | op(NAME, (inputs -- outputs))
135-
# TODO: Error out when there is something unexpected.
136145
# TODO: Make INST a keyword in the lexer.
137146
if (tkn := self.expect(lx.IDENTIFIER)) and (kind := tkn.text) in ("inst", "op"):
138-
if (self.expect(lx.LPAREN)
139-
and (tkn := self.expect(lx.IDENTIFIER))):
147+
if self.expect(lx.LPAREN) and (tkn := self.expect(lx.IDENTIFIER)):
140148
name = tkn.text
141149
if self.expect(lx.COMMA):
142150
inp, outp = self.stack_effect()
143151
if self.expect(lx.RPAREN):
144-
if ((tkn := self.peek())
145-
and tkn.kind == lx.LBRACE):
152+
if (tkn := self.peek()) and tkn.kind == lx.LBRACE:
146153
return InstHeader(kind, name, inp, outp)
147154
elif self.expect(lx.RPAREN) and kind == "inst":
148155
# No legacy stack effect if kind is "op".
@@ -176,18 +183,20 @@ def inputs(self) -> list[InputEffect] | None:
176183
def input(self) -> InputEffect | None:
177184
# IDENTIFIER '/' INTEGER (CacheEffect)
178185
# IDENTIFIER (StackEffect)
179-
if (tkn := self.expect(lx.IDENTIFIER)):
186+
if tkn := self.expect(lx.IDENTIFIER):
180187
if self.expect(lx.DIVIDE):
181188
if num := self.expect(lx.NUMBER):
182189
try:
183190
size = int(num.text)
184191
except ValueError:
185192
raise self.make_syntax_error(
186-
f"Expected integer, got {num.text!r}")
193+
f"Expected integer, got {num.text!r}"
194+
)
187195
else:
188196
return CacheEffect(tkn.text, size)
189197
raise self.make_syntax_error("Expected integer")
190198
else:
199+
# TODO: Arrays, conditions
191200
return StackEffect(tkn.text)
192201

193202
def outputs(self) -> list[OutputEffect] | None:
@@ -205,46 +214,91 @@ def outputs(self) -> list[OutputEffect] | None:
205214

206215
@contextual
207216
def output(self) -> OutputEffect | None:
208-
if (tkn := self.expect(lx.IDENTIFIER)):
217+
if tkn := self.expect(lx.IDENTIFIER):
209218
return StackEffect(tkn.text)
210219

211220
@contextual
212221
def super_def(self) -> Super | None:
213-
if (tkn := self.expect(lx.IDENTIFIER)) and (kind := tkn.text) in ("super", "macro"):
222+
if (tkn := self.expect(lx.IDENTIFIER)) and tkn.text == "super":
214223
if self.expect(lx.LPAREN):
215-
if (tkn := self.expect(lx.IDENTIFIER)):
224+
if tkn := self.expect(lx.IDENTIFIER):
216225
if self.expect(lx.RPAREN):
217226
if self.expect(lx.EQUALS):
218227
if ops := self.ops():
219-
res = Super(kind, tkn.text, ops)
228+
self.require(lx.SEMI)
229+
res = Super(tkn.text, ops)
220230
return res
221231

222-
def ops(self) -> list[str] | None:
223-
if tkn := self.expect(lx.IDENTIFIER):
224-
ops = [tkn.text]
232+
def ops(self) -> list[OpName] | None:
233+
if op := self.op():
234+
ops = [op]
225235
while self.expect(lx.PLUS):
226-
if tkn := self.require(lx.IDENTIFIER):
227-
ops.append(tkn.text)
228-
self.require(lx.SEMI)
236+
if op := self.op():
237+
ops.append(op)
229238
return ops
230239

240+
@contextual
241+
def op(self) -> OpName | None:
242+
if tkn := self.expect(lx.IDENTIFIER):
243+
return OpName(tkn.text)
244+
245+
@contextual
246+
def macro_def(self) -> Macro | None:
247+
if (tkn := self.expect(lx.IDENTIFIER)) and tkn.text == "macro":
248+
if self.expect(lx.LPAREN):
249+
if tkn := self.expect(lx.IDENTIFIER):
250+
if self.expect(lx.RPAREN):
251+
if self.expect(lx.EQUALS):
252+
if uops := self.uops():
253+
self.require(lx.SEMI)
254+
res = Macro(tkn.text, uops)
255+
return res
256+
257+
def uops(self) -> list[UOp] | None:
258+
if uop := self.uop():
259+
uops = [uop]
260+
while self.expect(lx.PLUS):
261+
if uop := self.uop():
262+
uops.append(uop)
263+
else:
264+
raise self.make_syntax_error("Expected op name or cache effect")
265+
return uops
266+
267+
@contextual
268+
def uop(self) -> UOp | None:
269+
if tkn := self.expect(lx.IDENTIFIER):
270+
if self.expect(lx.DIVIDE):
271+
if num := self.expect(lx.NUMBER):
272+
try:
273+
size = int(num.text)
274+
except ValueError:
275+
raise self.make_syntax_error(
276+
f"Expected integer, got {num.text!r}"
277+
)
278+
else:
279+
return CacheEffect(tkn.text, size)
280+
raise self.make_syntax_error("Expected integer")
281+
else:
282+
return OpName(tkn.text)
283+
231284
@contextual
232285
def family_def(self) -> Family | None:
233286
if (tkn := self.expect(lx.IDENTIFIER)) and tkn.text == "family":
234287
size = None
235288
if self.expect(lx.LPAREN):
236-
if (tkn := self.expect(lx.IDENTIFIER)):
289+
if tkn := self.expect(lx.IDENTIFIER):
237290
if self.expect(lx.COMMA):
238291
if not (size := self.expect(lx.IDENTIFIER)):
239-
raise self.make_syntax_error(
240-
"Expected identifier")
292+
raise self.make_syntax_error("Expected identifier")
241293
if self.expect(lx.RPAREN):
242294
if self.expect(lx.EQUALS):
243295
if not self.expect(lx.LBRACE):
244296
raise self.make_syntax_error("Expected {")
245297
if members := self.members():
246298
if self.expect(lx.RBRACE) and self.expect(lx.SEMI):
247-
return Family(tkn.text, size.text if size else "", members)
299+
return Family(
300+
tkn.text, size.text if size else "", members
301+
)
248302
return None
249303

250304
def members(self) -> list[str] | None:
@@ -284,6 +338,7 @@ def c_blob(self) -> list[lx.Token]:
284338

285339
if __name__ == "__main__":
286340
import sys
341+
287342
if sys.argv[1:]:
288343
filename = sys.argv[1]
289344
if filename == "-c" and sys.argv[2:]:
@@ -295,10 +350,10 @@ def c_blob(self) -> list[lx.Token]:
295350
srclines = src.splitlines()
296351
begin = srclines.index("// BEGIN BYTECODES //")
297352
end = srclines.index("// END BYTECODES //")
298-
src = "\n".join(srclines[begin+1 : end])
353+
src = "\n".join(srclines[begin + 1 : end])
299354
else:
300355
filename = "<default>"
301356
src = "if (x) { x.foo; // comment\n}"
302357
parser = Parser(src, filename)
303-
x = parser.inst_def() or parser.super_def() or parser.family_def()
358+
x = parser.definition()
304359
print(x)

0 commit comments

Comments
 (0)
Please sign in to comment.