OILS / pea / pea_main.py View on Github | oilshell.org

428 lines, 237 significant
1#!/usr/bin/env python3
2"""
3pea_main.py
4
5A potential rewrite of mycpp.
6"""
7import ast
8from ast import AST, stmt, Module, ClassDef, FunctionDef, Assign
9import collections
10from dataclasses import dataclass
11import io
12import optparse
13import os
14import pickle
15from pprint import pprint
16import sys
17import time
18
19if 0:
20 for p in sys.path:
21 print('*** syspath: %s' % p)
22
23import typing
24from typing import Optional, Any
25
26from mycpp import pass_state
27
28
29START_TIME = time.time()
30
31def log(msg: str, *args: Any) -> None:
32 if args:
33 msg = msg % args
34 print('%.2f %s' % (time.time() - START_TIME, msg), file=sys.stderr)
35
36
37@dataclass
38class PyFile:
39 filename: str
40 namespace: str # C++ namespace
41 module: ast.Module # parsed representation
42
43
44class Program:
45 """A program is a collection of PyFiles."""
46
47 def __init__(self) -> None:
48 self.py_files : list[PyFile] = []
49
50 # As we parse, we add modules, and fill in the dictionaries with parsed
51 # types. Then other passes can retrieve the types with the same
52 # dictionaries.
53
54 # right now types are modules? Could change that
55 self.func_types: dict[FunctionDef, AST] = {}
56 self.method_types : dict[FunctionDef, AST] = {}
57 self.class_types : dict[ClassDef, Module] = {}
58 self.assign_types : dict[Assign, Module] = {}
59
60 # like mycpp: type and variable string. TODO: We shouldn't flatten it to a
61 # C type until later.
62 #
63 # Note: ImplPass parses the types. So I guess this could be limited to
64 # that?
65 # DoFunctionMethod() could make two passes?
66 # 1. collect vars
67 # 2. print code
68
69 self.local_vars : dict[FunctionDef, list[tuple[str, str]]] = {}
70
71 # ForwardDeclPass:
72 # OnMethod()
73 # OnSubclass()
74
75 # Then
76 # Calculate()
77 #
78 # PrototypesPass: # IsVirtual
79 self.virtual = pass_state.Virtual()
80
81 self.stats: dict[str, int] = {
82 # parsing stats
83 'num_files': 0,
84 'num_funcs': 0,
85 'num_classes': 0,
86 'num_methods': 0,
87 'num_assign': 0,
88
89 # ConstPass stats
90 'num_strings': 0,
91 }
92
93 def PrintStats(self) -> None:
94 pprint(self.stats, stream=sys.stderr)
95 print('', file=sys.stderr)
96
97
98class TypeSyntaxError(Exception):
99
100 def __init__(self, lineno: int, code_str: str):
101 self.lineno = lineno
102 self.code_str = code_str
103
104
105def ParseFiles(files: list[str], prog: Program) -> bool:
106
107 for filename in files:
108 with open(filename) as f:
109 contents = f.read()
110
111 try:
112 # Python 3.8+ supports type_comments=True
113 module = ast.parse(contents, filename=filename, type_comments=True)
114 except SyntaxError as e:
115 # This raises an exception for some reason
116 #e.print_file_and_line()
117 print('Error parsing %s: %s' % (filename, e))
118 return False
119
120 tmp = os.path.basename(filename)
121 namespace, _ = os.path.splitext(tmp)
122
123 prog.py_files.append(PyFile(filename, namespace, module))
124
125 prog.stats['num_files'] += 1
126
127 return True
128
129
130class ConstVisitor(ast.NodeVisitor):
131
132 def __init__(self, const_lookup: dict[str, int]):
133 ast.NodeVisitor.__init__(self)
134 self.const_lookup = const_lookup
135 self.str_id = 0
136
137 def visit_Constant(self, o: ast.Constant) -> None:
138 if isinstance(o.value, str):
139 self.const_lookup[o.value] = self.str_id
140 self.str_id += 1
141
142
143class ForwardDeclPass:
144 """Emit forward declarations."""
145 # TODO: Move this to ParsePass after comparing with mycpp.
146
147 def __init__(self, f: typing.IO[str]) -> None:
148 self.f = f
149
150 def DoPyFile(self, py_file: PyFile) -> None:
151
152 # TODO: could omit empty namespaces
153 namespace = py_file.namespace
154 self.f.write(f'namespace {namespace} {{ // forward declare\n')
155
156 for stmt in py_file.module.body:
157 match stmt:
158 case ClassDef():
159 class_name = stmt.name
160 self.f.write(f' class {class_name};\n')
161
162 self.f.write(f'}} // forward declare {namespace}\n')
163 self.f.write('\n')
164
165
166def _ParseFuncType(st: stmt) -> AST:
167 assert st.type_comment # caller checks this
168 try:
169 # This parses with the func_type production in the grammar
170 return ast.parse(st.type_comment, mode='func_type')
171 except SyntaxError:
172 raise TypeSyntaxError(st.lineno, st.type_comment)
173
174
175class PrototypesPass:
176 """Parse signatures and Emit function prototypes."""
177
178 def __init__(self, opts: Any, prog: Program, f: typing.IO[str]) -> None:
179 self.opts = opts
180 self.prog = prog
181 self.f = f
182
183 def DoClass(self, cls: ClassDef) -> None:
184 for stmt in cls.body:
185 match stmt:
186 case FunctionDef():
187 if stmt.type_comment:
188 sig = _ParseFuncType(stmt) # may raise
189
190 if self.opts.verbose:
191 print('METHOD')
192 print(ast.dump(sig, indent=' '))
193 # TODO: We need to print virtual here
194
195 self.prog.method_types[stmt] = sig # save for ImplPass
196 self.prog.stats['num_methods'] += 1
197
198 # TODO: assert that there aren't top-level statements?
199 case _:
200 pass
201
202 def DoPyFile(self, py_file: PyFile) -> None:
203 for stmt in py_file.module.body:
204 match stmt:
205 case FunctionDef():
206 if stmt.type_comment:
207 sig = _ParseFuncType(stmt) # may raise
208
209 if self.opts.verbose:
210 print('FUNC')
211 print(ast.dump(sig, indent=' '))
212
213 self.prog.func_types[stmt] = sig # save for ImplPass
214
215 self.prog.stats['num_funcs'] += 1
216
217 case ClassDef():
218 self.DoClass(stmt)
219 self.prog.stats['num_classes'] += 1
220
221 case _:
222 # Import, Assign, etc.
223 #print(stmt)
224
225 # TODO: omit __name__ == '__main__' etc.
226 # if __name__ == '__main__'
227 pass
228
229
230class ImplPass:
231 """Emit function and method bodies.
232
233 Algorithm:
234 collect local variables first
235 """
236
237 def __init__(self, prog: Program, f: typing.IO[str]) -> None:
238 self.prog = prog
239 self.f = f
240
241 # TODO: needs to be fully recursive, so you get bodies of loops, etc.
242 def DoBlock(self, stmts: list[stmt], indent: int=0) -> None:
243 """e.g. body of function, method, etc."""
244
245
246 #print('STMTS %s' % stmts)
247
248 ind_str = ' ' * indent
249
250 for stmt in stmts:
251 match stmt:
252 case Assign():
253 #print('%s* Assign' % ind_str)
254 #print(ast.dump(stmt, indent=' '))
255
256 if stmt.type_comment:
257 # This parses with the func_type production in the grammar
258 try:
259 typ = ast.parse(stmt.type_comment)
260 except SyntaxError as e:
261 # New syntax error
262 raise TypeSyntaxError(stmt.lineno, stmt.type_comment)
263
264 self.prog.assign_types[stmt] = typ
265
266 #print('%s TYPE: Assign' % ind_str)
267 #print(ast.dump(typ, indent=' '))
268
269 self.prog.stats['num_assign'] += 1
270
271 case _:
272 pass
273
274 def DoClass(self, cls: ClassDef) -> None:
275 for stmt in cls.body:
276 match stmt:
277 case FunctionDef():
278 self.DoBlock(stmt.body, indent=1)
279
280 case _:
281 pass
282
283 def DoPyFile(self, py_file: PyFile) -> None:
284 for stmt in py_file.module.body:
285 match stmt:
286 case ClassDef():
287 self.DoClass(stmt)
288
289 case FunctionDef():
290 self.DoBlock(stmt.body, indent=1)
291
292
293def Options() -> optparse.OptionParser:
294 """Returns an option parser instance."""
295
296 p = optparse.OptionParser()
297 p.add_option(
298 '-v', '--verbose', dest='verbose', action='store_true', default=False,
299 help='Show details about translation')
300
301 # Control which modules are exported to the header. Used by
302 # build/translate.sh.
303 p.add_option(
304 '--to-header', dest='to_header', action='append', default=[],
305 help='Export this module to a header, e.g. frontend.args')
306
307 p.add_option(
308 '--header-out', dest='header_out', default=None,
309 help='Write this header')
310
311 return p
312
313
314def main(argv: list[str]) -> int:
315
316 o = Options()
317 opts, argv = o.parse_args(argv)
318
319 action = argv[1]
320
321 # TODO: get rid of 'parse'
322 if action in ('parse', 'cpp'):
323 files = argv[2:]
324
325 # TODO:
326 # pass_state.Virtual
327 # this loops over functions and methods. But it has to be done BEFORE
328 # the PrototypesPass, or we need two passes. Gah!
329 # Could it be done in ConstVisitor? ConstVirtualVisitor?
330
331 # local_vars
332
333 prog = Program()
334 log('Pea begin')
335
336 if not ParseFiles(files, prog):
337 return 1
338 log('Parsed %d files and their type comments', len(files))
339 prog.PrintStats()
340
341 # This is the first pass
342
343 const_lookup: dict[str, int] = {}
344
345 v = ConstVisitor(const_lookup)
346 for py_file in prog.py_files:
347 v.visit(py_file.module)
348
349 log('Collected %d constants', len(const_lookup))
350
351 # TODO: respect header_out for these two passes
352 #out_f = sys.stdout
353 out_f = io.StringIO()
354
355 # ForwardDeclPass: module -> class
356 # TODO: Move trivial ForwardDeclPass into ParsePass, BEFORE constants,
357 # after comparing output with mycpp.
358 pass2 = ForwardDeclPass(out_f)
359 for py_file in prog.py_files:
360 namespace = py_file.namespace
361 pass2.DoPyFile(py_file)
362
363 log('Wrote forward declarations')
364 prog.PrintStats()
365
366 try:
367 # PrototypesPass: module -> class/method, func
368
369 pass3 = PrototypesPass(opts, prog, out_f)
370 for py_file in prog.py_files:
371 pass3.DoPyFile(py_file) # parses type comments in signatures
372
373 log('Wrote prototypes')
374 prog.PrintStats()
375
376 # ImplPass: module -> class/method, func; then probably a fully recursive thing
377
378 pass4 = ImplPass(prog, out_f)
379 for py_file in prog.py_files:
380 pass4.DoPyFile(py_file) # parses type comments in assignments
381
382 log('Wrote implementation')
383 prog.PrintStats()
384
385 except TypeSyntaxError as e:
386 log('Type comment syntax error on line %d of %s: %r',
387 e.lineno, py_file.filename, e.code_str)
388 return 1
389
390 log('Done')
391
392 elif action == 'dump-pickles':
393 files = argv[2:]
394
395 prog = Program()
396 log('Pea begin')
397
398 if not ParseFiles(files, prog):
399 return 1
400 log('Parsed %d files and their type comments', len(files))
401 prog.PrintStats()
402
403 # Note: can't use marshal here, because it only accepts simple types
404 pickle.dump(prog.py_files, sys.stdout.buffer)
405 log('Dumped pickle')
406
407 elif action == 'load-pickles':
408 while True:
409 try:
410 py_files = pickle.load(sys.stdin.buffer)
411 except EOFError:
412 break
413 log('Loaded pickle with %d files', len(py_files))
414
415 else:
416 raise RuntimeError('Invalid action %r' % action)
417
418 return 0
419
420
421if __name__ == '__main__':
422 try:
423 sys.exit(main(sys.argv))
424 except RuntimeError as e:
425 print('FATAL: %s' % e, file=sys.stderr)
426 sys.exit(1)
427
428# vim: sw=2