OILS / asdl / asdl_main.py View on Github | oils.pub

343 lines, 202 significant
1#!/usr/bin/env python2
2"""
3asdl_main.py - Generate Python and C from ASDL schemas.
4"""
5from __future__ import print_function
6
7import optparse
8import os
9import sys
10
11from asdl import ast
12from asdl import front_end
13from asdl import gen_cpp
14from asdl import gen_python
15from asdl import metrics
16from asdl.util import log
17
18from typing import Dict
19from typing import Any
20from typing import List
21
22ARG_0 = os.path.basename(sys.argv[0])
23
24
25def Options():
26 # type: () -> Any
27 """Returns an option parser instance."""
28
29 p = optparse.OptionParser()
30 p.add_option('--no-pretty-print-methods',
31 dest='pretty_print_methods',
32 action='store_false',
33 default=True,
34 help='Whether to generate pretty printing methods')
35
36 # Control Python constructors
37
38 # for hnode.asdl
39 p.add_option('--py-init-N',
40 dest='py_init_n',
41 action='store_true',
42 default=False,
43 help='Generate Python __init__ that requires every field')
44
45 # The default, which matches C++
46 p.add_option(
47 '--init-zero-N',
48 dest='init_zero_n',
49 action='store_true',
50 default=True,
51 help='Generate 0 arg and N arg constructors, in Python and C++')
52
53 p.add_option('--abbrev-module',
54 dest='abbrev_module',
55 default=None,
56 help='Import this module to find abbreviations')
57
58 return p
59
60
61def main(argv):
62 # type: (List[str]) -> None
63 o = Options()
64 opts, argv = o.parse_args(argv)
65
66 try:
67 action = argv[1]
68 except IndexError:
69 raise RuntimeError('Action required')
70
71 try:
72 schema_path = argv[2]
73 except IndexError:
74 raise RuntimeError('Schema path required')
75
76 schema_filename = os.path.basename(schema_path)
77 if opts.abbrev_module:
78 # Weird Python rule for importing: fromlist needs to be non-empty.
79 abbrev_mod = __import__(opts.abbrev_module, fromlist=['.'])
80 else:
81 abbrev_mod = None
82
83 abbrev_mod_entries = dir(abbrev_mod) if abbrev_mod else []
84 # e.g. syntax_abbrev
85 abbrev_ns = opts.abbrev_module.split('.')[-1] if abbrev_mod else None
86
87 if action == 'metrics': # Sum type metrics
88 with open(schema_path) as f:
89 schema_ast, _ = front_end.LoadSchema(f)
90
91 v = metrics.MetricsVisitor(sys.stdout)
92 v.VisitModule(schema_ast)
93
94 elif action == 'closure': # count all types that command_t references
95 type_name = argv[3]
96 with open(schema_path) as f:
97 schema_ast, type_lookup = front_end.LoadSchema(f)
98
99 seen = {} # type: Dict[str, bool]
100 c = metrics.ClosureWalk(type_lookup, seen)
101 c.DoModule(schema_ast, type_name)
102 for name in sorted(seen):
103 print(name)
104 log('SHARED (%d): %s', len(c.shared), ' '.join(sorted(c.shared)))
105
106 elif action == 'c': # Generate C code for the lexer
107 with open(schema_path) as f:
108 schema_ast, _ = front_end.LoadSchema(f)
109
110 v0 = gen_cpp.CEnumVisitor(sys.stdout)
111 v0.VisitModule(schema_ast)
112
113 elif action == 'cpp': # Generate C++ code for ASDL schemas
114 out_prefix = argv[3]
115
116 with open(schema_path) as f:
117 schema_ast, _ = front_end.LoadSchema(f)
118
119 # asdl/typed_arith.asdl -> typed_arith_asdl
120 ns = os.path.basename(schema_path).replace('.', '_')
121
122 with open(out_prefix + '.h', 'w') as f:
123 guard = ns.upper()
124 f.write("""\
125// %s.h is generated by %s
126
127#ifndef %s
128#define %s
129
130""" % (out_prefix, ARG_0, guard, guard))
131
132 f.write("""\
133#include <cstdint>
134""")
135 f.write("""
136#include "mycpp/runtime.h"
137""")
138 if opts.pretty_print_methods:
139 f.write('#include "asdl/cpp_runtime.h"\n')
140
141 for use in schema_ast.uses:
142 # HACK to work around limitation: we can't "use" simple enums
143 if use.module_parts == ['frontend', 'id_kind']:
144 f.write('namespace %s_asdl { typedef uint16_t Id_t; }\n' %
145 use.module_parts[-1])
146 continue
147
148 # Forward declarations in the header, like
149 # namespace syntax_asdl { class command_t; }
150 # must come BEFORE namespace, so it can't be in the visitor.
151
152 # assume sum type for now!
153 cpp_names = []
154 for n in use.type_names:
155 py_name, _ = ast.TypeNameHeuristic(n)
156 cpp_names.append('class %s;' % py_name)
157
158 f.write('namespace %s_asdl { %s }\n' %
159 (use.module_parts[-1], ' '.join(cpp_names)))
160 f.write('\n')
161
162 for extern in schema_ast.externs:
163 names = extern.names
164 type_name = names[-1]
165 cpp_namespace = names[-2]
166
167 # TODO: This 'extern' feature isn't enough for Oils
168 # I think we would have to export header to
169 # _gen/bin/oils_for_unix.mycpp.cc or something
170 # Does that create circular dependencies?
171 #
172 # Or maybe of 'extern' we can have 'include' or something?
173 # Maybe we need `.pyi` files in MyPy?
174
175 f.write("""\
176namespace %s {
177class %s {
178 public:
179 hnode_t* PrettyTree(bool do_abbrev, Dict<int, bool>* seen);
180};
181}
182""" % (cpp_namespace, type_name))
183
184 f.write("""\
185namespace %s {
186
187// use struct instead of namespace so 'using' works consistently
188#define ASDL_NAMES struct
189
190""" % ns)
191
192 v1 = gen_cpp.ForwardDeclareVisitor(f)
193 v1.VisitModule(schema_ast)
194
195 debug_info = {} # type: Dict[str, int]
196 v2 = gen_cpp.ClassDefVisitor(
197 f,
198 pretty_print_methods=opts.pretty_print_methods,
199 debug_info=debug_info)
200 v2.VisitModule(schema_ast)
201
202 f.write("""
203} // namespace %s
204
205#endif // %s
206""" % (ns, guard))
207
208 try:
209 debug_info_path = argv[4]
210 except IndexError:
211 pass
212 else:
213 with open(debug_info_path, 'w') as f:
214 from pprint import pformat
215 f.write('''\
216cpp_namespace = %r
217tags_to_types = \\
218%s
219''' % (ns, pformat(debug_info)))
220
221 if not opts.pretty_print_methods:
222 # No .cc file at all
223 return
224
225 with open(out_prefix + '.cc', 'w') as f:
226 f.write("""\
227// %s.cc is generated by %s
228
229#include "%s.h"
230#include <assert.h>
231""" % (out_prefix, ARG_0, out_prefix))
232
233 if abbrev_mod_entries:
234 # This is somewhat hacky, works for frontend/syntax_abbrev.py and
235 # prebuilt/frontend/syntax_abbrev.mycpp.h
236 part0, part1 = opts.abbrev_module.split('.')
237 f.write("""\
238#include "prebuilt/%s/%s.mycpp.h"
239""" % (part0, part1))
240
241 f.write("""\
242#include "prebuilt/asdl/runtime.mycpp.h" // generated code uses wrappers here
243""")
244
245 # To call pretty-printing methods
246 for use in schema_ast.uses:
247 f.write('#include "_gen/%s.asdl.h" // "use" in ASDL\n' %
248 '/'.join(use.module_parts))
249 # HACK
250 if use.module_parts == ['frontend', 'id_kind']:
251 f.write('using id_kind_asdl::Id_str;\n')
252
253 f.write("""\
254
255// Generated code uses these types
256using hnode_asdl::hnode;
257using hnode_asdl::Field;
258using hnode_asdl::color_e;
259
260""")
261
262 f.write("""
263namespace %s {
264
265""" % ns)
266
267 v3 = gen_cpp.MethodDefVisitor(
268 f,
269 abbrev_ns=abbrev_ns,
270 abbrev_mod_entries=abbrev_mod_entries)
271 v3.VisitModule(schema_ast)
272
273 f.write("""
274} // namespace %s
275""" % ns)
276
277 elif action == 'mypy': # Generated typed MyPy code
278 with open(schema_path) as f:
279 schema_ast, _ = front_end.LoadSchema(f)
280
281 f = sys.stdout # type: ignore
282
283 # TODO: Remove Any once we stop using it
284 f.write("""\
285from asdl import pybase
286from mycpp import mops
287from typing import Optional, List, Tuple, Dict, Any, cast, TYPE_CHECKING
288""")
289
290 for use in schema_ast.uses:
291 # HACK
292 if use.module_parts == ['frontend', 'id_kind']:
293 f.write('from _devbuild.gen.id_kind_asdl import Id_str\n')
294
295 if schema_ast.uses:
296 f.write('\n')
297 f.write('if TYPE_CHECKING:\n')
298 for use in schema_ast.uses:
299 py_names = []
300 for n in use.type_names:
301 py_name, _ = ast.TypeNameHeuristic(n)
302 py_names.append(py_name)
303
304 # indented
305 f.write(' from _devbuild.gen.%s_asdl import %s\n' %
306 (use.module_parts[-1], ', '.join(py_names)))
307
308 if schema_ast.externs:
309 f.write('\n')
310 f.write('if TYPE_CHECKING:\n')
311 for extern in schema_ast.externs:
312 names = extern.names
313 mod_parts = names[:-2]
314 f.write(' from %s import %s\n' % ('.'.join(mod_parts), names[-2]))
315
316 if opts.pretty_print_methods:
317 f.write("""
318from asdl import runtime # For runtime.NO_SPID
319from asdl.runtime import NewRecord, NewLeaf, TraversalState
320from _devbuild.gen.hnode_asdl import color_e, hnode, hnode_e, hnode_t, Field
321
322""")
323 if opts.abbrev_module:
324 f.write('from %s import *\n' % opts.abbrev_module)
325 f.write('\n')
326
327 v4 = gen_python.GenMyPyVisitor(
328 f,
329 abbrev_mod_entries,
330 pretty_print_methods=opts.pretty_print_methods,
331 py_init_n=opts.py_init_n)
332 v4.VisitModule(schema_ast)
333
334 else:
335 raise RuntimeError('Invalid action %r' % action)
336
337
338if __name__ == '__main__':
339 try:
340 main(sys.argv)
341 except RuntimeError as e:
342 print('%s: FATAL: %s' % (ARG_0, e), file=sys.stderr)
343 sys.exit(1)