OILS / mycpp / NINJA_subgraph.py View on Github | oilshell.org

443 lines, 284 significant
1"""
2mycpp/NINJA_subgraph.py
3"""
4
5from __future__ import print_function
6
7import os
8import sys
9
10from build.ninja_lib import log, COMPILERS_VARIANTS, OTHER_VARIANTS
11
12_ = log
13
14
15def DefineTargets(ru):
16
17 ru.py_binary('mycpp/mycpp_main.py',
18 deps_base_dir='prebuilt/ninja',
19 template='mycpp')
20
21 ru.cc_library(
22 '//mycpp/runtime',
23 # Could separate into //mycpp/runtime_{marksweep,bumpleak}
24 srcs=[
25 'mycpp/bump_leak_heap.cc',
26 'mycpp/gc_builtins.cc',
27 'mycpp/gc_mops.cc',
28 'mycpp/gc_mylib.cc',
29 'mycpp/gc_str.cc',
30 'mycpp/hash.cc',
31 'mycpp/mark_sweep_heap.cc',
32 ])
33
34 # Special test with -D
35 ru.cc_binary('mycpp/bump_leak_heap_test.cc',
36 deps=['//mycpp/runtime'],
37 matrix=[
38 ('cxx', 'asan+bumpleak'),
39 ('cxx', 'ubsan+bumpleak'),
40 ('clang', 'ubsan+bumpleak'),
41 ('clang', 'coverage+bumpleak'),
42 ],
43 phony_prefix='mycpp-unit')
44
45 for test_main in [
46 'mycpp/mark_sweep_heap_test.cc',
47 'mycpp/gc_heap_test.cc',
48 'mycpp/gc_stress_test.cc',
49 'mycpp/gc_builtins_test.cc',
50 'mycpp/gc_mops_test.cc',
51 'mycpp/gc_mylib_test.cc',
52 'mycpp/gc_dict_test.cc',
53 'mycpp/gc_list_test.cc',
54 'mycpp/gc_str_test.cc',
55 'mycpp/gc_tuple_test.cc',
56 'mycpp/small_str_test.cc',
57 ]:
58 ru.cc_binary(test_main,
59 deps=['//mycpp/runtime'],
60 matrix=(COMPILERS_VARIANTS + OTHER_VARIANTS),
61 phony_prefix='mycpp-unit')
62
63 ru.cc_binary(
64 'mycpp/float_test.cc',
65 deps=['//mycpp/runtime'],
66 # Just test two compilers, in fast mode
67 matrix=[('cxx', 'opt'), ('clang', 'opt')],
68 phony_prefix='mycpp-unit')
69
70 for test_main in [
71 'mycpp/demo/gc_header.cc',
72 'mycpp/demo/hash_table.cc',
73 'mycpp/demo/target_lang.cc',
74 ]:
75 ru.cc_binary(test_main,
76 deps=['//mycpp/runtime'],
77 matrix=COMPILERS_VARIANTS,
78 phony_prefix='mycpp-unit')
79
80 # ASDL schema that examples/parse.py depends on
81 ru.asdl_library('mycpp/examples/expr.asdl')
82
83
84#
85# mycpp/examples build config
86#
87
88# TODO:
89# - Fold this dependency into a proper shwrap wrapper
90# - Make a n.build() wrapper that takes it into account automatically
91RULES_PY = 'build/ninja-rules-py.sh'
92
93# special ones in examples.sh:
94# - parse
95# - lexer_main -- these use Oil code
96# - pgen2_demo -- uses pgen2
97
98
99def ShouldSkipBuild(name):
100 if name.startswith('invalid_'):
101 return True
102
103 if name in [
104 # these use Oil code, and don't type check or compile. Maybe give up on
105 # them? pgen2_demo might be useful later.
106 'lexer_main',
107 'pgen2_demo',
108 ]:
109 return True
110
111 return False
112
113
114def ExamplesToBuild():
115 filenames = os.listdir('mycpp/examples')
116 py = [
117 name[:-3] for name in filenames
118 if name.endswith('.py') and name != '__init__.py'
119 ]
120
121 to_test = [name for name in py if not ShouldSkipBuild(name)]
122
123 return to_test
124
125
126def ShouldSkipTest(name):
127 return False
128
129
130def ShouldSkipBenchmark(name):
131 return name.startswith('test_')
132
133
134TRANSLATE_FILES = {
135 # TODO: We could also use app_deps.py here
136 # BUG: modules.py must be listed last. Order matters with inheritance
137 # across modules!
138 'modules': [
139 'mycpp/testpkg/module1.py',
140 'mycpp/testpkg/module2.py',
141 'mycpp/examples/modules.py',
142 ],
143 'parse': [], # added dynamically from mycpp/examples/parse.translate.txt
144}
145
146# Unused. Could use mycpp/examples/parse.typecheck.txt
147EXAMPLES_PY = {
148 'parse': [],
149}
150
151
152def TranslatorSubgraph(ru, translator, ex):
153 n = ru.n
154
155 raw = '_gen/mycpp/examples/%s_raw.%s.cc' % (ex, translator)
156
157 # Translate to C++
158 if ex in TRANSLATE_FILES:
159 to_translate = TRANSLATE_FILES[ex]
160 else:
161 to_translate = ['mycpp/examples/%s.py' % ex]
162
163 # Implicit dependency: if the translator changes, regenerate source code.
164 # But don't pass it on the command line.
165 if translator == 'pea':
166 translator_wrapper = '_bin/shwrap/pea_main'
167 base_translator = 'pea'
168 translate_rule = 'translate-pea'
169 else:
170 translate_rule = 'translate-mycpp'
171 base_translator = 'mycpp'
172 translator_wrapper = '_bin/shwrap/mycpp_main'
173
174 translator_vars = [
175 ('mypypath', '$NINJA_REPO_ROOT/mycpp:$NINJA_REPO_ROOT/pyext'),
176 ]
177 if translator == 'mycpp-souffle':
178 translator_vars.append(('extra_mycpp_opts', '--minimize-stack-roots'))
179
180 n.build(
181 raw,
182 translate_rule,
183 to_translate,
184 implicit=[translator_wrapper],
185 # examples/parse uses pyext/fastfunc.pyi
186 variables=translator_vars)
187
188 p = 'mycpp/examples/%s_preamble.h' % ex
189 # Ninja empty string!
190 preamble_path = p if os.path.exists(p) else "''"
191
192 main_cc_src = '_gen/mycpp/examples/%s.%s.cc' % (ex, translator)
193
194 # Make a translation unit
195 n.build(main_cc_src,
196 'wrap-cc',
197 raw,
198 implicit=[RULES_PY],
199 variables=[('name', ex), ('preamble_path', preamble_path),
200 ('translator', base_translator)])
201
202 n.newline()
203
204 if translator == 'pea':
205 ru.phony['pea-translate'].append(main_cc_src)
206
207 if translator == 'mycpp':
208 example_matrix = COMPILERS_VARIANTS
209 elif translator == 'mycpp-souffle':
210 # mycpp-souffle only has three variants for now
211 example_matrix = [
212 ('cxx', 'opt'), # for benchmarks
213 ('cxx', 'opt-sh'), # for benchmarks
214 ('cxx', 'asan'), # need this for running the examples in CI
215 ('cxx', 'asan+gcalways'),
216 ]
217 else:
218 # pea just has one variant for now
219 example_matrix = [('cxx', 'asan+gcalways')]
220
221 if translator == 'mycpp':
222 phony_prefix = 'mycpp-examples'
223 else:
224 phony_prefix = ''
225
226 deps = ['//mycpp/runtime']
227 if ex == 'parse':
228 deps = deps + ['//mycpp/examples/expr.asdl', '//cpp/data_lang']
229
230 ru.cc_binary(
231 main_cc_src,
232 deps=deps,
233 matrix=example_matrix,
234 phony_prefix=phony_prefix,
235 )
236
237 # TODO:
238 # - restore lost 'pea-compile' tag?
239
240
241def NinjaGraph(ru):
242 n = ru.n
243
244 ru.comment('Generated by %s' % __name__)
245
246 # Running build/ninja_main.py
247 this_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
248
249 n.variable('NINJA_REPO_ROOT', os.path.dirname(this_dir))
250 n.newline()
251
252 # mycpp and pea have the same interface
253 n.rule('translate-mycpp',
254 command='_bin/shwrap/mycpp_main $mypypath $out $in $extra_mycpp_opts',
255 description='mycpp $mypypath $out $in')
256 n.newline()
257
258 n.rule('translate-pea',
259 command='_bin/shwrap/pea_main $mypypath $out $in',
260 description='pea $mypypath $out $in')
261 n.newline()
262
263 n.rule(
264 'wrap-cc',
265 command=
266 'build/ninja-rules-py.sh wrap-cc $out $translator $name $in $preamble_path',
267 description='wrap-cc $out $translator $name $in $preamble_path $out')
268 n.newline()
269 n.rule(
270 'example-task',
271 # note: $out can be MULTIPLE FILES, shell-quoted
272 command='build/ninja-rules-py.sh example-task $name $impl $bin $out',
273 description='example-task $name $impl $bin $out')
274 n.newline()
275 n.rule(
276 'typecheck',
277 command='build/ninja-rules-py.sh typecheck $main_py $out $skip_imports',
278 description='typecheck $main_py $out $skip_imports')
279 n.newline()
280 n.rule('logs-equal',
281 command='build/ninja-rules-py.sh logs-equal $out $in',
282 description='logs-equal $out $in')
283 n.newline()
284 n.rule('benchmark-table',
285 command='build/ninja-rules-py.sh benchmark-table $out $in',
286 description='benchmark-table $out $in')
287 n.newline()
288
289 # For simplicity, this is committed to the repo. We could also have
290 # build/dev.sh minimal generate it?
291 with open('mycpp/examples/parse.translate.txt') as f:
292 for line in f:
293 path = line.strip()
294 TRANSLATE_FILES['parse'].append(path)
295
296 examples = ExamplesToBuild()
297 #examples = ['cgi', 'containers', 'fib_iter']
298
299 # Groups of targets. Not all of these are run by default.
300 ph = {
301 'mycpp-typecheck':
302 [], # optional: for debugging only. translation does it.
303 'mycpp-strip':
304 [], # optional: strip binaries. To see how big they are.
305
306 # Compare logs for tests AND benchmarks.
307 # It's a separate task because we have multiple variants to compare, and
308 # the timing of test/benchmark tasks should NOT include comparison.
309 'mycpp-logs-equal': [],
310
311 # NOTE: _test/benchmark-table.tsv isn't included in any phony target
312
313 # Targets dynamically added:
314 #
315 # mycpp-unit-$compiler-$variant
316 # mycpp-examples-$compiler-$variant
317 'pea-translate': [],
318 'pea-compile': [],
319 # TODO: eventually we will have pea-logs-equal, and pea-benchmark-table
320 }
321 ru.AddPhony(ph)
322
323 DefineTargets(ru)
324
325 #
326 # Build and run examples/
327 #
328
329 to_compare = []
330 benchmark_tasks = []
331
332 for ex in examples:
333 ru.comment('- mycpp/examples/%s' % ex)
334
335 # TODO: make a phony target for these, since they're not strictly necessary.
336 # Translation does everything that type checking does. Type checking only
337 # is useful for debugging.
338 t = '_test/tasks/typecheck/%s.log.txt' % ex
339 main_py = 'mycpp/examples/%s.py' % ex
340
341 # expr.asdl needs to import pylib.collections_, which doesn't type check
342 skip_imports = 'T' if (ex == 'parse') else "''"
343
344 n.build(
345 [t],
346 'typecheck',
347 # TODO: Use mycpp/examples/parse.typecheck.txt
348 EXAMPLES_PY.get(ex, []) + [main_py],
349 variables=[('main_py', main_py), ('skip_imports', skip_imports)])
350 n.newline()
351 ru.phony['mycpp-typecheck'].append(t)
352
353 # Run Python.
354 for mode in ['test', 'benchmark']:
355 prefix = '_test/tasks/%s/%s.py' % (mode, ex)
356 task_out = '%s.task.txt' % prefix
357
358 if mode == 'benchmark':
359 if ShouldSkipBenchmark(ex):
360 #log('Skipping benchmark of %s', ex)
361 continue
362 benchmark_tasks.append(task_out)
363
364 elif mode == 'test':
365 if ShouldSkipTest(ex):
366 #log('Skipping test of %s', ex)
367 continue
368
369 # TODO: This should be a Python stub!
370 log_out = '%s.log' % prefix
371 n.build([task_out, log_out],
372 'example-task',
373 EXAMPLES_PY.get(ex, []) + ['mycpp/examples/%s.py' % ex],
374 variables=[('bin', main_py), ('name', ex),
375 ('impl', 'Python')])
376
377 n.newline()
378
379 for translator in ['mycpp', 'mycpp-souffle', 'pea']:
380 TranslatorSubgraph(ru, translator, ex)
381
382 # Don't run it for now; just compile
383 if translator == 'pea':
384 continue
385
386 # minimal
387 MATRIX = [
388 ('test', 'asan'), # TODO: asan+gcalways is better!
389 ('benchmark', 'opt'),
390 ]
391
392 # Run the binary in two ways
393 for mode, variant in MATRIX:
394 task_out = '_test/tasks/%s/%s.%s.%s.task.txt' % (
395 mode, ex, translator, variant)
396
397 if mode == 'benchmark':
398 if ShouldSkipBenchmark(ex):
399 #log('Skipping benchmark of %s', ex)
400 continue
401 benchmark_tasks.append(task_out)
402
403 elif mode == 'test':
404 if ShouldSkipTest(ex):
405 #log('Skipping test of %s', ex)
406 continue
407
408 cc_log_out = '_test/tasks/%s/%s.%s.%s.log' % (
409 mode, ex, translator, variant)
410 py_log_out = '_test/tasks/%s/%s.py.log' % (mode, ex)
411
412 to_compare.append(cc_log_out)
413 to_compare.append(py_log_out)
414
415 # Only test cxx- variant
416 b_example = '_bin/cxx-%s/mycpp/examples/%s.%s' % (variant, ex,
417 translator)
418 impl = 'C++'
419 if translator == 'mycpp-souffle':
420 impl = 'C++-Souffle'
421
422 n.build([task_out, cc_log_out],
423 'example-task', [b_example],
424 variables=[('bin', b_example), ('name', ex),
425 ('impl', impl)])
426 n.newline()
427
428 # Compare the log of all examples
429 out = '_test/mycpp-compare-passing.txt'
430 n.build([out], 'logs-equal', to_compare)
431 n.newline()
432
433 # NOTE: Don't really need this
434 ru.phony['mycpp-logs-equal'].append(out)
435
436 # Timing of benchmarks
437 out = '_test/benchmark-table.tsv'
438 n.build([out], 'benchmark-table', benchmark_tasks)
439 n.newline()
440
441 ru.souffle_binary('prebuilt/datalog/call-graph.cc')
442 ru.souffle_binary('prebuilt/datalog/dataflow.cc')
443 ru.souffle_binary('prebuilt/datalog/smoke-test.cc')