OILS / mycpp / NINJA_subgraph.py View on Github | oils.pub

502 lines, 328 significant
1"""
2mycpp/NINJA_subgraph.py
3"""
4
5from __future__ import print_function
6
7import os
8import sys
9
10from build.ninja_lib import (log, mycpp_binary, COMPILERS_VARIANTS,
11 OTHER_VARIANTS, SHWRAP)
12
13_ = log
14_ = mycpp_binary
15
16
17def DefineTargets(ru):
18
19 # Creates _bin/shwrap/mycpp_main
20 ru.py_binary('mycpp/mycpp_main.py',
21 deps_base_dir='prebuilt/ninja',
22 template='mycpp')
23
24 # mycpp wrapper that depends on _bin/datalog/dataflow, a binary created
25 # from Souffle datalog!
26 ru.n.build(
27 '_bin/shwrap/mycpp_main_souffle',
28 'cp',
29 ['bin/mycpp_main_souffle.sh'],
30 implicit=['_bin/shwrap/mycpp_main', '_bin/datalog/dataflow'],
31 )
32
33 ru.cc_library(
34 '//mycpp/runtime',
35 # Could separate into //mycpp/runtime_{marksweep,bumpleak}
36 srcs=[
37 'mycpp/bump_leak_heap.cc',
38 'mycpp/gc_builtins.cc',
39 'mycpp/gc_iolib.cc',
40 'mycpp/gc_mops.cc',
41 'mycpp/gc_mylib.cc',
42 'mycpp/gc_str.cc',
43 'mycpp/hash.cc',
44 'mycpp/mark_sweep_heap.cc',
45 ])
46
47 # Special test with -D
48 ru.cc_binary('mycpp/bump_leak_heap_test.cc',
49 deps=['//mycpp/runtime'],
50 matrix=[
51 ('cxx', 'asan+bumpleak'),
52 ('cxx', 'ubsan+bumpleak'),
53 ('clang', 'ubsan+bumpleak'),
54 ('clang', 'coverage+bumpleak'),
55 ],
56 phony_prefix='mycpp-unit')
57
58 for test_main in [
59 'mycpp/mark_sweep_heap_test.cc',
60 'mycpp/gc_heap_test.cc',
61 'mycpp/gc_stress_test.cc',
62 'mycpp/gc_builtins_test.cc',
63 'mycpp/gc_iolib_test.cc',
64 'mycpp/gc_mops_test.cc',
65 'mycpp/gc_mylib_test.cc',
66 'mycpp/gc_dict_test.cc',
67 'mycpp/gc_list_test.cc',
68 'mycpp/gc_str_test.cc',
69 'mycpp/gc_tuple_test.cc',
70 'mycpp/small_str_test.cc',
71 ]:
72 ru.cc_binary(test_main,
73 deps=['//mycpp/runtime'],
74 matrix=(COMPILERS_VARIANTS + OTHER_VARIANTS),
75 phony_prefix='mycpp-unit')
76
77 ru.cc_binary(
78 'mycpp/float_test.cc',
79 deps=['//mycpp/runtime'],
80 # Just test two compilers, in fast mode
81 matrix=[('cxx', 'opt'), ('clang', 'opt')],
82 phony_prefix='mycpp-unit')
83
84 for test_main in [
85 'mycpp/demo/gc_header.cc',
86 'mycpp/demo/hash_table.cc',
87 'mycpp/demo/target_lang.cc',
88 ]:
89 ru.cc_binary(test_main,
90 deps=['//mycpp/runtime'],
91 matrix=COMPILERS_VARIANTS,
92 phony_prefix='mycpp-unit')
93
94 # ASDL schema that examples/parse.py depends on
95 ru.asdl_library('mycpp/examples/expr.asdl')
96
97
98#
99# mycpp/examples build config
100#
101
102# TODO:
103# - Fold this dependency into a proper shwrap wrapper
104# - Make a n.build() wrapper that takes it into account automatically
105RULES_PY = 'build/ninja-rules-py.sh'
106
107# special ones in examples.sh:
108# - parse
109# - lexer_main -- these use Oil code
110# - pgen2_demo -- uses pgen2
111
112
113def ShouldSkipBuild(name):
114 if name.startswith('invalid_'):
115 return True
116
117 if name in [
118 # these use Oil code, and don't type check or compile. Maybe give up on
119 # them? pgen2_demo might be useful later.
120 'lexer_main',
121 'pgen2_demo',
122 ]:
123 return True
124
125 return False
126
127
128def ExamplesToBuild():
129 filenames = os.listdir('mycpp/examples')
130 py = [
131 name[:-3] for name in filenames
132 if name.endswith('.py') and name != '__init__.py'
133 ]
134
135 to_test = [name for name in py if not ShouldSkipBuild(name)]
136
137 result = []
138 for ex in to_test:
139 py_main = 'mycpp/examples/%s.py' % ex
140 result.append((ex, py_main))
141
142 return result
143
144
145def ShouldSkipTest(name):
146 return False
147
148
149def ShouldSkipBenchmark(name):
150 return name.startswith('test_')
151
152
153TRANSLATE_FILES = {
154 # TODO: We could also use app_deps.py here
155 # BUG: modules.py must be listed last. Order matters with inheritance
156 # across modules!
157 'modules': [
158 'mycpp/testpkg/module1.py',
159 'mycpp/testpkg/module2.py',
160 'mycpp/examples/modules.py',
161 ],
162 'parse': [], # added dynamically from mycpp/examples/parse.translate.txt
163}
164
165# Unused. Could use mycpp/examples/parse.typecheck.txt
166EXAMPLES_PY = {
167 'parse': [],
168}
169
170EXAMPLES_DEPS = {
171 'parse': [
172 '//mycpp/runtime',
173 '//mycpp/examples/expr.asdl',
174 '//cpp/data_lang',
175 ],
176}
177
178# mycpp-souffle only has three variants for now
179SOUFFLE_MATRIX = [
180 ('cxx', 'opt'), # for benchmarks
181 #('cxx', 'opt-sh'), # for benchmarks
182 ('cxx', 'asan'), # need this for running the examples in CI
183 ('cxx', 'asan+gcalways'),
184]
185MYPY_PATH = '$NINJA_REPO_ROOT/mycpp:$NINJA_REPO_ROOT/pyext'
186
187# TODO Next:
188# - separate parse.main.cc files, instead of _gen/_tmp
189
190
191def TranslatorSubgraph(
192 ru,
193 py_main,
194 mypy_path,
195 bin_path=None,
196 symlinks=None,
197 preprocessed=False,
198 preamble=None,
199 translator='mycpp',
200 main_style='example-main-wrapper',
201 py_inputs=None, # list of source files, including main
202 phony_prefix=None,
203 matrix=None,
204 deps=None):
205 """Create rules for a single example."""
206 # e.g. mycpp/examples/parse
207 py_rel_path, _ = os.path.splitext(py_main)
208 # e.g. mycpp.examples.parse
209 py_module = py_rel_path.replace('/', '.')
210
211 py_inputs = py_inputs or [py_main] # if not specified, it's a single file
212 matrix = matrix or COMPILERS_VARIANTS
213 deps = deps or ['//mycpp/runtime']
214 if preamble is None:
215 p = '%s_preamble.h' % py_rel_path
216 preamble = p if os.path.exists(p) else "''" # Ninja empty string!
217
218 n = ru.n
219
220 # Two steps
221 raw = '_gen/_tmp/%s.%s-raw.cc' % (py_rel_path, translator)
222 main_cc_src = '_gen/%s.%s.cc' % (py_rel_path, translator)
223
224 translator_shwrap = SHWRAP[translator]
225
226 n.build(
227 raw,
228 'translate-%s' % translator,
229 py_inputs, # files to translate
230 # Implicit dependency: if the translator changes, regenerate source
231 # code. But don't pass it on the command line.
232 implicit=[translator_shwrap],
233 # examples/parse uses pyext/fastfunc.pyi
234 variables=[('mypypath', mypy_path)])
235
236 # Make a translation unit
237 n.build(main_cc_src,
238 'wrap-cc',
239 raw,
240 implicit=[RULES_PY],
241 variables=[
242 ('main_namespace', os.path.basename(py_rel_path)),
243 ('main_style', main_style),
244 ('preamble', preamble),
245 ])
246
247 n.newline()
248
249 ru.cc_binary(
250 main_cc_src,
251 bin_path=bin_path,
252 symlinks=symlinks,
253 preprocessed=preprocessed,
254 deps=deps,
255 matrix=matrix,
256 phony_prefix=phony_prefix,
257 )
258
259
260def NinjaGraph(ru):
261 n = ru.n
262
263 ru.comment('Generated by %s' % __name__)
264
265 # Running build/ninja_main.py
266 this_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
267
268 n.variable('NINJA_REPO_ROOT', os.path.dirname(this_dir))
269 n.newline()
270
271 # mycpp and pea have the same interface
272 n.rule('translate-mycpp',
273 command='_bin/shwrap/mycpp_main $mypypath $out $in',
274 description='mycpp $mypypath $out $in')
275 n.newline()
276
277 n.rule('translate-mycpp-souffle',
278 command='_bin/shwrap/mycpp_main_souffle $mypypath $out $in',
279 description='mycpp-souffle $mypypath $out $in')
280 n.newline()
281
282 n.rule('translate-pea',
283 command='_bin/shwrap/pea_main $mypypath $out $in',
284 description='pea $mypypath $out $in')
285 n.newline()
286
287 n.rule(
288 'wrap-cc',
289 command=
290 'build/ninja-rules-py.sh wrap-cc $out $main_style $main_namespace $in $preamble',
291 description='wrap-cc $out $main_style $main_namespace $in $preamble')
292 n.newline()
293 n.rule(
294 'example-task',
295 # note: $out can be MULTIPLE FILES, shell-quoted
296 command='build/ninja-rules-py.sh example-task $name $impl $bin $out',
297 description='example-task $name $impl $bin $out')
298 n.newline()
299 n.rule(
300 'typecheck',
301 command='build/ninja-rules-py.sh typecheck $main_py $out $skip_imports',
302 description='typecheck $main_py $out $skip_imports')
303 n.newline()
304 n.rule('logs-equal',
305 command='build/ninja-rules-py.sh logs-equal $out $in',
306 description='logs-equal $out $in')
307 n.newline()
308 n.rule('benchmark-table',
309 command='build/ninja-rules-py.sh benchmark-table $out $in',
310 description='benchmark-table $out $in')
311 n.newline()
312
313 # For simplicity, this is committed to the repo. We could also have
314 # build/dev.sh minimal generate it?
315 with open('_build/NINJA/mycpp.examples.parse/translate.txt') as f:
316 for line in f:
317 path = line.strip()
318 TRANSLATE_FILES['parse'].append(path)
319
320 examples = ExamplesToBuild()
321 #examples = ['cgi', 'containers', 'fib_iter']
322
323 # Groups of targets. Not all of these are run by default.
324 ph = {
325 'mycpp-typecheck':
326 [], # optional: for debugging only. translation does it.
327 'mycpp-strip':
328 [], # optional: strip binaries. To see how big they are.
329
330 # Compare logs for tests AND benchmarks.
331 # It's a separate task because we have multiple variants to compare, and
332 # the timing of test/benchmark tasks should NOT include comparison.
333 'mycpp-logs-equal': [],
334
335 # NOTE: _test/benchmark-table.tsv isn't included in any phony target
336
337 # Targets dynamically added:
338 #
339 # mycpp-unit-$compiler-$variant
340 # mycpp-examples-$compiler-$variant
341 'pea-translate': [],
342 'pea-compile': [],
343 # TODO: eventually we will have pea-logs-equal, and pea-benchmark-table
344 }
345 ru.AddPhony(ph)
346
347 DefineTargets(ru)
348
349 #
350 # Build and run examples/
351 #
352
353 ## Pea Examples
354 for ex, py_main in examples:
355 # Special case: mycpp/examples/pea_* are only translated with pea.
356 # TODO: pea examples don't have the same main()
357
358 if ex.startswith('pea_'):
359 mycpp_binary(ru,
360 py_main,
361 mypy_path=MYPY_PATH,
362 main_style='main-wrapper',
363 translator='pea',
364 matrix=[
365 ('cxx', 'asan'),
366 ('cxx', 'opt'),
367 ],
368 deps=['//mycpp/runtime'])
369
370 to_compare = []
371 benchmark_tasks = []
372
373 for ex, py_main in examples:
374 if ex.startswith('pea_'): # Only non-pea examples
375 continue
376
377 ru.comment('- mycpp/examples/%s' % ex)
378
379 # TODO: make a phony target for these, since they're not strictly necessary.
380 # Translation does everything that type checking does. Type checking only
381 # is useful for debugging.
382 t = '_test/tasks/typecheck/%s.log.txt' % ex
383 main_py = 'mycpp/examples/%s.py' % ex
384
385 # expr.asdl needs to import pylib.collections_, which doesn't type check
386 skip_imports = 'T' if (ex == 'parse') else "''"
387
388 ## Type check the example
389
390 n.build(
391 [t],
392 'typecheck',
393 # TODO: Use mycpp/examples/parse.typecheck.txt
394 EXAMPLES_PY.get(ex, []) + [main_py],
395 variables=[('main_py', main_py), ('skip_imports', skip_imports)])
396 n.newline()
397 ru.phony['mycpp-typecheck'].append(t)
398
399 ## Run example as Python
400
401 for mode in ['test', 'benchmark']:
402 prefix = '_test/tasks/%s/%s.py' % (mode, ex)
403 task_out = '%s.task.txt' % prefix
404
405 if mode == 'benchmark':
406 if ShouldSkipBenchmark(ex):
407 #log('Skipping benchmark of %s', ex)
408 continue
409 benchmark_tasks.append(task_out)
410
411 elif mode == 'test':
412 if ShouldSkipTest(ex):
413 #log('Skipping test of %s', ex)
414 continue
415
416 # TODO: This should be a Python stub!
417 log_out = '%s.log' % prefix
418 n.build([task_out, log_out],
419 'example-task',
420 EXAMPLES_PY.get(ex, []) + ['mycpp/examples/%s.py' % ex],
421 variables=[('bin', main_py), ('name', ex),
422 ('impl', 'Python')])
423
424 n.newline()
425
426 ## Translate the example 2 ways, and benchmark and test it
427
428 for translator in ['mycpp', 'mycpp-souffle']:
429
430 matrix = SOUFFLE_MATRIX if translator == 'mycpp-souffle' else None
431 phony_prefix = 'mycpp-examples' if translator == 'mycpp' else None
432 py_inputs = TRANSLATE_FILES.get(ex)
433
434 deps = EXAMPLES_DEPS.get(ex, ['//mycpp/runtime'])
435 mycpp_binary(ru,
436 py_main,
437 mypy_path=MYPY_PATH,
438 translator=translator,
439 main_style='example-main-wrapper',
440 py_inputs=py_inputs,
441 phony_prefix=phony_prefix,
442 matrix=matrix,
443 deps=deps)
444
445 # minimal
446 TEST_MATRIX = [
447 ('test', 'asan'), # TODO: asan+gcalways is better!
448 ('benchmark', 'opt'),
449 ]
450
451 # Run the binary in two ways
452 for mode, variant in TEST_MATRIX:
453 task_out = '_test/tasks/%s/%s.%s.%s.task.txt' % (
454 mode, ex, translator, variant)
455
456 if mode == 'benchmark':
457 if ShouldSkipBenchmark(ex):
458 #log('Skipping benchmark of %s', ex)
459 continue
460 benchmark_tasks.append(task_out)
461
462 elif mode == 'test':
463 if ShouldSkipTest(ex):
464 #log('Skipping test of %s', ex)
465 continue
466
467 cc_log_out = '_test/tasks/%s/%s.%s.%s.log' % (
468 mode, ex, translator, variant)
469 py_log_out = '_test/tasks/%s/%s.py.log' % (mode, ex)
470
471 to_compare.append(cc_log_out)
472 to_compare.append(py_log_out)
473
474 # Only test cxx- variant
475 b_example = '_bin/cxx-%s/mycpp/examples/%s.%s' % (variant, ex,
476 translator)
477 impl = 'C++'
478 if translator == 'mycpp-souffle':
479 impl = 'C++-Souffle'
480
481 n.build([task_out, cc_log_out],
482 'example-task', [b_example],
483 variables=[('bin', b_example), ('name', ex),
484 ('impl', impl)])
485 n.newline()
486
487 # Compare the log of all examples
488 out = '_test/mycpp-compare-passing.txt'
489 n.build([out], 'logs-equal', to_compare)
490 n.newline()
491
492 # NOTE: Don't really need this
493 ru.phony['mycpp-logs-equal'].append(out)
494
495 # Timing of benchmarks
496 out = '_test/benchmark-table.tsv'
497 n.build([out], 'benchmark-table', benchmark_tasks)
498 n.newline()
499
500 ru.souffle_binary('prebuilt/datalog/call-graph.cc')
501 ru.souffle_binary('prebuilt/datalog/dataflow.cc')
502 ru.souffle_binary('prebuilt/datalog/smoke-test.cc')