OILS / mycpp / NINJA_subgraph.py View on Github | oils.pub

492 lines, 315 significant
1"""
2mycpp/NINJA_subgraph.py
3"""
4
5from __future__ import print_function
6
7import os
8import sys
9
10from build.ninja_lib import (log, mycpp_binary, COMPILERS_VARIANTS,
11 OTHER_VARIANTS, SHWRAP)
12
13_ = log
14
15
16def DefineTargets(ru):
17
18 # Creates _bin/shwrap/mycpp_main
19 ru.py_binary('mycpp/mycpp_main.py',
20 deps_base_dir='prebuilt/ninja',
21 template='mycpp')
22
23 # mycpp wrapper that depends on _bin/datalog/dataflow, a binary created
24 # from Souffle datalog!
25 ru.n.build(
26 '_bin/shwrap/mycpp_main_souffle',
27 'cp',
28 ['bin/mycpp_main_souffle.sh'],
29 implicit=['_bin/shwrap/mycpp_main', '_bin/datalog/dataflow'],
30 )
31
32 ru.cc_library(
33 '//mycpp/runtime',
34 # Could separate into //mycpp/runtime_{marksweep,bumpleak}
35 srcs=[
36 'mycpp/bump_leak_heap.cc',
37 'mycpp/gc_builtins.cc',
38 'mycpp/gc_iolib.cc',
39 'mycpp/gc_mops.cc',
40 'mycpp/gc_mylib.cc',
41 'mycpp/gc_str.cc',
42 'mycpp/hash.cc',
43 'mycpp/mark_sweep_heap.cc',
44 ])
45
46 # Special test with -D
47 ru.cc_binary('mycpp/bump_leak_heap_test.cc',
48 deps=['//mycpp/runtime'],
49 matrix=[
50 ('cxx', 'asan+bumpleak'),
51 ('cxx', 'ubsan+bumpleak'),
52 ('clang', 'ubsan+bumpleak'),
53 ('clang', 'coverage+bumpleak'),
54 ],
55 phony_prefix='mycpp-unit')
56
57 for test_main in [
58 'mycpp/mark_sweep_heap_test.cc',
59 'mycpp/gc_heap_test.cc',
60 'mycpp/gc_stress_test.cc',
61 'mycpp/gc_builtins_test.cc',
62 'mycpp/gc_iolib_test.cc',
63 'mycpp/gc_mops_test.cc',
64 'mycpp/gc_mylib_test.cc',
65 'mycpp/gc_dict_test.cc',
66 'mycpp/gc_list_test.cc',
67 'mycpp/gc_str_test.cc',
68 'mycpp/gc_tuple_test.cc',
69 'mycpp/small_str_test.cc',
70 ]:
71 ru.cc_binary(test_main,
72 deps=['//mycpp/runtime'],
73 matrix=(COMPILERS_VARIANTS + OTHER_VARIANTS),
74 phony_prefix='mycpp-unit')
75
76 ru.cc_binary(
77 'mycpp/float_test.cc',
78 deps=['//mycpp/runtime'],
79 # Just test two compilers, in fast mode
80 matrix=[('cxx', 'opt'), ('clang', 'opt')],
81 phony_prefix='mycpp-unit')
82
83 for test_main in [
84 'mycpp/demo/gc_header.cc',
85 'mycpp/demo/hash_table.cc',
86 'mycpp/demo/target_lang.cc',
87 ]:
88 ru.cc_binary(test_main,
89 deps=['//mycpp/runtime'],
90 matrix=COMPILERS_VARIANTS,
91 phony_prefix='mycpp-unit')
92
93 # ASDL schema that examples/parse.py depends on
94 ru.asdl_library('mycpp/examples/expr.asdl')
95
96
97#
98# mycpp/examples build config
99#
100
101# TODO:
102# - Fold this dependency into a proper shwrap wrapper
103# - Make a n.build() wrapper that takes it into account automatically
104RULES_PY = 'build/ninja-rules-py.sh'
105
106# special ones in examples.sh:
107# - parse
108# - lexer_main -- these use Oil code
109# - pgen2_demo -- uses pgen2
110
111
112def ShouldSkipBuild(name):
113 if name.startswith('invalid_'):
114 return True
115
116 if name in [
117 # these use Oil code, and don't type check or compile. Maybe give up on
118 # them? pgen2_demo might be useful later.
119 'lexer_main',
120 'pgen2_demo',
121 ]:
122 return True
123
124 return False
125
126
127def ExamplesToBuild():
128 filenames = os.listdir('mycpp/examples')
129 py = [
130 name[:-3] for name in filenames
131 if name.endswith('.py') and name != '__init__.py'
132 ]
133
134 to_test = [name for name in py if not ShouldSkipBuild(name)]
135
136 result = []
137 for ex in to_test:
138 py_main = 'mycpp/examples/%s.py' % ex
139 result.append((ex, py_main))
140
141 return result
142
143
144def ShouldSkipTest(name):
145 return False
146
147
148def ShouldSkipBenchmark(name):
149 return name.startswith('test_')
150
151
152TRANSLATE_FILES = {
153 # TODO: We could also use app_deps.py here
154 # BUG: modules.py must be listed last. Order matters with inheritance
155 # across modules!
156 'modules': [
157 'mycpp/testpkg/module1.py',
158 'mycpp/testpkg/module2.py',
159 'mycpp/examples/modules.py',
160 ],
161 'parse': [], # added dynamically from mycpp/examples/parse.translate.txt
162}
163
164# Unused. Could use mycpp/examples/parse.typecheck.txt
165EXAMPLES_PY = {
166 'parse': [],
167}
168
169EXAMPLES_DEPS = {
170 'parse': [
171 '//mycpp/runtime',
172 '//mycpp/examples/expr.asdl',
173 '//cpp/data_lang',
174 ],
175}
176
177# mycpp-souffle only has three variants for now
178SOUFFLE_MATRIX = [
179 ('cxx', 'opt'), # for benchmarks
180 #('cxx', 'opt-sh'), # for benchmarks
181 ('cxx', 'asan'), # need this for running the examples in CI
182 ('cxx', 'asan+gcalways'),
183]
184MYPY_PATH = '$NINJA_REPO_ROOT/mycpp:$NINJA_REPO_ROOT/pyext'
185
186# TODO Next:
187# - separate parse.main.cc files, instead of _gen/_tmp
188
189
190def TranslatorSubgraph(
191 ru,
192 translator,
193 py_main,
194 mypy_path,
195 py_inputs=None, # list of source files, including main
196 phony_prefix=None,
197 matrix=None,
198 deps=None):
199 """Create rules for a single example."""
200 py_inputs = py_inputs or [py_main] # if not specified, it's a single file
201 matrix = matrix or COMPILERS_VARIANTS
202 deps = deps or ['//mycpp/runtime']
203
204 n = ru.n
205
206 # e.g. mycpp/examples/parse
207 py_rel_path, _ = os.path.splitext(py_main)
208
209 # e.g. mycpp.examples.parse
210 py_module = py_rel_path.replace('/', '.')
211
212 ex = os.path.basename(py_rel_path)
213
214 # Two steps
215 raw = '_gen/_tmp/%s.%s-raw.cc' % (py_rel_path, translator)
216 main_cc_src = '_gen/%s.%s.cc' % (py_rel_path, translator)
217
218 translator_shwrap = SHWRAP[translator]
219
220 n.build(
221 raw,
222 'translate-%s' % translator,
223 py_inputs, # files to translate
224 # Implicit dependency: if the translator changes, regenerate source
225 # code. But don't pass it on the command line.
226 implicit=[translator_shwrap],
227 # examples/parse uses pyext/fastfunc.pyi
228 variables=[('mypypath', mypy_path)])
229
230 p = '%s_preamble.h' % py_rel_path
231 preamble = p if os.path.exists(p) else "''" # Ninja empty string!
232
233 # Make a translation unit
234 n.build(main_cc_src,
235 'wrap-cc',
236 raw,
237 implicit=[RULES_PY],
238 variables=[
239 ('main_namespace', ex),
240 ('main_func', 'example-main-wrapper'),
241 ('preamble', preamble),
242 ])
243
244 n.newline()
245
246 ru.cc_binary(
247 main_cc_src,
248 deps=deps,
249 matrix=matrix,
250 phony_prefix=phony_prefix,
251 )
252
253
254def NinjaGraph(ru):
255 n = ru.n
256
257 ru.comment('Generated by %s' % __name__)
258
259 # Running build/ninja_main.py
260 this_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
261
262 n.variable('NINJA_REPO_ROOT', os.path.dirname(this_dir))
263 n.newline()
264
265 # mycpp and pea have the same interface
266 n.rule('translate-mycpp',
267 command='_bin/shwrap/mycpp_main $mypypath $out $in',
268 description='mycpp $mypypath $out $in')
269 n.newline()
270
271 n.rule('translate-mycpp-souffle',
272 command='_bin/shwrap/mycpp_main_souffle $mypypath $out $in',
273 description='mycpp-souffle $mypypath $out $in')
274 n.newline()
275
276 n.rule('translate-pea',
277 command='_bin/shwrap/pea_main $mypypath $out $in',
278 description='pea $mypypath $out $in')
279 n.newline()
280
281 n.rule(
282 'wrap-cc',
283 command=
284 'build/ninja-rules-py.sh wrap-cc $out $main_func $main_namespace $in $preamble',
285 description='wrap-cc $out $main_func $main_namespace $in $preamble')
286 n.newline()
287 n.rule(
288 'example-task',
289 # note: $out can be MULTIPLE FILES, shell-quoted
290 command='build/ninja-rules-py.sh example-task $name $impl $bin $out',
291 description='example-task $name $impl $bin $out')
292 n.newline()
293 n.rule(
294 'typecheck',
295 command='build/ninja-rules-py.sh typecheck $main_py $out $skip_imports',
296 description='typecheck $main_py $out $skip_imports')
297 n.newline()
298 n.rule('logs-equal',
299 command='build/ninja-rules-py.sh logs-equal $out $in',
300 description='logs-equal $out $in')
301 n.newline()
302 n.rule('benchmark-table',
303 command='build/ninja-rules-py.sh benchmark-table $out $in',
304 description='benchmark-table $out $in')
305 n.newline()
306
307 # For simplicity, this is committed to the repo. We could also have
308 # build/dev.sh minimal generate it?
309 with open('_build/NINJA/mycpp.examples.parse/translate.txt') as f:
310 for line in f:
311 path = line.strip()
312 TRANSLATE_FILES['parse'].append(path)
313
314 examples = ExamplesToBuild()
315 #examples = ['cgi', 'containers', 'fib_iter']
316
317 # Groups of targets. Not all of these are run by default.
318 ph = {
319 'mycpp-typecheck':
320 [], # optional: for debugging only. translation does it.
321 'mycpp-strip':
322 [], # optional: strip binaries. To see how big they are.
323
324 # Compare logs for tests AND benchmarks.
325 # It's a separate task because we have multiple variants to compare, and
326 # the timing of test/benchmark tasks should NOT include comparison.
327 'mycpp-logs-equal': [],
328
329 # NOTE: _test/benchmark-table.tsv isn't included in any phony target
330
331 # Targets dynamically added:
332 #
333 # mycpp-unit-$compiler-$variant
334 # mycpp-examples-$compiler-$variant
335 'pea-translate': [],
336 'pea-compile': [],
337 # TODO: eventually we will have pea-logs-equal, and pea-benchmark-table
338 }
339 ru.AddPhony(ph)
340
341 DefineTargets(ru)
342
343 #
344 # Build and run examples/
345 #
346
347 ## Pea Examples
348 for ex, py_main in examples:
349 # Special case: mycpp/examples/pea_* are only translated with pea.
350 # TODO: pea examples don't have the same main()
351
352 if ex.startswith('pea_'):
353 mycpp_binary(ru,
354 'mycpp.examples.%s' % ex,
355 translator='pea',
356 matrix=[
357 ('cxx', 'asan'),
358 ('cxx', 'opt'),
359 ],
360 deps=['//mycpp/runtime'])
361
362 to_compare = []
363 benchmark_tasks = []
364
365 for ex, py_main in examples:
366 if ex.startswith('pea_'): # Only non-pea examples
367 continue
368
369 ru.comment('- mycpp/examples/%s' % ex)
370
371 # TODO: make a phony target for these, since they're not strictly necessary.
372 # Translation does everything that type checking does. Type checking only
373 # is useful for debugging.
374 t = '_test/tasks/typecheck/%s.log.txt' % ex
375 main_py = 'mycpp/examples/%s.py' % ex
376
377 # expr.asdl needs to import pylib.collections_, which doesn't type check
378 skip_imports = 'T' if (ex == 'parse') else "''"
379
380 ## Type check the example
381
382 n.build(
383 [t],
384 'typecheck',
385 # TODO: Use mycpp/examples/parse.typecheck.txt
386 EXAMPLES_PY.get(ex, []) + [main_py],
387 variables=[('main_py', main_py), ('skip_imports', skip_imports)])
388 n.newline()
389 ru.phony['mycpp-typecheck'].append(t)
390
391 ## Run example as Python
392
393 for mode in ['test', 'benchmark']:
394 prefix = '_test/tasks/%s/%s.py' % (mode, ex)
395 task_out = '%s.task.txt' % prefix
396
397 if mode == 'benchmark':
398 if ShouldSkipBenchmark(ex):
399 #log('Skipping benchmark of %s', ex)
400 continue
401 benchmark_tasks.append(task_out)
402
403 elif mode == 'test':
404 if ShouldSkipTest(ex):
405 #log('Skipping test of %s', ex)
406 continue
407
408 # TODO: This should be a Python stub!
409 log_out = '%s.log' % prefix
410 n.build([task_out, log_out],
411 'example-task',
412 EXAMPLES_PY.get(ex, []) + ['mycpp/examples/%s.py' % ex],
413 variables=[('bin', main_py), ('name', ex),
414 ('impl', 'Python')])
415
416 n.newline()
417
418 ## Translate the example 2 ways, and benchmark and test it
419
420 for translator in ['mycpp', 'mycpp-souffle']:
421
422 matrix = SOUFFLE_MATRIX if translator == 'mycpp-souffle' else None
423 phony_prefix = 'mycpp-examples' if translator == 'mycpp' else None
424 py_inputs = TRANSLATE_FILES.get(ex)
425
426 TranslatorSubgraph(ru,
427 translator,
428 py_main,
429 MYPY_PATH,
430 py_inputs=py_inputs,
431 phony_prefix=phony_prefix,
432 matrix=matrix,
433 deps=EXAMPLES_DEPS.get(ex))
434
435 # minimal
436 TEST_MATRIX = [
437 ('test', 'asan'), # TODO: asan+gcalways is better!
438 ('benchmark', 'opt'),
439 ]
440
441 # Run the binary in two ways
442 for mode, variant in TEST_MATRIX:
443 task_out = '_test/tasks/%s/%s.%s.%s.task.txt' % (
444 mode, ex, translator, variant)
445
446 if mode == 'benchmark':
447 if ShouldSkipBenchmark(ex):
448 #log('Skipping benchmark of %s', ex)
449 continue
450 benchmark_tasks.append(task_out)
451
452 elif mode == 'test':
453 if ShouldSkipTest(ex):
454 #log('Skipping test of %s', ex)
455 continue
456
457 cc_log_out = '_test/tasks/%s/%s.%s.%s.log' % (
458 mode, ex, translator, variant)
459 py_log_out = '_test/tasks/%s/%s.py.log' % (mode, ex)
460
461 to_compare.append(cc_log_out)
462 to_compare.append(py_log_out)
463
464 # Only test cxx- variant
465 b_example = '_bin/cxx-%s/mycpp/examples/%s.%s' % (variant, ex,
466 translator)
467 impl = 'C++'
468 if translator == 'mycpp-souffle':
469 impl = 'C++-Souffle'
470
471 n.build([task_out, cc_log_out],
472 'example-task', [b_example],
473 variables=[('bin', b_example), ('name', ex),
474 ('impl', impl)])
475 n.newline()
476
477 # Compare the log of all examples
478 out = '_test/mycpp-compare-passing.txt'
479 n.build([out], 'logs-equal', to_compare)
480 n.newline()
481
482 # NOTE: Don't really need this
483 ru.phony['mycpp-logs-equal'].append(out)
484
485 # Timing of benchmarks
486 out = '_test/benchmark-table.tsv'
487 n.build([out], 'benchmark-table', benchmark_tasks)
488 n.newline()
489
490 ru.souffle_binary('prebuilt/datalog/call-graph.cc')
491 ru.souffle_binary('prebuilt/datalog/dataflow.cc')
492 ru.souffle_binary('prebuilt/datalog/smoke-test.cc')