OILS / mycpp / NINJA_subgraph.py View on Github | oilshell.org

446 lines, 287 significant
1"""
2mycpp/NINJA_subgraph.py
3"""
4
5from __future__ import print_function
6
7import os
8import sys
9
10from build.ninja_lib import log, COMPILERS_VARIANTS, OTHER_VARIANTS
11
12_ = log
13
14
15def DefineTargets(ru):
16
17 ru.py_binary('mycpp/mycpp_main.py',
18 deps_base_dir='prebuilt/ninja',
19 template='mycpp')
20
21 ru.cc_library(
22 '//mycpp/runtime',
23 # Could separate into //mycpp/runtime_{marksweep,bumpleak}
24 srcs=[
25 'mycpp/bump_leak_heap.cc',
26 'mycpp/gc_builtins.cc',
27 'mycpp/gc_iolib.cc',
28 'mycpp/gc_mops.cc',
29 'mycpp/gc_mylib.cc',
30 'mycpp/gc_str.cc',
31 'mycpp/hash.cc',
32 'mycpp/mark_sweep_heap.cc',
33 ])
34
35 # Special test with -D
36 ru.cc_binary('mycpp/bump_leak_heap_test.cc',
37 deps=['//mycpp/runtime'],
38 matrix=[
39 ('cxx', 'asan+bumpleak'),
40 ('cxx', 'ubsan+bumpleak'),
41 ('clang', 'ubsan+bumpleak'),
42 ('clang', 'coverage+bumpleak'),
43 ],
44 phony_prefix='mycpp-unit')
45
46 for test_main in [
47 'mycpp/mark_sweep_heap_test.cc',
48 'mycpp/gc_heap_test.cc',
49 'mycpp/gc_stress_test.cc',
50 'mycpp/gc_builtins_test.cc',
51 'mycpp/gc_iolib_test.cc',
52 'mycpp/gc_mops_test.cc',
53 'mycpp/gc_mylib_test.cc',
54 'mycpp/gc_dict_test.cc',
55 'mycpp/gc_list_test.cc',
56 'mycpp/gc_str_test.cc',
57 'mycpp/gc_tuple_test.cc',
58 'mycpp/small_str_test.cc',
59 ]:
60 ru.cc_binary(test_main,
61 deps=['//mycpp/runtime'],
62 matrix=(COMPILERS_VARIANTS + OTHER_VARIANTS),
63 phony_prefix='mycpp-unit')
64
65 ru.cc_binary(
66 'mycpp/float_test.cc',
67 deps=['//mycpp/runtime'],
68 # Just test two compilers, in fast mode
69 matrix=[('cxx', 'opt'), ('clang', 'opt')],
70 phony_prefix='mycpp-unit')
71
72 for test_main in [
73 'mycpp/demo/gc_header.cc',
74 'mycpp/demo/hash_table.cc',
75 'mycpp/demo/target_lang.cc',
76 ]:
77 ru.cc_binary(test_main,
78 deps=['//mycpp/runtime'],
79 matrix=COMPILERS_VARIANTS,
80 phony_prefix='mycpp-unit')
81
82 # ASDL schema that examples/parse.py depends on
83 ru.asdl_library('mycpp/examples/expr.asdl')
84
85
86#
87# mycpp/examples build config
88#
89
90# TODO:
91# - Fold this dependency into a proper shwrap wrapper
92# - Make a n.build() wrapper that takes it into account automatically
93RULES_PY = 'build/ninja-rules-py.sh'
94
95# special ones in examples.sh:
96# - parse
97# - lexer_main -- these use Oil code
98# - pgen2_demo -- uses pgen2
99
100
101def ShouldSkipBuild(name):
102 if name.startswith('invalid_'):
103 return True
104
105 if name in [
106 # these use Oil code, and don't type check or compile. Maybe give up on
107 # them? pgen2_demo might be useful later.
108 'lexer_main',
109 'pgen2_demo',
110 ]:
111 return True
112
113 return False
114
115
116def ExamplesToBuild():
117 filenames = os.listdir('mycpp/examples')
118 py = [
119 name[:-3] for name in filenames
120 if name.endswith('.py') and name != '__init__.py'
121 ]
122
123 to_test = [name for name in py if not ShouldSkipBuild(name)]
124
125 return to_test
126
127
128def ShouldSkipTest(name):
129 return False
130
131
132def ShouldSkipBenchmark(name):
133 return name.startswith('test_')
134
135
136TRANSLATE_FILES = {
137 # TODO: We could also use app_deps.py here
138 # BUG: modules.py must be listed last. Order matters with inheritance
139 # across modules!
140 'modules': [
141 'mycpp/testpkg/module1.py',
142 'mycpp/testpkg/module2.py',
143 'mycpp/examples/modules.py',
144 ],
145 'parse': [], # added dynamically from mycpp/examples/parse.translate.txt
146}
147
148# Unused. Could use mycpp/examples/parse.typecheck.txt
149EXAMPLES_PY = {
150 'parse': [],
151}
152
153
154def TranslatorSubgraph(ru, translator, ex):
155 n = ru.n
156
157 raw = '_gen/mycpp/examples/%s_raw.%s.cc' % (ex, translator)
158
159 # Translate to C++
160 if ex in TRANSLATE_FILES:
161 to_translate = TRANSLATE_FILES[ex]
162 else:
163 to_translate = ['mycpp/examples/%s.py' % ex]
164
165 # Implicit dependency: if the translator changes, regenerate source code.
166 # But don't pass it on the command line.
167 if translator == 'pea':
168 translator_wrapper = '_bin/shwrap/pea_main'
169 base_translator = 'pea'
170 translate_rule = 'translate-pea'
171 else:
172 translate_rule = 'translate-mycpp'
173 base_translator = 'mycpp'
174 translator_wrapper = '_bin/shwrap/mycpp_main'
175
176 translator_vars = [
177 ('mypypath', '$NINJA_REPO_ROOT/mycpp:$NINJA_REPO_ROOT/pyext'),
178 ]
179 if translator == 'mycpp-souffle':
180 translator_vars.append(('extra_mycpp_opts', '--minimize-stack-roots'))
181
182 n.build(
183 raw,
184 translate_rule,
185 to_translate,
186 implicit=[translator_wrapper],
187 # examples/parse uses pyext/fastfunc.pyi
188 variables=translator_vars)
189
190 p = 'mycpp/examples/%s_preamble.h' % ex
191 # Ninja empty string!
192 preamble_path = p if os.path.exists(p) else "''"
193
194 main_cc_src = '_gen/mycpp/examples/%s.%s.cc' % (ex, translator)
195
196 # Make a translation unit
197 n.build(main_cc_src,
198 'wrap-cc',
199 raw,
200 implicit=[RULES_PY],
201 variables=[('name', ex), ('preamble_path', preamble_path),
202 ('translator', base_translator)])
203
204 n.newline()
205
206 if translator == 'pea':
207 ru.phony['pea-translate'].append(main_cc_src)
208
209 if translator == 'mycpp':
210 example_matrix = COMPILERS_VARIANTS
211 elif translator == 'mycpp-souffle':
212 # mycpp-souffle only has three variants for now
213 example_matrix = [
214 ('cxx', 'opt'), # for benchmarks
215 ('cxx', 'opt-sh'), # for benchmarks
216 ('cxx', 'asan'), # need this for running the examples in CI
217 ('cxx', 'asan+gcalways'),
218 ]
219 else:
220 # pea just has one variant for now
221 example_matrix = [('cxx', 'asan+gcalways')]
222
223 if translator == 'mycpp':
224 phony_prefix = 'mycpp-examples'
225 else:
226 phony_prefix = ''
227
228 deps = ['//mycpp/runtime']
229 if ex == 'parse':
230 deps = deps + ['//mycpp/examples/expr.asdl', '//cpp/data_lang']
231
232 ru.cc_binary(
233 main_cc_src,
234 deps=deps,
235 matrix=example_matrix,
236 phony_prefix=phony_prefix,
237 )
238
239 # TODO:
240 # - restore lost 'pea-compile' tag?
241
242
243def NinjaGraph(ru):
244 n = ru.n
245
246 ru.comment('Generated by %s' % __name__)
247
248 # Running build/ninja_main.py
249 this_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
250
251 n.variable('NINJA_REPO_ROOT', os.path.dirname(this_dir))
252 n.newline()
253
254 # mycpp and pea have the same interface
255 n.rule(
256 'translate-mycpp',
257 command='_bin/shwrap/mycpp_main $mypypath $out $in $extra_mycpp_opts',
258 description='mycpp $mypypath $out $in')
259 n.newline()
260
261 n.rule('translate-pea',
262 command='_bin/shwrap/pea_main $mypypath $out $in',
263 description='pea $mypypath $out $in')
264 n.newline()
265
266 n.rule(
267 'wrap-cc',
268 command=
269 'build/ninja-rules-py.sh wrap-cc $out $translator $name $in $preamble_path',
270 description='wrap-cc $out $translator $name $in $preamble_path $out')
271 n.newline()
272 n.rule(
273 'example-task',
274 # note: $out can be MULTIPLE FILES, shell-quoted
275 command='build/ninja-rules-py.sh example-task $name $impl $bin $out',
276 description='example-task $name $impl $bin $out')
277 n.newline()
278 n.rule(
279 'typecheck',
280 command='build/ninja-rules-py.sh typecheck $main_py $out $skip_imports',
281 description='typecheck $main_py $out $skip_imports')
282 n.newline()
283 n.rule('logs-equal',
284 command='build/ninja-rules-py.sh logs-equal $out $in',
285 description='logs-equal $out $in')
286 n.newline()
287 n.rule('benchmark-table',
288 command='build/ninja-rules-py.sh benchmark-table $out $in',
289 description='benchmark-table $out $in')
290 n.newline()
291
292 # For simplicity, this is committed to the repo. We could also have
293 # build/dev.sh minimal generate it?
294 with open('mycpp/examples/parse.translate.txt') as f:
295 for line in f:
296 path = line.strip()
297 TRANSLATE_FILES['parse'].append(path)
298
299 examples = ExamplesToBuild()
300 #examples = ['cgi', 'containers', 'fib_iter']
301
302 # Groups of targets. Not all of these are run by default.
303 ph = {
304 'mycpp-typecheck':
305 [], # optional: for debugging only. translation does it.
306 'mycpp-strip':
307 [], # optional: strip binaries. To see how big they are.
308
309 # Compare logs for tests AND benchmarks.
310 # It's a separate task because we have multiple variants to compare, and
311 # the timing of test/benchmark tasks should NOT include comparison.
312 'mycpp-logs-equal': [],
313
314 # NOTE: _test/benchmark-table.tsv isn't included in any phony target
315
316 # Targets dynamically added:
317 #
318 # mycpp-unit-$compiler-$variant
319 # mycpp-examples-$compiler-$variant
320 'pea-translate': [],
321 'pea-compile': [],
322 # TODO: eventually we will have pea-logs-equal, and pea-benchmark-table
323 }
324 ru.AddPhony(ph)
325
326 DefineTargets(ru)
327
328 #
329 # Build and run examples/
330 #
331
332 to_compare = []
333 benchmark_tasks = []
334
335 for ex in examples:
336 ru.comment('- mycpp/examples/%s' % ex)
337
338 # TODO: make a phony target for these, since they're not strictly necessary.
339 # Translation does everything that type checking does. Type checking only
340 # is useful for debugging.
341 t = '_test/tasks/typecheck/%s.log.txt' % ex
342 main_py = 'mycpp/examples/%s.py' % ex
343
344 # expr.asdl needs to import pylib.collections_, which doesn't type check
345 skip_imports = 'T' if (ex == 'parse') else "''"
346
347 n.build(
348 [t],
349 'typecheck',
350 # TODO: Use mycpp/examples/parse.typecheck.txt
351 EXAMPLES_PY.get(ex, []) + [main_py],
352 variables=[('main_py', main_py), ('skip_imports', skip_imports)])
353 n.newline()
354 ru.phony['mycpp-typecheck'].append(t)
355
356 # Run Python.
357 for mode in ['test', 'benchmark']:
358 prefix = '_test/tasks/%s/%s.py' % (mode, ex)
359 task_out = '%s.task.txt' % prefix
360
361 if mode == 'benchmark':
362 if ShouldSkipBenchmark(ex):
363 #log('Skipping benchmark of %s', ex)
364 continue
365 benchmark_tasks.append(task_out)
366
367 elif mode == 'test':
368 if ShouldSkipTest(ex):
369 #log('Skipping test of %s', ex)
370 continue
371
372 # TODO: This should be a Python stub!
373 log_out = '%s.log' % prefix
374 n.build([task_out, log_out],
375 'example-task',
376 EXAMPLES_PY.get(ex, []) + ['mycpp/examples/%s.py' % ex],
377 variables=[('bin', main_py), ('name', ex),
378 ('impl', 'Python')])
379
380 n.newline()
381
382 for translator in ['mycpp', 'mycpp-souffle', 'pea']:
383 TranslatorSubgraph(ru, translator, ex)
384
385 # Don't run it for now; just compile
386 if translator == 'pea':
387 continue
388
389 # minimal
390 MATRIX = [
391 ('test', 'asan'), # TODO: asan+gcalways is better!
392 ('benchmark', 'opt'),
393 ]
394
395 # Run the binary in two ways
396 for mode, variant in MATRIX:
397 task_out = '_test/tasks/%s/%s.%s.%s.task.txt' % (
398 mode, ex, translator, variant)
399
400 if mode == 'benchmark':
401 if ShouldSkipBenchmark(ex):
402 #log('Skipping benchmark of %s', ex)
403 continue
404 benchmark_tasks.append(task_out)
405
406 elif mode == 'test':
407 if ShouldSkipTest(ex):
408 #log('Skipping test of %s', ex)
409 continue
410
411 cc_log_out = '_test/tasks/%s/%s.%s.%s.log' % (
412 mode, ex, translator, variant)
413 py_log_out = '_test/tasks/%s/%s.py.log' % (mode, ex)
414
415 to_compare.append(cc_log_out)
416 to_compare.append(py_log_out)
417
418 # Only test cxx- variant
419 b_example = '_bin/cxx-%s/mycpp/examples/%s.%s' % (variant, ex,
420 translator)
421 impl = 'C++'
422 if translator == 'mycpp-souffle':
423 impl = 'C++-Souffle'
424
425 n.build([task_out, cc_log_out],
426 'example-task', [b_example],
427 variables=[('bin', b_example), ('name', ex),
428 ('impl', impl)])
429 n.newline()
430
431 # Compare the log of all examples
432 out = '_test/mycpp-compare-passing.txt'
433 n.build([out], 'logs-equal', to_compare)
434 n.newline()
435
436 # NOTE: Don't really need this
437 ru.phony['mycpp-logs-equal'].append(out)
438
439 # Timing of benchmarks
440 out = '_test/benchmark-table.tsv'
441 n.build([out], 'benchmark-table', benchmark_tasks)
442 n.newline()
443
444 ru.souffle_binary('prebuilt/datalog/call-graph.cc')
445 ru.souffle_binary('prebuilt/datalog/dataflow.cc')
446 ru.souffle_binary('prebuilt/datalog/smoke-test.cc')