OILS / mycpp / NINJA_subgraph.py View on Github | oils.pub

467 lines, 301 significant
1"""
2mycpp/NINJA_subgraph.py
3"""
4from __future__ import print_function
5
6import os
7import sys
8
9from build.ninja_lib import (log, mycpp_library, mycpp_binary,
10 COMPILERS_VARIANTS, OTHER_VARIANTS)
11
12_ = log
13
14
15def DefineTargets(ru):
16
17 # Creates _bin/shwrap/mycpp_main
18 # This isn't used directly - see SHWRAP in build/ninja_lib.py
19 ru.py_binary(
20 'mycpp/mycpp_main.py',
21 deps_base_dir='prebuilt/ninja',
22 template='mycpp',
23 )
24
25 # Note: could move bin/mycpp_main_{souffle,nosouffle}.sh to mycpp/ dir, so
26 # that they live next to these rules
27
28 # mycpp wrapper that depends on _bin/datalog/dataflow, a binary created
29 # from Souffle datalog!
30 ru.n.build(
31 '_bin/shwrap/mycpp_main_souffle',
32 'cp',
33 ['bin/mycpp_main_souffle.sh'],
34 implicit=['_bin/shwrap/mycpp_main', '_bin/datalog/dataflow'],
35 )
36
37 ru.n.build(
38 '_bin/shwrap/mycpp_main_nosouffle',
39 'cp',
40 ['bin/mycpp_main_nosouffle.sh'],
41 implicit=['_bin/shwrap/mycpp_main'],
42 )
43
44 ru.cc_library(
45 '//mycpp/runtime',
46 # Could separate into //mycpp/runtime_{marksweep,bumpleak}
47 srcs=[
48 'mycpp/bump_leak_heap.cc',
49 'mycpp/gc_iolib.cc',
50 'mycpp/gc_mylib.cc',
51 ],
52 deps=['//mycpp/runtime_pure'],
53 )
54
55 ru.cc_library('//mycpp/runtime_pure',
56 srcs=[
57 'mycpp/gc_builtins.cc',
58 'mycpp/gc_mops.cc',
59 'mycpp/gc_str.cc',
60 'mycpp/hash.cc',
61 'mycpp/mark_sweep_heap.cc',
62 ])
63
64 # Special test with -D
65 ru.cc_binary('mycpp/bump_leak_heap_test.cc',
66 deps=['//mycpp/runtime'],
67 matrix=[
68 ('cxx', 'asan+bumpleak'),
69 ('cxx', 'ubsan+bumpleak'),
70 ('clang', 'ubsan+bumpleak'),
71 ('clang', 'coverage+bumpleak'),
72 ],
73 phony_prefix='mycpp-unit')
74
75 for test_main in [
76 'mycpp/mark_sweep_heap_test.cc',
77 'mycpp/gc_heap_test.cc',
78 'mycpp/gc_stress_test.cc',
79 'mycpp/gc_builtins_test.cc',
80 'mycpp/gc_iolib_test.cc',
81 'mycpp/gc_mops_test.cc',
82 'mycpp/gc_mylib_test.cc',
83 'mycpp/gc_dict_test.cc',
84 'mycpp/gc_list_test.cc',
85 'mycpp/gc_str_test.cc',
86 'mycpp/gc_tuple_test.cc',
87 'mycpp/small_str_test.cc',
88 ]:
89 ru.cc_binary(test_main,
90 deps=['//mycpp/runtime'],
91 matrix=(COMPILERS_VARIANTS + OTHER_VARIANTS),
92 phony_prefix='mycpp-unit')
93
94 ru.cc_binary(
95 'mycpp/float_test.cc',
96 deps=['//mycpp/runtime'],
97 # Just test two compilers, in fast mode
98 matrix=[('cxx', 'opt'), ('clang', 'opt')],
99 phony_prefix='mycpp-unit')
100
101 for test_main in [
102 'mycpp/demo/gc_header.cc',
103 'mycpp/demo/hash_table.cc',
104 'mycpp/demo/target_lang.cc',
105 ]:
106 ru.cc_binary(test_main,
107 deps=['//mycpp/runtime'],
108 matrix=COMPILERS_VARIANTS,
109 phony_prefix='mycpp-unit')
110
111 # ASDL schema that examples/parse.py depends on
112 ru.asdl_library('mycpp/examples/expr.asdl')
113
114
115#
116# mycpp/examples build config
117#
118
119# TODO:
120# - Fold this dependency into a proper shwrap wrapper
121# - Make a n.build() wrapper that takes it into account automatically
122RULES_PY = 'build/ninja-rules-py.sh'
123
124# special ones in examples.sh:
125# - parse
126# - lexer_main -- these use Oil code
127# - pgen2_demo -- uses pgen2
128
129
130def ShouldSkipBuild(name):
131 if name.startswith('invalid_'):
132 return True
133
134 if name in [
135 # these use Oils code, and don't type check or compile. Maybe give
136 # up on them? pgen2_demo might be useful later.
137 'lexer_main',
138 'pgen2_demo',
139 ]:
140 return True
141
142 return False
143
144
145def ExamplesToBuild():
146 filenames = os.listdir('mycpp/examples')
147 py = [
148 name[:-3] for name in filenames
149 if name.endswith('.py') and name != '__init__.py'
150 ]
151
152 to_test = [name for name in py if not ShouldSkipBuild(name)]
153
154 result = []
155 for ex in to_test:
156 py_main = 'mycpp/examples/%s.py' % ex
157 result.append((ex, py_main))
158
159 return result
160
161
162def ShouldSkipTest(name):
163 return False
164
165
166def ShouldSkipBenchmark(name):
167 return name.startswith('test_')
168
169
170TRANSLATE_FILES = {
171 # TODO: Use build/dynamic_deps.py here
172 # BUG: modules.py must be listed last. Order matters with inheritance
173 # across modules!
174 'modules': [
175 'mycpp/testpkg/module1.py',
176 'mycpp/testpkg/module2.py',
177 'mycpp/examples/modules.py',
178 ],
179 'parse': [], # added dynamically from mycpp/examples/parse.translate.txt
180}
181
182# Unused. Could use _build/NINJA/mycpp.examples.parse/typecheck.txt
183EXAMPLES_PY = {
184 'parse': [],
185}
186
187EXAMPLES_DEPS = {
188 'parse': [
189 '//mycpp/runtime',
190 '//mycpp/examples/expr.asdl',
191 '//cpp/data_lang',
192 ],
193}
194
195# mycpp-nosouffle only has three variants for now
196NOSOUFFLE_MATRIX = [
197 ('cxx', 'opt'), # for benchmarks
198 ('cxx', 'asan'), # need this for running the examples in CI
199 ('cxx', 'asan+gcalways'),
200]
201MYPY_PATH = '$NINJA_REPO_ROOT/mycpp:$NINJA_REPO_ROOT/pyext'
202
203
204def NinjaGraph(ru):
205 n = ru.n
206
207 ru.comment('Generated by %s' % __name__)
208
209 # Running build/ninja_main.py
210 this_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
211
212 n.variable('NINJA_REPO_ROOT', os.path.dirname(this_dir))
213 n.newline()
214
215 # mycpp and pea have the same interface
216 n.rule('translate-mycpp',
217 command='_bin/shwrap/mycpp_main $mypypath $preamble_path $out $in',
218 description='mycpp $mypypath $preamble_path $out $in')
219 n.newline()
220
221 n.rule('translate-mycpp-souffle',
222 command=
223 '_bin/shwrap/mycpp_main_souffle $mypypath $preamble_path $out $in',
224 description='mycpp-souffle $mypypath $preamble_path $out $in')
225 n.newline()
226
227 n.rule(
228 'translate-mycpp-nosouffle',
229 command=
230 '_bin/shwrap/mycpp_main_nosouffle $mypypath $preamble_path $out $in',
231 description='mycpp-nosouffle $mypypath $preamble_path $out $in')
232 n.newline()
233
234 n.rule('translate-pea',
235 command='_bin/shwrap/pea_main $mypypath $preamble_path $out $in',
236 description='pea $mypypath $preamble_path $out $in')
237 n.newline()
238
239 n.rule(
240 'example-task',
241 # note: $out can be MULTIPLE FILES, shell-quoted
242 command='build/ninja-rules-py.sh example-task $name $impl $bin $out',
243 description='example-task $name $impl $bin $out')
244 n.newline()
245 n.rule(
246 'typecheck',
247 command='build/ninja-rules-py.sh typecheck $main_py $out $skip_imports',
248 description='typecheck $main_py $out $skip_imports')
249 n.newline()
250 n.rule('logs-equal',
251 command='build/ninja-rules-py.sh logs-equal $out $in',
252 description='logs-equal $out $in')
253 n.newline()
254 n.rule('benchmark-table',
255 command='build/ninja-rules-py.sh benchmark-table $out $in',
256 description='benchmark-table $out $in')
257 n.newline()
258
259 # Groups of targets. Not all of these are run by default.
260 ph = {
261 'mycpp-typecheck':
262 [], # optional: for debugging only. translation does it.
263 'mycpp-strip':
264 [], # optional: strip binaries. To see how big they are.
265
266 # Compare logs for tests AND benchmarks.
267 # It's a separate task because we have multiple variants to compare, and
268 # the timing of test/benchmark tasks should NOT include comparison.
269 'mycpp-logs-equal': [],
270
271 # NOTE: _test/benchmark-table.tsv isn't included in any phony target
272
273 # Targets dynamically added:
274 #
275 # mycpp-unit-$compiler-$variant
276 # mycpp-examples-$compiler-$variant
277 'pea-translate': [],
278 'pea-compile': [],
279 # TODO: eventually we will have pea-logs-equal, and pea-benchmark-table
280 }
281 ru.AddPhony(ph)
282
283 DefineTargets(ru)
284
285 #
286 # Build and run examples/
287 #
288
289 MycppExamples(ru, ph)
290
291 #
292 # Prebuilt
293 #
294
295 ru.souffle_binary('prebuilt/datalog/call-graph.cc')
296 ru.souffle_binary('prebuilt/datalog/dataflow.cc')
297 ru.souffle_binary('prebuilt/datalog/smoke-test.cc')
298
299
300def MycppExamples(ru, ph):
301 n = ru.n
302
303 # For simplicity, this is committed to the repo. We could also have
304 # build/dev.sh minimal generate it?
305 with open('_build/NINJA/mycpp.examples.parse/translate.txt') as f:
306 for line in f:
307 path = line.strip()
308 TRANSLATE_FILES['parse'].append(path)
309
310 examples = ExamplesToBuild()
311 #examples = ['cgi', 'containers', 'fib_iter']
312
313 ## Pea Examples
314 for ex, py_main in examples:
315 # Special case: mycpp/examples/pea_* are only translated with pea.
316 # TODO: pea examples don't have the same main()
317 py_rel_path, _ = os.path.splitext(py_main)
318
319 if ex.startswith('pea_'):
320 mycpp_library(ru,
321 py_main,
322 mypy_path=MYPY_PATH,
323 translator='pea',
324 deps=['//mycpp/runtime'])
325 mycpp_binary(ru,
326 '//%s.pea' % py_rel_path,
327 matrix=[
328 ('cxx', 'asan'),
329 ('cxx', 'opt'),
330 ])
331
332 to_compare = []
333 benchmark_tasks = []
334
335 for ex, py_main in examples:
336 if ex.startswith('pea_'): # Only non-pea examples
337 continue
338 py_rel_path, _ = os.path.splitext(py_main)
339
340 ru.comment('- mycpp/examples/%s' % ex)
341
342 # TODO: make a phony target for these, since they're not strictly necessary.
343 # Translation does everything that type checking does. Type checking only
344 # is useful for debugging.
345 t = '_test/tasks/typecheck/%s.log.txt' % ex
346 main_py = 'mycpp/examples/%s.py' % ex
347
348 # expr.asdl needs to import pylib.collections_, which doesn't type check
349 skip_imports = 'T' if (ex == 'parse') else "''"
350
351 ## Type check the example
352
353 n.build(
354 [t],
355 'typecheck',
356 # TODO: Use mycpp/examples/parse.typecheck.txt
357 EXAMPLES_PY.get(ex, []) + [main_py],
358 variables=[('main_py', main_py), ('skip_imports', skip_imports)])
359 n.newline()
360 ru.phony['mycpp-typecheck'].append(t)
361
362 ## Run example as Python
363
364 for mode in ['test', 'benchmark']:
365 prefix = '_test/tasks/%s/%s.py' % (mode, ex)
366 task_out = '%s.task.txt' % prefix
367
368 if mode == 'benchmark':
369 if ShouldSkipBenchmark(ex):
370 #log('Skipping benchmark of %s', ex)
371 continue
372 benchmark_tasks.append(task_out)
373
374 elif mode == 'test':
375 if ShouldSkipTest(ex):
376 #log('Skipping test of %s', ex)
377 continue
378
379 # TODO: This should be a Python stub!
380 log_out = '%s.log' % prefix
381 n.build([task_out, log_out],
382 'example-task',
383 EXAMPLES_PY.get(ex, []) + ['mycpp/examples/%s.py' % ex],
384 variables=[('bin', main_py), ('name', ex),
385 ('impl', 'Python')])
386
387 n.newline()
388
389 ## Translate the example 2 ways, and benchmark and test it
390
391 for translator in ['mycpp', 'mycpp-souffle', 'mycpp-nosouffle']:
392
393 matrix = (NOSOUFFLE_MATRIX if 'souffle' in translator else
394 COMPILERS_VARIANTS)
395 phony_prefix = 'mycpp-examples' if translator == 'mycpp' else None
396 py_inputs = TRANSLATE_FILES.get(ex)
397
398 deps = EXAMPLES_DEPS.get(ex, ['//mycpp/runtime'])
399 mycpp_library(ru,
400 py_main,
401 mypy_path=MYPY_PATH,
402 translator=translator,
403 py_inputs=py_inputs,
404 deps=deps)
405
406 mycpp_binary(ru,
407 '//%s.%s' % (py_rel_path, translator),
408 template='example-unix',
409 phony_prefix=phony_prefix,
410 matrix=matrix)
411
412 # minimal
413 TEST_MATRIX = [
414 ('test', 'asan'), # TODO: asan+gcalways is better!
415 ('benchmark', 'opt'),
416 ]
417
418 # Run the binary in two ways
419 for mode, variant in TEST_MATRIX:
420 task_out = '_test/tasks/%s/%s.%s.%s.task.txt' % (
421 mode, ex, translator, variant)
422
423 if mode == 'benchmark':
424 if ShouldSkipBenchmark(ex):
425 #log('Skipping benchmark of %s', ex)
426 continue
427 benchmark_tasks.append(task_out)
428
429 elif mode == 'test':
430 if ShouldSkipTest(ex):
431 #log('Skipping test of %s', ex)
432 continue
433
434 cc_log_out = '_test/tasks/%s/%s.%s.%s.log' % (
435 mode, ex, translator, variant)
436 py_log_out = '_test/tasks/%s/%s.py.log' % (mode, ex)
437
438 to_compare.append(cc_log_out)
439 to_compare.append(py_log_out)
440
441 # Only test cxx- variant
442 b_example = '_bin/cxx-%s/mycpp/examples/%s.%s' % (variant, ex,
443 translator)
444 impl = 'C++'
445 if translator == 'mycpp-nosouffle':
446 impl = 'C++-NoSouffle'
447 elif translator == 'mycpp-souffle':
448 impl = 'C++-Souffle'
449
450 n.build([task_out, cc_log_out],
451 'example-task', [b_example],
452 variables=[('bin', b_example), ('name', ex),
453 ('impl', impl)])
454 n.newline()
455
456 # Compare the log of all examples
457 out = '_test/mycpp-compare-passing.txt'
458 n.build([out], 'logs-equal', to_compare)
459 n.newline()
460
461 # NOTE: Don't really need this
462 ru.phony['mycpp-logs-equal'].append(out)
463
464 # Timing of benchmarks
465 out = '_test/benchmark-table.tsv'
466 n.build([out], 'benchmark-table', benchmark_tasks)
467 n.newline()