OILS / mycpp / NINJA_subgraph.py View on Github | oils.pub

444 lines, 285 significant
1"""
2mycpp/NINJA_subgraph.py
3"""
4from __future__ import print_function
5
6import os
7import sys
8
9from build.ninja_lib import (log, mycpp_binary, COMPILERS_VARIANTS,
10 OTHER_VARIANTS)
11
12_ = log
13
14
15def DefineTargets(ru):
16
17 # Creates _bin/shwrap/mycpp_main
18 ru.py_binary('mycpp/mycpp_main.py',
19 deps_base_dir='prebuilt/ninja',
20 template='mycpp')
21
22 # mycpp wrapper that depends on _bin/datalog/dataflow, a binary created
23 # from Souffle datalog!
24 ru.n.build(
25 '_bin/shwrap/mycpp_main_souffle',
26 'cp',
27 ['bin/mycpp_main_souffle.sh'],
28 implicit=['_bin/shwrap/mycpp_main', '_bin/datalog/dataflow'],
29 )
30
31 ru.cc_library(
32 '//mycpp/runtime',
33 # Could separate into //mycpp/runtime_{marksweep,bumpleak}
34 srcs=[
35 'mycpp/bump_leak_heap.cc',
36 'mycpp/gc_iolib.cc',
37 'mycpp/gc_mylib.cc',
38 ],
39 deps=['//mycpp/runtime_pure'],
40 )
41
42 ru.cc_library('//mycpp/runtime_pure',
43 srcs=[
44 'mycpp/gc_builtins.cc',
45 'mycpp/gc_mops.cc',
46 'mycpp/gc_str.cc',
47 'mycpp/hash.cc',
48 'mycpp/mark_sweep_heap.cc',
49 ])
50
51 # Special test with -D
52 ru.cc_binary('mycpp/bump_leak_heap_test.cc',
53 deps=['//mycpp/runtime'],
54 matrix=[
55 ('cxx', 'asan+bumpleak'),
56 ('cxx', 'ubsan+bumpleak'),
57 ('clang', 'ubsan+bumpleak'),
58 ('clang', 'coverage+bumpleak'),
59 ],
60 phony_prefix='mycpp-unit')
61
62 for test_main in [
63 'mycpp/mark_sweep_heap_test.cc',
64 'mycpp/gc_heap_test.cc',
65 'mycpp/gc_stress_test.cc',
66 'mycpp/gc_builtins_test.cc',
67 'mycpp/gc_iolib_test.cc',
68 'mycpp/gc_mops_test.cc',
69 'mycpp/gc_mylib_test.cc',
70 'mycpp/gc_dict_test.cc',
71 'mycpp/gc_list_test.cc',
72 'mycpp/gc_str_test.cc',
73 'mycpp/gc_tuple_test.cc',
74 'mycpp/small_str_test.cc',
75 ]:
76 ru.cc_binary(test_main,
77 deps=['//mycpp/runtime'],
78 matrix=(COMPILERS_VARIANTS + OTHER_VARIANTS),
79 phony_prefix='mycpp-unit')
80
81 ru.cc_binary(
82 'mycpp/float_test.cc',
83 deps=['//mycpp/runtime'],
84 # Just test two compilers, in fast mode
85 matrix=[('cxx', 'opt'), ('clang', 'opt')],
86 phony_prefix='mycpp-unit')
87
88 for test_main in [
89 'mycpp/demo/gc_header.cc',
90 'mycpp/demo/hash_table.cc',
91 'mycpp/demo/target_lang.cc',
92 ]:
93 ru.cc_binary(test_main,
94 deps=['//mycpp/runtime'],
95 matrix=COMPILERS_VARIANTS,
96 phony_prefix='mycpp-unit')
97
98 # ASDL schema that examples/parse.py depends on
99 ru.asdl_library('mycpp/examples/expr.asdl')
100
101
102#
103# mycpp/examples build config
104#
105
106# TODO:
107# - Fold this dependency into a proper shwrap wrapper
108# - Make a n.build() wrapper that takes it into account automatically
109RULES_PY = 'build/ninja-rules-py.sh'
110
111# special ones in examples.sh:
112# - parse
113# - lexer_main -- these use Oil code
114# - pgen2_demo -- uses pgen2
115
116
117def ShouldSkipBuild(name):
118 if name.startswith('invalid_'):
119 return True
120
121 if name in [
122 # these use Oils code, and don't type check or compile. Maybe give
123 # up on them? pgen2_demo might be useful later.
124 'lexer_main',
125 'pgen2_demo',
126 ]:
127 return True
128
129 return False
130
131
132def ExamplesToBuild():
133 filenames = os.listdir('mycpp/examples')
134 py = [
135 name[:-3] for name in filenames
136 if name.endswith('.py') and name != '__init__.py'
137 ]
138
139 to_test = [name for name in py if not ShouldSkipBuild(name)]
140
141 result = []
142 for ex in to_test:
143 py_main = 'mycpp/examples/%s.py' % ex
144 result.append((ex, py_main))
145
146 return result
147
148
149def ShouldSkipTest(name):
150 return False
151
152
153def ShouldSkipBenchmark(name):
154 return name.startswith('test_')
155
156
157TRANSLATE_FILES = {
158 # TODO: Use build/dynamic_deps.py here
159 # BUG: modules.py must be listed last. Order matters with inheritance
160 # across modules!
161 'modules': [
162 'mycpp/testpkg/module1.py',
163 'mycpp/testpkg/module2.py',
164 'mycpp/examples/modules.py',
165 ],
166 'parse': [], # added dynamically from mycpp/examples/parse.translate.txt
167}
168
169# Unused. Could use _build/NINJA/mycpp.examples.parse/typecheck.txt
170EXAMPLES_PY = {
171 'parse': [],
172}
173
174EXAMPLES_DEPS = {
175 'parse': [
176 '//mycpp/runtime',
177 '//mycpp/examples/expr.asdl',
178 '//cpp/data_lang',
179 ],
180}
181
182# mycpp-souffle only has three variants for now
183SOUFFLE_MATRIX = [
184 ('cxx', 'opt'), # for benchmarks
185 ('cxx', 'asan'), # need this for running the examples in CI
186 ('cxx', 'asan+gcalways'),
187]
188MYPY_PATH = '$NINJA_REPO_ROOT/mycpp:$NINJA_REPO_ROOT/pyext'
189
190
191def NinjaGraph(ru):
192 n = ru.n
193
194 ru.comment('Generated by %s' % __name__)
195
196 # Running build/ninja_main.py
197 this_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
198
199 n.variable('NINJA_REPO_ROOT', os.path.dirname(this_dir))
200 n.newline()
201
202 # mycpp and pea have the same interface
203 n.rule('translate-mycpp',
204 command='_bin/shwrap/mycpp_main $mypypath $preamble_path $out $in',
205 description='mycpp $mypypath $preamble_path $out $in')
206 n.newline()
207
208 n.rule('translate-mycpp-souffle',
209 command=
210 '_bin/shwrap/mycpp_main_souffle $mypypath $preamble_path $out $in',
211 description='mycpp-souffle $mypypath $preamble_path $out $in')
212 n.newline()
213
214 n.rule('translate-pea',
215 command='_bin/shwrap/pea_main $mypypath $preamble_path $out $in',
216 description='pea $mypypath $preamble_path $out $in')
217 n.newline()
218
219 n.rule(
220 'wrap-cc',
221 command=
222 'build/ninja-rules-py.sh wrap-cc $out $main_style $main_namespace $in $preamble',
223 description='wrap-cc $out $main_style $main_namespace $in $preamble')
224 n.newline()
225 n.rule(
226 'example-task',
227 # note: $out can be MULTIPLE FILES, shell-quoted
228 command='build/ninja-rules-py.sh example-task $name $impl $bin $out',
229 description='example-task $name $impl $bin $out')
230 n.newline()
231 n.rule(
232 'typecheck',
233 command='build/ninja-rules-py.sh typecheck $main_py $out $skip_imports',
234 description='typecheck $main_py $out $skip_imports')
235 n.newline()
236 n.rule('logs-equal',
237 command='build/ninja-rules-py.sh logs-equal $out $in',
238 description='logs-equal $out $in')
239 n.newline()
240 n.rule('benchmark-table',
241 command='build/ninja-rules-py.sh benchmark-table $out $in',
242 description='benchmark-table $out $in')
243 n.newline()
244
245 # Groups of targets. Not all of these are run by default.
246 ph = {
247 'mycpp-typecheck':
248 [], # optional: for debugging only. translation does it.
249 'mycpp-strip':
250 [], # optional: strip binaries. To see how big they are.
251
252 # Compare logs for tests AND benchmarks.
253 # It's a separate task because we have multiple variants to compare, and
254 # the timing of test/benchmark tasks should NOT include comparison.
255 'mycpp-logs-equal': [],
256
257 # NOTE: _test/benchmark-table.tsv isn't included in any phony target
258
259 # Targets dynamically added:
260 #
261 # mycpp-unit-$compiler-$variant
262 # mycpp-examples-$compiler-$variant
263 'pea-translate': [],
264 'pea-compile': [],
265 # TODO: eventually we will have pea-logs-equal, and pea-benchmark-table
266 }
267 ru.AddPhony(ph)
268
269 DefineTargets(ru)
270
271 #
272 # Build and run examples/
273 #
274
275 MycppExamples(ru, ph)
276
277 #
278 # Prebuilt
279 #
280
281 ru.souffle_binary('prebuilt/datalog/call-graph.cc')
282 ru.souffle_binary('prebuilt/datalog/dataflow.cc')
283 ru.souffle_binary('prebuilt/datalog/smoke-test.cc')
284
285
286def MycppExamples(ru, ph):
287 n = ru.n
288
289 # For simplicity, this is committed to the repo. We could also have
290 # build/dev.sh minimal generate it?
291 with open('_build/NINJA/mycpp.examples.parse/translate.txt') as f:
292 for line in f:
293 path = line.strip()
294 TRANSLATE_FILES['parse'].append(path)
295
296 examples = ExamplesToBuild()
297 #examples = ['cgi', 'containers', 'fib_iter']
298
299 ## Pea Examples
300 for ex, py_main in examples:
301 # Special case: mycpp/examples/pea_* are only translated with pea.
302 # TODO: pea examples don't have the same main()
303
304 if ex.startswith('pea_'):
305 mycpp_binary(ru,
306 py_main,
307 mypy_path=MYPY_PATH,
308 main_style='main-wrapper',
309 translator='pea',
310 matrix=[
311 ('cxx', 'asan'),
312 ('cxx', 'opt'),
313 ],
314 deps=['//mycpp/runtime'])
315
316 to_compare = []
317 benchmark_tasks = []
318
319 for ex, py_main in examples:
320 if ex.startswith('pea_'): # Only non-pea examples
321 continue
322
323 ru.comment('- mycpp/examples/%s' % ex)
324
325 # TODO: make a phony target for these, since they're not strictly necessary.
326 # Translation does everything that type checking does. Type checking only
327 # is useful for debugging.
328 t = '_test/tasks/typecheck/%s.log.txt' % ex
329 main_py = 'mycpp/examples/%s.py' % ex
330
331 # expr.asdl needs to import pylib.collections_, which doesn't type check
332 skip_imports = 'T' if (ex == 'parse') else "''"
333
334 ## Type check the example
335
336 n.build(
337 [t],
338 'typecheck',
339 # TODO: Use mycpp/examples/parse.typecheck.txt
340 EXAMPLES_PY.get(ex, []) + [main_py],
341 variables=[('main_py', main_py), ('skip_imports', skip_imports)])
342 n.newline()
343 ru.phony['mycpp-typecheck'].append(t)
344
345 ## Run example as Python
346
347 for mode in ['test', 'benchmark']:
348 prefix = '_test/tasks/%s/%s.py' % (mode, ex)
349 task_out = '%s.task.txt' % prefix
350
351 if mode == 'benchmark':
352 if ShouldSkipBenchmark(ex):
353 #log('Skipping benchmark of %s', ex)
354 continue
355 benchmark_tasks.append(task_out)
356
357 elif mode == 'test':
358 if ShouldSkipTest(ex):
359 #log('Skipping test of %s', ex)
360 continue
361
362 # TODO: This should be a Python stub!
363 log_out = '%s.log' % prefix
364 n.build([task_out, log_out],
365 'example-task',
366 EXAMPLES_PY.get(ex, []) + ['mycpp/examples/%s.py' % ex],
367 variables=[('bin', main_py), ('name', ex),
368 ('impl', 'Python')])
369
370 n.newline()
371
372 ## Translate the example 2 ways, and benchmark and test it
373
374 for translator in ['mycpp', 'mycpp-souffle']:
375
376 matrix = SOUFFLE_MATRIX if translator == 'mycpp-souffle' else None
377 phony_prefix = 'mycpp-examples' if translator == 'mycpp' else None
378 py_inputs = TRANSLATE_FILES.get(ex)
379
380 deps = EXAMPLES_DEPS.get(ex, ['//mycpp/runtime'])
381 mycpp_binary(ru,
382 py_main,
383 mypy_path=MYPY_PATH,
384 translator=translator,
385 main_style='example-main-wrapper',
386 py_inputs=py_inputs,
387 phony_prefix=phony_prefix,
388 matrix=matrix,
389 deps=deps)
390
391 # minimal
392 TEST_MATRIX = [
393 ('test', 'asan'), # TODO: asan+gcalways is better!
394 ('benchmark', 'opt'),
395 ]
396
397 # Run the binary in two ways
398 for mode, variant in TEST_MATRIX:
399 task_out = '_test/tasks/%s/%s.%s.%s.task.txt' % (
400 mode, ex, translator, variant)
401
402 if mode == 'benchmark':
403 if ShouldSkipBenchmark(ex):
404 #log('Skipping benchmark of %s', ex)
405 continue
406 benchmark_tasks.append(task_out)
407
408 elif mode == 'test':
409 if ShouldSkipTest(ex):
410 #log('Skipping test of %s', ex)
411 continue
412
413 cc_log_out = '_test/tasks/%s/%s.%s.%s.log' % (
414 mode, ex, translator, variant)
415 py_log_out = '_test/tasks/%s/%s.py.log' % (mode, ex)
416
417 to_compare.append(cc_log_out)
418 to_compare.append(py_log_out)
419
420 # Only test cxx- variant
421 b_example = '_bin/cxx-%s/mycpp/examples/%s.%s' % (variant, ex,
422 translator)
423 impl = 'C++'
424 if translator == 'mycpp-souffle':
425 impl = 'C++-Souffle'
426
427 n.build([task_out, cc_log_out],
428 'example-task', [b_example],
429 variables=[('bin', b_example), ('name', ex),
430 ('impl', impl)])
431 n.newline()
432
433 # Compare the log of all examples
434 out = '_test/mycpp-compare-passing.txt'
435 n.build([out], 'logs-equal', to_compare)
436 n.newline()
437
438 # NOTE: Don't really need this
439 ru.phony['mycpp-logs-equal'].append(out)
440
441 # Timing of benchmarks
442 out = '_test/benchmark-table.tsv'
443 n.build([out], 'benchmark-table', benchmark_tasks)
444 n.newline()