OILS / mycpp / NINJA_subgraph.py View on Github | oils.pub

443 lines, 284 significant
1"""
2mycpp/NINJA_subgraph.py
3"""
4from __future__ import print_function
5
6import os
7import sys
8
9from build.ninja_lib import (log, mycpp_binary, COMPILERS_VARIANTS,
10 OTHER_VARIANTS)
11
12_ = log
13
14
15def DefineTargets(ru):
16
17 # Creates _bin/shwrap/mycpp_main
18 ru.py_binary('mycpp/mycpp_main.py',
19 deps_base_dir='prebuilt/ninja',
20 template='mycpp')
21
22 # mycpp wrapper that depends on _bin/datalog/dataflow, a binary created
23 # from Souffle datalog!
24 ru.n.build(
25 '_bin/shwrap/mycpp_main_souffle',
26 'cp',
27 ['bin/mycpp_main_souffle.sh'],
28 implicit=['_bin/shwrap/mycpp_main', '_bin/datalog/dataflow'],
29 )
30
31 ru.cc_library(
32 '//mycpp/runtime',
33 # Could separate into //mycpp/runtime_{marksweep,bumpleak}
34 srcs=[
35 'mycpp/bump_leak_heap.cc',
36 'mycpp/gc_iolib.cc',
37 'mycpp/gc_mylib.cc',
38 ],
39 deps=['//mycpp/runtime_pure'],
40 )
41
42 ru.cc_library('//mycpp/runtime_pure',
43 srcs=[
44 'mycpp/gc_builtins.cc',
45 'mycpp/gc_mops.cc',
46 'mycpp/gc_str.cc',
47 'mycpp/hash.cc',
48 'mycpp/mark_sweep_heap.cc',
49 ])
50
51 # Special test with -D
52 ru.cc_binary('mycpp/bump_leak_heap_test.cc',
53 deps=['//mycpp/runtime'],
54 matrix=[
55 ('cxx', 'asan+bumpleak'),
56 ('cxx', 'ubsan+bumpleak'),
57 ('clang', 'ubsan+bumpleak'),
58 ('clang', 'coverage+bumpleak'),
59 ],
60 phony_prefix='mycpp-unit')
61
62 for test_main in [
63 'mycpp/mark_sweep_heap_test.cc',
64 'mycpp/gc_heap_test.cc',
65 'mycpp/gc_stress_test.cc',
66 'mycpp/gc_builtins_test.cc',
67 'mycpp/gc_iolib_test.cc',
68 'mycpp/gc_mops_test.cc',
69 'mycpp/gc_mylib_test.cc',
70 'mycpp/gc_dict_test.cc',
71 'mycpp/gc_list_test.cc',
72 'mycpp/gc_str_test.cc',
73 'mycpp/gc_tuple_test.cc',
74 'mycpp/small_str_test.cc',
75 ]:
76 ru.cc_binary(test_main,
77 deps=['//mycpp/runtime'],
78 matrix=(COMPILERS_VARIANTS + OTHER_VARIANTS),
79 phony_prefix='mycpp-unit')
80
81 ru.cc_binary(
82 'mycpp/float_test.cc',
83 deps=['//mycpp/runtime'],
84 # Just test two compilers, in fast mode
85 matrix=[('cxx', 'opt'), ('clang', 'opt')],
86 phony_prefix='mycpp-unit')
87
88 for test_main in [
89 'mycpp/demo/gc_header.cc',
90 'mycpp/demo/hash_table.cc',
91 'mycpp/demo/target_lang.cc',
92 ]:
93 ru.cc_binary(test_main,
94 deps=['//mycpp/runtime'],
95 matrix=COMPILERS_VARIANTS,
96 phony_prefix='mycpp-unit')
97
98 # ASDL schema that examples/parse.py depends on
99 ru.asdl_library('mycpp/examples/expr.asdl')
100
101
102#
103# mycpp/examples build config
104#
105
106# TODO:
107# - Fold this dependency into a proper shwrap wrapper
108# - Make a n.build() wrapper that takes it into account automatically
109RULES_PY = 'build/ninja-rules-py.sh'
110
111# special ones in examples.sh:
112# - parse
113# - lexer_main -- these use Oil code
114# - pgen2_demo -- uses pgen2
115
116
117def ShouldSkipBuild(name):
118 if name.startswith('invalid_'):
119 return True
120
121 if name in [
122 # these use Oils code, and don't type check or compile. Maybe give
123 # up on them? pgen2_demo might be useful later.
124 'lexer_main',
125 'pgen2_demo',
126 ]:
127 return True
128
129 return False
130
131
132def ExamplesToBuild():
133 filenames = os.listdir('mycpp/examples')
134 py = [
135 name[:-3] for name in filenames
136 if name.endswith('.py') and name != '__init__.py'
137 ]
138
139 to_test = [name for name in py if not ShouldSkipBuild(name)]
140
141 result = []
142 for ex in to_test:
143 py_main = 'mycpp/examples/%s.py' % ex
144 result.append((ex, py_main))
145
146 return result
147
148
149def ShouldSkipTest(name):
150 return False
151
152
153def ShouldSkipBenchmark(name):
154 return name.startswith('test_')
155
156
157TRANSLATE_FILES = {
158 # TODO: Use build/dynamic_deps.py here
159 # BUG: modules.py must be listed last. Order matters with inheritance
160 # across modules!
161 'modules': [
162 'mycpp/testpkg/module1.py',
163 'mycpp/testpkg/module2.py',
164 'mycpp/examples/modules.py',
165 ],
166 'parse': [], # added dynamically from mycpp/examples/parse.translate.txt
167}
168
169# Unused. Could use _build/NINJA/mycpp.examples.parse/typecheck.txt
170EXAMPLES_PY = {
171 'parse': [],
172}
173
174EXAMPLES_DEPS = {
175 'parse': [
176 '//mycpp/runtime',
177 '//mycpp/examples/expr.asdl',
178 '//cpp/data_lang',
179 ],
180}
181
182# mycpp-souffle only has three variants for now
183SOUFFLE_MATRIX = [
184 ('cxx', 'opt'), # for benchmarks
185 ('cxx', 'asan'), # need this for running the examples in CI
186 ('cxx', 'asan+gcalways'),
187]
188MYPY_PATH = '$NINJA_REPO_ROOT/mycpp:$NINJA_REPO_ROOT/pyext'
189
190
191def NinjaGraph(ru):
192 n = ru.n
193
194 ru.comment('Generated by %s' % __name__)
195
196 # Running build/ninja_main.py
197 this_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
198
199 n.variable('NINJA_REPO_ROOT', os.path.dirname(this_dir))
200 n.newline()
201
202 # mycpp and pea have the same interface
203 n.rule('translate-mycpp',
204 command='_bin/shwrap/mycpp_main $mypypath $out $in',
205 description='mycpp $mypypath $out $in')
206 n.newline()
207
208 n.rule('translate-mycpp-souffle',
209 command='_bin/shwrap/mycpp_main_souffle $mypypath $out $in',
210 description='mycpp-souffle $mypypath $out $in')
211 n.newline()
212
213 n.rule('translate-pea',
214 command='_bin/shwrap/pea_main $mypypath $out $in',
215 description='pea $mypypath $out $in')
216 n.newline()
217
218 n.rule(
219 'wrap-cc',
220 command=
221 'build/ninja-rules-py.sh wrap-cc $out $main_style $main_namespace $in $preamble',
222 description='wrap-cc $out $main_style $main_namespace $in $preamble')
223 n.newline()
224 n.rule(
225 'example-task',
226 # note: $out can be MULTIPLE FILES, shell-quoted
227 command='build/ninja-rules-py.sh example-task $name $impl $bin $out',
228 description='example-task $name $impl $bin $out')
229 n.newline()
230 n.rule(
231 'typecheck',
232 command='build/ninja-rules-py.sh typecheck $main_py $out $skip_imports',
233 description='typecheck $main_py $out $skip_imports')
234 n.newline()
235 n.rule('logs-equal',
236 command='build/ninja-rules-py.sh logs-equal $out $in',
237 description='logs-equal $out $in')
238 n.newline()
239 n.rule('benchmark-table',
240 command='build/ninja-rules-py.sh benchmark-table $out $in',
241 description='benchmark-table $out $in')
242 n.newline()
243
244 # Groups of targets. Not all of these are run by default.
245 ph = {
246 'mycpp-typecheck':
247 [], # optional: for debugging only. translation does it.
248 'mycpp-strip':
249 [], # optional: strip binaries. To see how big they are.
250
251 # Compare logs for tests AND benchmarks.
252 # It's a separate task because we have multiple variants to compare, and
253 # the timing of test/benchmark tasks should NOT include comparison.
254 'mycpp-logs-equal': [],
255
256 # NOTE: _test/benchmark-table.tsv isn't included in any phony target
257
258 # Targets dynamically added:
259 #
260 # mycpp-unit-$compiler-$variant
261 # mycpp-examples-$compiler-$variant
262 'pea-translate': [],
263 'pea-compile': [],
264 # TODO: eventually we will have pea-logs-equal, and pea-benchmark-table
265 }
266 ru.AddPhony(ph)
267
268 DefineTargets(ru)
269
270 #
271 # Build and run examples/
272 #
273
274 MycppExamples(ru, ph)
275
276 #
277 # Prebuilt
278 #
279
280 ru.souffle_binary('prebuilt/datalog/call-graph.cc')
281 ru.souffle_binary('prebuilt/datalog/dataflow.cc')
282 ru.souffle_binary('prebuilt/datalog/smoke-test.cc')
283
284
285def MycppExamples(ru, ph):
286 n = ru.n
287
288 # For simplicity, this is committed to the repo. We could also have
289 # build/dev.sh minimal generate it?
290 with open('_build/NINJA/mycpp.examples.parse/translate.txt') as f:
291 for line in f:
292 path = line.strip()
293 TRANSLATE_FILES['parse'].append(path)
294
295 examples = ExamplesToBuild()
296 #examples = ['cgi', 'containers', 'fib_iter']
297
298 ## Pea Examples
299 for ex, py_main in examples:
300 # Special case: mycpp/examples/pea_* are only translated with pea.
301 # TODO: pea examples don't have the same main()
302
303 if ex.startswith('pea_'):
304 mycpp_binary(ru,
305 py_main,
306 mypy_path=MYPY_PATH,
307 main_style='main-wrapper',
308 translator='pea',
309 matrix=[
310 ('cxx', 'asan'),
311 ('cxx', 'opt'),
312 ],
313 deps=['//mycpp/runtime'])
314
315 to_compare = []
316 benchmark_tasks = []
317
318 for ex, py_main in examples:
319 if ex.startswith('pea_'): # Only non-pea examples
320 continue
321
322 ru.comment('- mycpp/examples/%s' % ex)
323
324 # TODO: make a phony target for these, since they're not strictly necessary.
325 # Translation does everything that type checking does. Type checking only
326 # is useful for debugging.
327 t = '_test/tasks/typecheck/%s.log.txt' % ex
328 main_py = 'mycpp/examples/%s.py' % ex
329
330 # expr.asdl needs to import pylib.collections_, which doesn't type check
331 skip_imports = 'T' if (ex == 'parse') else "''"
332
333 ## Type check the example
334
335 n.build(
336 [t],
337 'typecheck',
338 # TODO: Use mycpp/examples/parse.typecheck.txt
339 EXAMPLES_PY.get(ex, []) + [main_py],
340 variables=[('main_py', main_py), ('skip_imports', skip_imports)])
341 n.newline()
342 ru.phony['mycpp-typecheck'].append(t)
343
344 ## Run example as Python
345
346 for mode in ['test', 'benchmark']:
347 prefix = '_test/tasks/%s/%s.py' % (mode, ex)
348 task_out = '%s.task.txt' % prefix
349
350 if mode == 'benchmark':
351 if ShouldSkipBenchmark(ex):
352 #log('Skipping benchmark of %s', ex)
353 continue
354 benchmark_tasks.append(task_out)
355
356 elif mode == 'test':
357 if ShouldSkipTest(ex):
358 #log('Skipping test of %s', ex)
359 continue
360
361 # TODO: This should be a Python stub!
362 log_out = '%s.log' % prefix
363 n.build([task_out, log_out],
364 'example-task',
365 EXAMPLES_PY.get(ex, []) + ['mycpp/examples/%s.py' % ex],
366 variables=[('bin', main_py), ('name', ex),
367 ('impl', 'Python')])
368
369 n.newline()
370
371 ## Translate the example 2 ways, and benchmark and test it
372
373 for translator in ['mycpp', 'mycpp-souffle']:
374
375 matrix = SOUFFLE_MATRIX if translator == 'mycpp-souffle' else None
376 phony_prefix = 'mycpp-examples' if translator == 'mycpp' else None
377 py_inputs = TRANSLATE_FILES.get(ex)
378
379 deps = EXAMPLES_DEPS.get(ex, ['//mycpp/runtime'])
380 mycpp_binary(ru,
381 py_main,
382 mypy_path=MYPY_PATH,
383 translator=translator,
384 main_style='example-main-wrapper',
385 py_inputs=py_inputs,
386 phony_prefix=phony_prefix,
387 matrix=matrix,
388 deps=deps)
389
390 # minimal
391 TEST_MATRIX = [
392 ('test', 'asan'), # TODO: asan+gcalways is better!
393 ('benchmark', 'opt'),
394 ]
395
396 # Run the binary in two ways
397 for mode, variant in TEST_MATRIX:
398 task_out = '_test/tasks/%s/%s.%s.%s.task.txt' % (
399 mode, ex, translator, variant)
400
401 if mode == 'benchmark':
402 if ShouldSkipBenchmark(ex):
403 #log('Skipping benchmark of %s', ex)
404 continue
405 benchmark_tasks.append(task_out)
406
407 elif mode == 'test':
408 if ShouldSkipTest(ex):
409 #log('Skipping test of %s', ex)
410 continue
411
412 cc_log_out = '_test/tasks/%s/%s.%s.%s.log' % (
413 mode, ex, translator, variant)
414 py_log_out = '_test/tasks/%s/%s.py.log' % (mode, ex)
415
416 to_compare.append(cc_log_out)
417 to_compare.append(py_log_out)
418
419 # Only test cxx- variant
420 b_example = '_bin/cxx-%s/mycpp/examples/%s.%s' % (variant, ex,
421 translator)
422 impl = 'C++'
423 if translator == 'mycpp-souffle':
424 impl = 'C++-Souffle'
425
426 n.build([task_out, cc_log_out],
427 'example-task', [b_example],
428 variables=[('bin', b_example), ('name', ex),
429 ('impl', impl)])
430 n.newline()
431
432 # Compare the log of all examples
433 out = '_test/mycpp-compare-passing.txt'
434 n.build([out], 'logs-equal', to_compare)
435 n.newline()
436
437 # NOTE: Don't really need this
438 ru.phony['mycpp-logs-equal'].append(out)
439
440 # Timing of benchmarks
441 out = '_test/benchmark-table.tsv'
442 n.build([out], 'benchmark-table', benchmark_tasks)
443 n.newline()