OILS / mycpp / NINJA_subgraph.py View on Github | oils.pub

444 lines, 284 significant
1"""
2mycpp/NINJA_subgraph.py
3"""
4from __future__ import print_function
5
6import os
7import sys
8
9from build.ninja_lib import (log, mycpp_library, mycpp_binary,
10 COMPILERS_VARIANTS, OTHER_VARIANTS)
11
12_ = log
13
14
15def DefineTargets(ru):
16
17 # Creates _bin/shwrap/mycpp_main
18 ru.py_binary('mycpp/mycpp_main.py',
19 deps_base_dir='prebuilt/ninja',
20 template='mycpp')
21
22 # mycpp wrapper that depends on _bin/datalog/dataflow, a binary created
23 # from Souffle datalog!
24 ru.n.build(
25 '_bin/shwrap/mycpp_main_souffle',
26 'cp',
27 ['bin/mycpp_main_souffle.sh'],
28 implicit=['_bin/shwrap/mycpp_main', '_bin/datalog/dataflow'],
29 )
30
31 ru.cc_library(
32 '//mycpp/runtime',
33 # Could separate into //mycpp/runtime_{marksweep,bumpleak}
34 srcs=[
35 'mycpp/bump_leak_heap.cc',
36 'mycpp/gc_iolib.cc',
37 'mycpp/gc_mylib.cc',
38 ],
39 deps=['//mycpp/runtime_pure'],
40 )
41
42 ru.cc_library('//mycpp/runtime_pure',
43 srcs=[
44 'mycpp/gc_builtins.cc',
45 'mycpp/gc_mops.cc',
46 'mycpp/gc_str.cc',
47 'mycpp/hash.cc',
48 'mycpp/mark_sweep_heap.cc',
49 ])
50
51 # Special test with -D
52 ru.cc_binary('mycpp/bump_leak_heap_test.cc',
53 deps=['//mycpp/runtime'],
54 matrix=[
55 ('cxx', 'asan+bumpleak'),
56 ('cxx', 'ubsan+bumpleak'),
57 ('clang', 'ubsan+bumpleak'),
58 ('clang', 'coverage+bumpleak'),
59 ],
60 phony_prefix='mycpp-unit')
61
62 for test_main in [
63 'mycpp/mark_sweep_heap_test.cc',
64 'mycpp/gc_heap_test.cc',
65 'mycpp/gc_stress_test.cc',
66 'mycpp/gc_builtins_test.cc',
67 'mycpp/gc_iolib_test.cc',
68 'mycpp/gc_mops_test.cc',
69 'mycpp/gc_mylib_test.cc',
70 'mycpp/gc_dict_test.cc',
71 'mycpp/gc_list_test.cc',
72 'mycpp/gc_str_test.cc',
73 'mycpp/gc_tuple_test.cc',
74 'mycpp/small_str_test.cc',
75 ]:
76 ru.cc_binary(test_main,
77 deps=['//mycpp/runtime'],
78 matrix=(COMPILERS_VARIANTS + OTHER_VARIANTS),
79 phony_prefix='mycpp-unit')
80
81 ru.cc_binary(
82 'mycpp/float_test.cc',
83 deps=['//mycpp/runtime'],
84 # Just test two compilers, in fast mode
85 matrix=[('cxx', 'opt'), ('clang', 'opt')],
86 phony_prefix='mycpp-unit')
87
88 for test_main in [
89 'mycpp/demo/gc_header.cc',
90 'mycpp/demo/hash_table.cc',
91 'mycpp/demo/target_lang.cc',
92 ]:
93 ru.cc_binary(test_main,
94 deps=['//mycpp/runtime'],
95 matrix=COMPILERS_VARIANTS,
96 phony_prefix='mycpp-unit')
97
98 # ASDL schema that examples/parse.py depends on
99 ru.asdl_library('mycpp/examples/expr.asdl')
100
101
102#
103# mycpp/examples build config
104#
105
106# TODO:
107# - Fold this dependency into a proper shwrap wrapper
108# - Make a n.build() wrapper that takes it into account automatically
109RULES_PY = 'build/ninja-rules-py.sh'
110
111# special ones in examples.sh:
112# - parse
113# - lexer_main -- these use Oil code
114# - pgen2_demo -- uses pgen2
115
116
117def ShouldSkipBuild(name):
118 if name.startswith('invalid_'):
119 return True
120
121 if name in [
122 # these use Oils code, and don't type check or compile. Maybe give
123 # up on them? pgen2_demo might be useful later.
124 'lexer_main',
125 'pgen2_demo',
126 ]:
127 return True
128
129 return False
130
131
132def ExamplesToBuild():
133 filenames = os.listdir('mycpp/examples')
134 py = [
135 name[:-3] for name in filenames
136 if name.endswith('.py') and name != '__init__.py'
137 ]
138
139 to_test = [name for name in py if not ShouldSkipBuild(name)]
140
141 result = []
142 for ex in to_test:
143 py_main = 'mycpp/examples/%s.py' % ex
144 result.append((ex, py_main))
145
146 return result
147
148
149def ShouldSkipTest(name):
150 return False
151
152
153def ShouldSkipBenchmark(name):
154 return name.startswith('test_')
155
156
157TRANSLATE_FILES = {
158 # TODO: Use build/dynamic_deps.py here
159 # BUG: modules.py must be listed last. Order matters with inheritance
160 # across modules!
161 'modules': [
162 'mycpp/testpkg/module1.py',
163 'mycpp/testpkg/module2.py',
164 'mycpp/examples/modules.py',
165 ],
166 'parse': [], # added dynamically from mycpp/examples/parse.translate.txt
167}
168
169# Unused. Could use _build/NINJA/mycpp.examples.parse/typecheck.txt
170EXAMPLES_PY = {
171 'parse': [],
172}
173
174EXAMPLES_DEPS = {
175 'parse': [
176 '//mycpp/runtime',
177 '//mycpp/examples/expr.asdl',
178 '//cpp/data_lang',
179 ],
180}
181
182# mycpp-souffle only has three variants for now
183SOUFFLE_MATRIX = [
184 ('cxx', 'opt'), # for benchmarks
185 ('cxx', 'asan'), # need this for running the examples in CI
186 ('cxx', 'asan+gcalways'),
187]
188MYPY_PATH = '$NINJA_REPO_ROOT/mycpp:$NINJA_REPO_ROOT/pyext'
189
190
191def NinjaGraph(ru):
192 n = ru.n
193
194 ru.comment('Generated by %s' % __name__)
195
196 # Running build/ninja_main.py
197 this_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
198
199 n.variable('NINJA_REPO_ROOT', os.path.dirname(this_dir))
200 n.newline()
201
202 # mycpp and pea have the same interface
203 n.rule('translate-mycpp',
204 command='_bin/shwrap/mycpp_main $mypypath $preamble_path $out $in',
205 description='mycpp $mypypath $preamble_path $out $in')
206 n.newline()
207
208 n.rule('translate-mycpp-souffle',
209 command=
210 '_bin/shwrap/mycpp_main_souffle $mypypath $preamble_path $out $in',
211 description='mycpp-souffle $mypypath $preamble_path $out $in')
212 n.newline()
213
214 n.rule('translate-pea',
215 command='_bin/shwrap/pea_main $mypypath $preamble_path $out $in',
216 description='pea $mypypath $preamble_path $out $in')
217 n.newline()
218
219 n.rule(
220 'example-task',
221 # note: $out can be MULTIPLE FILES, shell-quoted
222 command='build/ninja-rules-py.sh example-task $name $impl $bin $out',
223 description='example-task $name $impl $bin $out')
224 n.newline()
225 n.rule(
226 'typecheck',
227 command='build/ninja-rules-py.sh typecheck $main_py $out $skip_imports',
228 description='typecheck $main_py $out $skip_imports')
229 n.newline()
230 n.rule('logs-equal',
231 command='build/ninja-rules-py.sh logs-equal $out $in',
232 description='logs-equal $out $in')
233 n.newline()
234 n.rule('benchmark-table',
235 command='build/ninja-rules-py.sh benchmark-table $out $in',
236 description='benchmark-table $out $in')
237 n.newline()
238
239 # Groups of targets. Not all of these are run by default.
240 ph = {
241 'mycpp-typecheck':
242 [], # optional: for debugging only. translation does it.
243 'mycpp-strip':
244 [], # optional: strip binaries. To see how big they are.
245
246 # Compare logs for tests AND benchmarks.
247 # It's a separate task because we have multiple variants to compare, and
248 # the timing of test/benchmark tasks should NOT include comparison.
249 'mycpp-logs-equal': [],
250
251 # NOTE: _test/benchmark-table.tsv isn't included in any phony target
252
253 # Targets dynamically added:
254 #
255 # mycpp-unit-$compiler-$variant
256 # mycpp-examples-$compiler-$variant
257 'pea-translate': [],
258 'pea-compile': [],
259 # TODO: eventually we will have pea-logs-equal, and pea-benchmark-table
260 }
261 ru.AddPhony(ph)
262
263 DefineTargets(ru)
264
265 #
266 # Build and run examples/
267 #
268
269 MycppExamples(ru, ph)
270
271 #
272 # Prebuilt
273 #
274
275 ru.souffle_binary('prebuilt/datalog/call-graph.cc')
276 ru.souffle_binary('prebuilt/datalog/dataflow.cc')
277 ru.souffle_binary('prebuilt/datalog/smoke-test.cc')
278
279
280def MycppExamples(ru, ph):
281 n = ru.n
282
283 # For simplicity, this is committed to the repo. We could also have
284 # build/dev.sh minimal generate it?
285 with open('_build/NINJA/mycpp.examples.parse/translate.txt') as f:
286 for line in f:
287 path = line.strip()
288 TRANSLATE_FILES['parse'].append(path)
289
290 examples = ExamplesToBuild()
291 #examples = ['cgi', 'containers', 'fib_iter']
292
293 ## Pea Examples
294 for ex, py_main in examples:
295 # Special case: mycpp/examples/pea_* are only translated with pea.
296 # TODO: pea examples don't have the same main()
297 py_rel_path, _ = os.path.splitext(py_main)
298
299 if ex.startswith('pea_'):
300 mycpp_library(ru,
301 py_main,
302 mypy_path=MYPY_PATH,
303 translator='pea',
304 deps=['//mycpp/runtime'])
305 mycpp_binary(ru,
306 '//%s.pea' % py_rel_path,
307 matrix=[
308 ('cxx', 'asan'),
309 ('cxx', 'opt'),
310 ])
311
312 to_compare = []
313 benchmark_tasks = []
314
315 for ex, py_main in examples:
316 if ex.startswith('pea_'): # Only non-pea examples
317 continue
318 py_rel_path, _ = os.path.splitext(py_main)
319
320 ru.comment('- mycpp/examples/%s' % ex)
321
322 # TODO: make a phony target for these, since they're not strictly necessary.
323 # Translation does everything that type checking does. Type checking only
324 # is useful for debugging.
325 t = '_test/tasks/typecheck/%s.log.txt' % ex
326 main_py = 'mycpp/examples/%s.py' % ex
327
328 # expr.asdl needs to import pylib.collections_, which doesn't type check
329 skip_imports = 'T' if (ex == 'parse') else "''"
330
331 ## Type check the example
332
333 n.build(
334 [t],
335 'typecheck',
336 # TODO: Use mycpp/examples/parse.typecheck.txt
337 EXAMPLES_PY.get(ex, []) + [main_py],
338 variables=[('main_py', main_py), ('skip_imports', skip_imports)])
339 n.newline()
340 ru.phony['mycpp-typecheck'].append(t)
341
342 ## Run example as Python
343
344 for mode in ['test', 'benchmark']:
345 prefix = '_test/tasks/%s/%s.py' % (mode, ex)
346 task_out = '%s.task.txt' % prefix
347
348 if mode == 'benchmark':
349 if ShouldSkipBenchmark(ex):
350 #log('Skipping benchmark of %s', ex)
351 continue
352 benchmark_tasks.append(task_out)
353
354 elif mode == 'test':
355 if ShouldSkipTest(ex):
356 #log('Skipping test of %s', ex)
357 continue
358
359 # TODO: This should be a Python stub!
360 log_out = '%s.log' % prefix
361 n.build([task_out, log_out],
362 'example-task',
363 EXAMPLES_PY.get(ex, []) + ['mycpp/examples/%s.py' % ex],
364 variables=[('bin', main_py), ('name', ex),
365 ('impl', 'Python')])
366
367 n.newline()
368
369 ## Translate the example 2 ways, and benchmark and test it
370
371 for translator in ['mycpp', 'mycpp-souffle']:
372
373 matrix = SOUFFLE_MATRIX if translator == 'mycpp-souffle' else COMPILERS_VARIANTS
374 phony_prefix = 'mycpp-examples' if translator == 'mycpp' else None
375 py_inputs = TRANSLATE_FILES.get(ex)
376
377 deps = EXAMPLES_DEPS.get(ex, ['//mycpp/runtime'])
378 mycpp_library(ru,
379 py_main,
380 mypy_path=MYPY_PATH,
381 translator=translator,
382 py_inputs=py_inputs,
383 deps=deps)
384
385 mycpp_binary(ru,
386 '//%s.%s' % (py_rel_path, translator),
387 template='example-unix',
388 phony_prefix=phony_prefix,
389 matrix=matrix)
390
391 # minimal
392 TEST_MATRIX = [
393 ('test', 'asan'), # TODO: asan+gcalways is better!
394 ('benchmark', 'opt'),
395 ]
396
397 # Run the binary in two ways
398 for mode, variant in TEST_MATRIX:
399 task_out = '_test/tasks/%s/%s.%s.%s.task.txt' % (
400 mode, ex, translator, variant)
401
402 if mode == 'benchmark':
403 if ShouldSkipBenchmark(ex):
404 #log('Skipping benchmark of %s', ex)
405 continue
406 benchmark_tasks.append(task_out)
407
408 elif mode == 'test':
409 if ShouldSkipTest(ex):
410 #log('Skipping test of %s', ex)
411 continue
412
413 cc_log_out = '_test/tasks/%s/%s.%s.%s.log' % (
414 mode, ex, translator, variant)
415 py_log_out = '_test/tasks/%s/%s.py.log' % (mode, ex)
416
417 to_compare.append(cc_log_out)
418 to_compare.append(py_log_out)
419
420 # Only test cxx- variant
421 b_example = '_bin/cxx-%s/mycpp/examples/%s.%s' % (variant, ex,
422 translator)
423 impl = 'C++'
424 if translator == 'mycpp-souffle':
425 impl = 'C++-Souffle'
426
427 n.build([task_out, cc_log_out],
428 'example-task', [b_example],
429 variables=[('bin', b_example), ('name', ex),
430 ('impl', impl)])
431 n.newline()
432
433 # Compare the log of all examples
434 out = '_test/mycpp-compare-passing.txt'
435 n.build([out], 'logs-equal', to_compare)
436 n.newline()
437
438 # NOTE: Don't really need this
439 ru.phony['mycpp-logs-equal'].append(out)
440
441 # Timing of benchmarks
442 out = '_test/benchmark-table.tsv'
443 n.build([out], 'benchmark-table', benchmark_tasks)
444 n.newline()