OILS / build / ninja_main.py View on Github | oils.pub

463 lines, 242 significant
1#!/usr/bin/env python2
2"""
3build/ninja_main.py - invoked by ./NINJA-config.sh
4
5See build/README.md for the code and data layout.
6"""
7from __future__ import print_function
8
9import cStringIO
10from glob import glob
11import os
12import re
13import sys
14
15from build import ninja_lib
16from build.ninja_lib import log
17
18from asdl import NINJA_subgraph as asdl_subgraph
19from bin import NINJA_subgraph as bin_subgraph
20from core import NINJA_subgraph as core_subgraph
21from cpp import NINJA_subgraph as cpp_subgraph
22from data_lang import NINJA_subgraph as data_lang_subgraph
23from display import NINJA_subgraph as display_subgraph
24from frontend import NINJA_subgraph as frontend_subgraph
25from osh import NINJA_subgraph as osh_subgraph
26from mycpp import NINJA_subgraph as mycpp_subgraph
27from pea import NINJA_subgraph as pea_subgraph
28from prebuilt import NINJA_subgraph as prebuilt_subgraph
29from yaks import NINJA_subgraph as yaks_subgraph
30from ysh import NINJA_subgraph as ysh_subgraph
31
32from vendor import ninja_syntax
33
34# The file Ninja runs by default.
35BUILD_NINJA = 'build.ninja'
36
37
38def TarballManifest(cc_h_files):
39 names = []
40
41 # Code we know about
42 names.extend(cc_h_files)
43
44 names.extend([
45 # Text
46 'LICENSE.txt',
47 'README-native.txt',
48 'INSTALL.txt',
49 'configure',
50 'install',
51 'doc/osh.1',
52
53 # Build Scripts
54 'build/common.sh',
55 'build/native.sh',
56
57 # These 2 are used by build/ninja-rules-cpp.sh
58 'build/py2.sh',
59 'build/dev-shell.sh',
60 'build/ninja-rules-cpp.sh',
61
62 # Generated
63 '_build/oils.sh',
64
65 # These are in build/py.sh, not Ninja. Should probably put them in Ninja.
66 #'_gen/frontend/help_meta.h',
67 '_gen/frontend/match.re2c.h',
68 '_gen/frontend/id_kind.asdl_c.h',
69 '_gen/frontend/types.asdl_c.h',
70 ])
71
72 # For configure
73 names.extend(glob('build/detect-*.c'))
74
75 # TODO: crawl headers
76 # We can now use the headers=[] attribute
77 names.extend(glob('bin/oils_for_unix_preamble.h'))
78
79 # TODO: Parameterize by app
80 names.extend(glob('bin/hello_preamble.h'))
81
82 names.extend(glob('cpp/*.h'))
83 names.extend(glob('mycpp/*.h'))
84 names.extend(glob('asdl/*.h'))
85
86 # ONLY the headers
87 names.extend(glob('prebuilt/*/*.h'))
88
89 names.sort() # Pass them to tar sorted
90
91 # Check for dupes here
92 unique = sorted(set(names))
93 if names != unique:
94 dupes = [n for n in names if names.count(n) > 1]
95 raise AssertionError("Tarball manifest shouldn't have duplicates: %s" %
96 dupes)
97
98 for name in names:
99 print(name)
100
101
102def ShellFunctions(app_name, cc_sources, f, argv0):
103 """
104 Generate a shell fragment that invokes the same function that build.ninja
105 does
106 """
107 # oils_for_unix -> oils-for-unix
108 app_name_hyphens = app_name.replace('_', '-')
109
110 print('''\
111main() {
112 ### Compile oils-for-unix into _bin/$compiler-$variant-sh/ (not with ninja)
113
114 parse_flags "$@"
115
116 # Copy into locals
117 local compiler=$FLAG_cxx
118 local variant=$FLAG_variant
119 local translator=$FLAG_translator
120 local skip_rebuild=$FLAG_skip_rebuild
121
122 local out_name=%s
123
124 local out_dir
125 case $translator in
126 mycpp)
127 out_dir=_bin/$compiler-$variant-sh
128 ;;
129 *)
130 out_dir=_bin/$compiler-$variant-sh/$translator
131 ;;
132 esac
133 local out=$out_dir/$out_name
134
135 echo
136 echo "$0: Building $out_name: $out"
137 echo " PWD = $PWD"
138 echo " cxx = $compiler"
139 echo " variant = $variant"
140 echo " translator = $translator"
141 if test -n "$skip_rebuild"; then
142 echo " skip_rebuild = $skip_rebuild"
143 fi
144
145 if test -n "$skip_rebuild" && test -f "$out"; then
146 echo
147 echo "$0: SKIPPING build because $out exists"
148 echo
149 return
150 fi
151
152 echo
153''' % app_name_hyphens,
154 file=f)
155
156 objects = []
157
158 # Special case for souffle
159 in_out = [
160 ('_gen/bin/%s.$translator.cc' % app_name,
161 '_build/obj/$compiler-$variant-sh/_gen/bin/%s.$translator.o' %
162 app_name),
163 ]
164 for src in sorted(cc_sources):
165 # e.g. _build/obj/cxx-dbg-sh/posix.o
166 prefix, _ = os.path.splitext(src)
167
168 # Skip the one we started with
169 if prefix.startswith('_gen/bin/%s' % app_name):
170 continue
171
172 obj = '_build/obj/$compiler-$variant-sh/%s.o' % prefix
173 in_out.append((src, obj))
174
175 bin_dir = '_bin/$compiler-$variant-sh/$translator'
176 obj_dirs = sorted(set(os.path.dirname(obj) for _, obj in in_out))
177
178 all_dirs = [bin_dir] + obj_dirs
179 # Double quote
180 all_dirs = ['"%s"' % d for d in all_dirs]
181
182 print(' mkdir -p \\', file=f)
183 print(' %s' % ' \\\n '.join(all_dirs), file=f)
184 print('', file=f)
185
186 do_fork = ''
187
188 for i, (src, obj) in enumerate(in_out):
189 obj_quoted = '"%s"' % obj
190 objects.append(obj_quoted)
191
192 # Only fork one translation unit that we know to be slow
193 if re.match('.*oils_for_unix\..*\.cc', src):
194 # There should only be one forked translation unit
195 # It can be turned off with OILS_PARALLEL_BUILD= _build/oils
196 assert do_fork == ''
197 do_fork = '_do_fork=$OILS_PARALLEL_BUILD'
198 else:
199 do_fork = ''
200
201 if do_fork:
202 print(' # Potentially fork this translation unit with &', file=f)
203 print(' %s \\' % do_fork, file=f)
204 indent = ' ' if do_fork else ''
205 print(' %s_compile_one "$compiler" "$variant" "" \\' % indent, file=f)
206 print(' %s %s' % (src, obj_quoted), file=f)
207 if do_fork:
208 print(
209 ' _do_fork= # work around bug in some versions of the dash shell',
210 file=f)
211 print('', file=f)
212
213 print(' # wait for the translation unit before linking', file=f)
214 print(' echo WAIT', file=f)
215 # time -p shows any excess parallelism on 2 cores
216 # example: oils_for_unix.mycpp.cc takes ~8 seconds longer to compile than all
217 # other translation units combined!
218
219 # Timing isn't POSIX
220 #print(' time -p wait', file=f)
221 print(' wait', file=f)
222 print('', file=f)
223
224 print(' echo "LINK $out"', file=f)
225 # note: can't have spaces in filenames
226 print(' link "$compiler" "$variant" "" "$out" \\', file=f)
227 # put each object on its own line, and indent by 4
228 print(' %s' % (' \\\n '.join(objects)), file=f)
229 print('', file=f)
230
231 # Strip opt binary
232 # TODO: provide a way for the user to get symbols?
233
234 print('''\
235 if test "$variant" = opt; then
236 strip -o "$out.stripped" "$out"
237
238 # Symlink to unstripped binary for benchmarking
239 # out_name=$out_name.stripped
240 fi
241
242 cd "$out_dir" # dir may have spaces
243 for symlink in osh ysh; do
244 # like ln -v, which we can't use portably
245 echo " $symlink -> $out_name"
246 ln -s -f $out_name $symlink
247 done
248}
249
250main "$@"
251''',
252 file=f)
253
254
255def Preprocessed(n, cc_sources):
256 # See how much input we're feeding to the compiler. Test C++ template
257 # explosion, e.g. <unordered_map>
258 #
259 # Limit to {dbg,opt} so we don't generate useless rules. Invoked by
260 # metrics/source-code.sh
261
262 pre_matrix = [
263 ('cxx', 'dbg'),
264 ('cxx', 'opt'),
265 ('clang', 'dbg'),
266 ('clang', 'opt'),
267 ]
268 for compiler, variant in pre_matrix:
269 preprocessed = []
270 for src in cc_sources:
271 # e.g. mycpp/gc_heap.cc -> _build/preprocessed/cxx-dbg/mycpp/gc_heap.cc
272 pre = '_build/preprocessed/%s-%s/%s' % (compiler, variant, src)
273 preprocessed.append(pre)
274
275 # Summary file
276 n.build('_build/preprocessed/%s-%s.txt' % (compiler, variant),
277 'line_count', preprocessed)
278 n.newline()
279
280
281def InitSteps(n):
282 """Wrappers for build/ninja-rules-*.sh
283
284 Some of these are defined in mycpp/NINJA_subgraph.py. Could move them here.
285 """
286 #
287 # Compiling and linking
288 #
289
290 # Preprocess one translation unit
291 n.rule(
292 'preprocess',
293 # compile_one detects the _build/preprocessed path
294 command=
295 'build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out',
296 description='PP $compiler $variant $more_cxx_flags $in $out')
297 n.newline()
298
299 n.rule('line_count',
300 command='build/ninja-rules-cpp.sh line_count $out $in',
301 description='line_count $out $in')
302 n.newline()
303
304 # Compile one translation unit
305 n.rule(
306 'compile_one',
307 command=
308 'build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out $out.d',
309 depfile='$out.d',
310 # no prefix since the compiler is the first arg
311 description='$compiler $variant $more_cxx_flags $in $out')
312 n.newline()
313
314 # Link objects together
315 n.rule(
316 'link',
317 command=
318 'build/ninja-rules-cpp.sh link $compiler $variant $more_link_flags $out $in',
319 description='LINK $compiler $variant $more_link_flags $out $in')
320 n.newline()
321
322 # 1 input and 2 outputs
323 n.rule('strip',
324 command='build/ninja-rules-cpp.sh strip_ $in $out',
325 description='STRIP $in $out')
326 n.newline()
327
328 # cc_binary can have symliks
329 n.rule('symlink',
330 command='build/ninja-rules-cpp.sh symlink $dir $target $new',
331 description='SYMLINK $dir $target $new')
332 n.newline()
333
334 #
335 # Code generators
336 #
337
338 n.rule(
339 'write-shwrap',
340 # $in must start with main program
341 command='build/ninja-rules-py.sh write-shwrap $template $out $in',
342 description='make-shwrap $template $out $in')
343 n.newline()
344
345 n.rule('write-main',
346 command=
347 'build/ninja-rules-py.sh write-main $template $out $main_namespace',
348 description='write-main $out $main_namespace')
349 n.newline()
350
351 # Trivial build rule, for bin/mycpp_main -> _bin/shwrap/mycpp_souffle
352 # while adding implicit deps
353 n.rule('cp', command='cp $in $out', description='cp $in $out')
354 n.newline()
355
356
357def main(argv):
358 try:
359 action = argv[1]
360 except IndexError:
361 action = 'ninja'
362
363 try:
364 app_name = argv[2]
365 except IndexError:
366 app_name = 'oils_for_unix'
367
368 if action == 'ninja':
369 f = open(BUILD_NINJA, 'w')
370 else:
371 f = cStringIO.StringIO() # thrown away
372
373 n = ninja_syntax.Writer(f)
374 ru = ninja_lib.Rules(n)
375
376 ru.comment('InitSteps()')
377 InitSteps(n)
378
379 #
380 # Create the graph.
381 #
382
383 asdl_subgraph.NinjaGraph(ru)
384 ru.comment('')
385
386 # translate-mycpp rule
387 mycpp_subgraph.NinjaGraph(ru)
388 ru.comment('')
389
390 bin_subgraph.NinjaGraph(ru)
391 ru.comment('')
392
393 core_subgraph.NinjaGraph(ru)
394 ru.comment('')
395
396 cpp_subgraph.NinjaGraph(ru)
397 ru.comment('')
398
399 data_lang_subgraph.NinjaGraph(ru)
400 ru.comment('')
401
402 display_subgraph.NinjaGraph(ru)
403 ru.comment('')
404
405 frontend_subgraph.NinjaGraph(ru)
406 ru.comment('')
407
408 ysh_subgraph.NinjaGraph(ru)
409 ru.comment('')
410
411 osh_subgraph.NinjaGraph(ru)
412 ru.comment('')
413
414 pea_subgraph.NinjaGraph(ru)
415 ru.comment('')
416
417 prebuilt_subgraph.NinjaGraph(ru)
418 ru.comment('')
419
420 yaks_subgraph.NinjaGraph(ru)
421 ru.comment('')
422
423 # Materialize all the cc_binary() rules
424 ru.WriteRules()
425
426 # Collect sources for metrics, tarball, shell script
427 cc_sources = ru.SourcesForBinary('_gen/bin/%s.mycpp.cc' % app_name)
428
429 if 0:
430 from pprint import pprint
431 pprint(cc_sources)
432
433 # TODO: could thin these out, not generate for unit tests, etc.
434 Preprocessed(n, cc_sources)
435
436 ru.WritePhony()
437
438 n.default(['_bin/cxx-asan/osh', '_bin/cxx-asan/ysh'])
439
440 if action == 'ninja':
441 log(' (%s) -> %s (%d targets)', argv[0], BUILD_NINJA,
442 n.num_build_targets())
443
444 elif action == 'shell':
445 ShellFunctions(app_name, cc_sources, sys.stdout, argv[0])
446
447 elif action == 'tarball-manifest':
448 h = ru.HeadersForBinary('_gen/bin/%s.mycpp.cc' % app_name)
449 tar_cc_sources = cc_sources + [
450 '_gen/bin/%s.mycpp-souffle.cc' % app_name
451 ]
452 TarballManifest(tar_cc_sources + h)
453
454 else:
455 raise RuntimeError('Invalid action %r' % action)
456
457
458if __name__ == '__main__':
459 try:
460 main(sys.argv)
461 except RuntimeError as e:
462 print('FATAL: %s' % e, file=sys.stderr)
463 sys.exit(1)