OILS / build / ninja_main.py View on Github | oils.pub

446 lines, 236 significant
1#!/usr/bin/env python2
2"""
3build/ninja_main.py - invoked by ./NINJA-config.sh
4
5See build/README.md for the code and data layout.
6"""
7from __future__ import print_function
8
9import cStringIO
10from glob import glob
11import os
12import re
13import sys
14
15from build import ninja_lib
16from build.ninja_lib import log
17
18from asdl import NINJA_subgraph as asdl_subgraph
19from bin import NINJA_subgraph as bin_subgraph
20from core import NINJA_subgraph as core_subgraph
21from cpp import NINJA_subgraph as cpp_subgraph
22from data_lang import NINJA_subgraph as data_lang_subgraph
23from display import NINJA_subgraph as display_subgraph
24from frontend import NINJA_subgraph as frontend_subgraph
25from osh import NINJA_subgraph as osh_subgraph
26from mycpp import NINJA_subgraph as mycpp_subgraph
27from pea import NINJA_subgraph as pea_subgraph
28from prebuilt import NINJA_subgraph as prebuilt_subgraph
29from yaks import NINJA_subgraph as yaks_subgraph
30from ysh import NINJA_subgraph as ysh_subgraph
31
32from vendor import ninja_syntax
33
34# The file Ninja runs by default.
35BUILD_NINJA = 'build.ninja'
36
37
38def TarballManifest(cc_h_files):
39 names = []
40
41 # Code we know about
42 names.extend(cc_h_files)
43
44 names.extend([
45 # Text
46 'LICENSE.txt',
47 'README-native.txt',
48 'INSTALL.txt',
49 'configure',
50 'install',
51 'doc/osh.1',
52
53 # Build Scripts
54 'build/common.sh',
55 'build/native.sh',
56
57 # These 2 are used by build/ninja-rules-cpp.sh
58 'build/py2.sh',
59 'build/dev-shell.sh',
60 'build/ninja-rules-cpp.sh',
61
62 # Generated
63 '_build/oils.sh',
64
65 # These are in build/py.sh, not Ninja. Should probably put them in Ninja.
66 #'_gen/frontend/help_meta.h',
67 '_gen/frontend/match.re2c.h',
68 '_gen/frontend/id_kind.asdl_c.h',
69 '_gen/frontend/types.asdl_c.h',
70 ])
71
72 # For configure
73 names.extend(glob('build/detect-*.c'))
74
75 # TODO: crawl headers
76 # We can now use the headers=[] attribute
77 names.extend(glob('bin/oils_for_unix_preamble.h'))
78 names.extend(glob('cpp/*.h'))
79 names.extend(glob('mycpp/*.h'))
80 names.extend(glob('asdl/*.h'))
81
82 # ONLY the headers
83 names.extend(glob('prebuilt/*/*.h'))
84
85 names.sort() # Pass them to tar sorted
86
87 # Check for dupes here
88 unique = sorted(set(names))
89 if names != unique:
90 dupes = [n for n in names if names.count(n) > 1]
91 raise AssertionError("Tarball manifest shouldn't have duplicates: %s" %
92 dupes)
93
94 for name in names:
95 print(name)
96
97
98def ShellFunctions(cc_sources, f, argv0):
99 """
100 Generate a shell fragment that invokes the same function that build.ninja
101 does
102 """
103 print('''\
104main() {
105 ### Compile oils-for-unix into _bin/$compiler-$variant-sh/ (not with ninja)
106
107 parse_flags "$@"
108
109 # Copy into locals
110 local compiler=$FLAG_cxx
111 local variant=$FLAG_variant
112 local translator=$FLAG_translator
113 local skip_rebuild=$FLAG_skip_rebuild
114
115 local out_dir
116 case $translator in
117 mycpp)
118 out_dir=_bin/$compiler-$variant-sh
119 ;;
120 *)
121 out_dir=_bin/$compiler-$variant-sh/$translator
122 ;;
123 esac
124 local out=$out_dir/oils-for-unix
125
126 echo
127 echo "$0: Building oils-for-unix: $out"
128 echo " PWD = $PWD"
129 echo " cxx = $compiler"
130 echo " variant = $variant"
131 echo " translator = $translator"
132 if test -n "$skip_rebuild"; then
133 echo " skip_rebuild = $skip_rebuild"
134 fi
135
136 if test -n "$skip_rebuild" && test -f "$out"; then
137 echo
138 echo "$0: SKIPPING build because $out exists"
139 echo
140 return
141 fi
142
143 echo
144''',
145 file=f)
146
147 objects = []
148
149 in_out = [
150 ('_gen/bin/oils_for_unix.$translator.cc',
151 '_build/obj/$compiler-$variant-sh/_gen/bin/oils_for_unix.o'),
152 ]
153 for src in sorted(cc_sources):
154 # e.g. _build/obj/cxx-dbg-sh/posix.o
155 prefix, _ = os.path.splitext(src)
156 if prefix.startswith('_gen/bin/oils_for_unix'):
157 continue
158 obj = '_build/obj/$compiler-$variant-sh/%s.o' % prefix
159 in_out.append((src, obj))
160
161 bin_dir = '_bin/$compiler-$variant-sh/$translator'
162 obj_dirs = sorted(set(os.path.dirname(obj) for _, obj in in_out))
163
164 all_dirs = [bin_dir] + obj_dirs
165 # Double quote
166 all_dirs = ['"%s"' % d for d in all_dirs]
167
168 print(' mkdir -p \\', file=f)
169 print(' %s' % ' \\\n '.join(all_dirs), file=f)
170 print('', file=f)
171
172 do_fork = ''
173
174 for i, (src, obj) in enumerate(in_out):
175 obj_quoted = '"%s"' % obj
176 objects.append(obj_quoted)
177
178 # Only fork one translation unit that we know to be slow
179 if re.match('.*oils_for_unix\..*\.cc', src):
180 # There should only be one forked translation unit
181 # It can be turned off with OILS_PARALLEL_BUILD= _build/oils
182 assert do_fork == ''
183 do_fork = '_do_fork=$OILS_PARALLEL_BUILD'
184 else:
185 do_fork = ''
186
187 if do_fork:
188 print(' # Potentially fork this translation unit with &', file=f)
189 print(' %s \\' % do_fork, file=f)
190 indent = ' ' if do_fork else ''
191 print(' %s_compile_one "$compiler" "$variant" "" \\' % indent, file=f)
192 print(' %s %s' % (src, obj_quoted), file=f)
193 if do_fork:
194 print(
195 ' _do_fork= # work around bug in some versions of the dash shell',
196 file=f)
197 print('', file=f)
198
199 print(' # wait for the translation unit before linking', file=f)
200 print(' echo WAIT', file=f)
201 # time -p shows any excess parallelism on 2 cores
202 # example: oils_for_unix.mycpp.cc takes ~8 seconds longer to compile than all
203 # other translation units combined!
204
205 # Timing isn't POSIX
206 #print(' time -p wait', file=f)
207 print(' wait', file=f)
208 print('', file=f)
209
210 print(' echo "LINK $out"', file=f)
211 # note: can't have spaces in filenames
212 print(' link "$compiler" "$variant" "" "$out" \\', file=f)
213 # put each object on its own line, and indent by 4
214 print(' %s' % (' \\\n '.join(objects)), file=f)
215 print('', file=f)
216
217 # Strip opt binary
218 # TODO: provide a way for the user to get symbols?
219
220 print('''\
221 local out_name=oils-for-unix
222 if test "$variant" = opt; then
223 strip -o "$out.stripped" "$out"
224
225 # Symlink to unstripped binary for benchmarking
226 # out_name=$out_name.stripped
227 fi
228
229 cd "$out_dir" # dir may have spaces
230 for symlink in osh ysh; do
231 # like ln -v, which we can't use portably
232 echo " $symlink -> $out_name"
233 ln -s -f $out_name $symlink
234 done
235}
236
237main "$@"
238''',
239 file=f)
240
241
242def Preprocessed(n, cc_sources):
243 # See how much input we're feeding to the compiler. Test C++ template
244 # explosion, e.g. <unordered_map>
245 #
246 # Limit to {dbg,opt} so we don't generate useless rules. Invoked by
247 # metrics/source-code.sh
248
249 pre_matrix = [
250 ('cxx', 'dbg'),
251 ('cxx', 'opt'),
252 ('clang', 'dbg'),
253 ('clang', 'opt'),
254 ]
255 for compiler, variant in pre_matrix:
256 preprocessed = []
257 for src in cc_sources:
258 # e.g. mycpp/gc_heap.cc -> _build/preprocessed/cxx-dbg/mycpp/gc_heap.cc
259 pre = '_build/preprocessed/%s-%s/%s' % (compiler, variant, src)
260 preprocessed.append(pre)
261
262 # Summary file
263 n.build('_build/preprocessed/%s-%s.txt' % (compiler, variant),
264 'line_count', preprocessed)
265 n.newline()
266
267
268def InitSteps(n):
269 """Wrappers for build/ninja-rules-*.sh
270
271 Some of these are defined in mycpp/NINJA_subgraph.py. Could move them here.
272 """
273 #
274 # Compiling and linking
275 #
276
277 # Preprocess one translation unit
278 n.rule(
279 'preprocess',
280 # compile_one detects the _build/preprocessed path
281 command=
282 'build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out',
283 description='PP $compiler $variant $more_cxx_flags $in $out')
284 n.newline()
285
286 n.rule('line_count',
287 command='build/ninja-rules-cpp.sh line_count $out $in',
288 description='line_count $out $in')
289 n.newline()
290
291 # Compile one translation unit
292 n.rule(
293 'compile_one',
294 command=
295 'build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out $out.d',
296 depfile='$out.d',
297 # no prefix since the compiler is the first arg
298 description='$compiler $variant $more_cxx_flags $in $out')
299 n.newline()
300
301 # Link objects together
302 n.rule(
303 'link',
304 command=
305 'build/ninja-rules-cpp.sh link $compiler $variant $more_link_flags $out $in',
306 description='LINK $compiler $variant $more_link_flags $out $in')
307 n.newline()
308
309 # 1 input and 2 outputs
310 n.rule('strip',
311 command='build/ninja-rules-cpp.sh strip_ $in $out',
312 description='STRIP $in $out')
313 n.newline()
314
315 # cc_binary can have symliks
316 n.rule('symlink',
317 command='build/ninja-rules-cpp.sh symlink $dir $target $new',
318 description='SYMLINK $dir $target $new')
319 n.newline()
320
321 #
322 # Code generators
323 #
324
325 n.rule(
326 'write-shwrap',
327 # $in must start with main program
328 command='build/ninja-rules-py.sh write-shwrap $template $out $in',
329 description='make-pystub $out $in')
330 n.newline()
331
332 # Trivial build rule, for bin/mycpp_main -> _bin/shwrap/mycpp_souffle
333 # while adding implicit deps
334 n.rule('cp', command='cp $in $out', description='cp $in $out')
335 n.newline()
336
337 n.rule(
338 'mycpp-gen',
339 command=
340 'build/ninja-rules-py.sh mycpp-gen $py_module $shwrap_path $out_prefix $preamble $in',
341 description=
342 'mycpp-gen $py_module $shwrap_path $out_prefix $preamble $in')
343 n.newline()
344
345
346def main(argv):
347 try:
348 action = argv[1]
349 except IndexError:
350 action = 'ninja'
351
352 if action == 'ninja':
353 f = open(BUILD_NINJA, 'w')
354 else:
355 f = cStringIO.StringIO() # thrown away
356
357 n = ninja_syntax.Writer(f)
358 ru = ninja_lib.Rules(n)
359
360 ru.comment('InitSteps()')
361 InitSteps(n)
362
363 #
364 # Create the graph.
365 #
366
367 asdl_subgraph.NinjaGraph(ru)
368 ru.comment('')
369
370 bin_subgraph.NinjaGraph(ru)
371 ru.comment('')
372
373 core_subgraph.NinjaGraph(ru)
374 ru.comment('')
375
376 cpp_subgraph.NinjaGraph(ru)
377 ru.comment('')
378
379 data_lang_subgraph.NinjaGraph(ru)
380 ru.comment('')
381
382 display_subgraph.NinjaGraph(ru)
383 ru.comment('')
384
385 frontend_subgraph.NinjaGraph(ru)
386 ru.comment('')
387
388 mycpp_subgraph.NinjaGraph(ru)
389 ru.comment('')
390
391 ysh_subgraph.NinjaGraph(ru)
392 ru.comment('')
393
394 osh_subgraph.NinjaGraph(ru)
395 ru.comment('')
396
397 pea_subgraph.NinjaGraph(ru)
398 ru.comment('')
399
400 prebuilt_subgraph.NinjaGraph(ru)
401 ru.comment('')
402
403 yaks_subgraph.NinjaGraph(ru)
404 ru.comment('')
405
406 # Materialize all the cc_binary() rules
407 ru.WriteRules()
408
409 # Collect sources for metrics, tarball, shell script
410 cc_sources = ru.SourcesForBinary('_gen/bin/oils_for_unix.mycpp.cc')
411
412 if 0:
413 from pprint import pprint
414 pprint(cc_sources)
415
416 # TODO: could thin these out, not generate for unit tests, etc.
417 Preprocessed(n, cc_sources)
418
419 ru.WritePhony()
420
421 n.default(['_bin/cxx-asan/osh', '_bin/cxx-asan/ysh'])
422
423 if action == 'ninja':
424 log(' (%s) -> %s (%d targets)', argv[0], BUILD_NINJA,
425 n.num_build_targets())
426
427 elif action == 'shell':
428 ShellFunctions(cc_sources, sys.stdout, argv[0])
429
430 elif action == 'tarball-manifest':
431 h = ru.HeadersForBinary('_gen/bin/oils_for_unix.mycpp.cc')
432 tar_cc_sources = cc_sources + [
433 '_gen/bin/oils_for_unix.mycpp-souffle.cc'
434 ]
435 TarballManifest(tar_cc_sources + h)
436
437 else:
438 raise RuntimeError('Invalid action %r' % action)
439
440
441if __name__ == '__main__':
442 try:
443 main(sys.argv)
444 except RuntimeError as e:
445 print('FATAL: %s' % e, file=sys.stderr)
446 sys.exit(1)