OILS / build / ninja_main.py View on Github | oilshell.org

498 lines, 231 significant
1#!/usr/bin/env python2
2"""
3build/ninja_main.py - invoked by ./NINJA-config.sh
4
5See build/README.md for the code and data layout.
6"""
7from __future__ import print_function
8
9import cStringIO
10from glob import glob
11import os
12import re
13import sys
14
15from build import ninja_lib
16from build.ninja_lib import log
17
18from asdl import NINJA_subgraph as asdl_subgraph
19from bin import NINJA_subgraph as bin_subgraph
20from core import NINJA_subgraph as core_subgraph
21from cpp import NINJA_subgraph as cpp_subgraph
22from data_lang import NINJA_subgraph as data_lang_subgraph
23from display import NINJA_subgraph as display_subgraph
24from frontend import NINJA_subgraph as frontend_subgraph
25from ysh import NINJA_subgraph as ysh_subgraph
26from osh import NINJA_subgraph as osh_subgraph
27from mycpp import NINJA_subgraph as mycpp_subgraph
28from pea import NINJA_subgraph as pea_subgraph
29from prebuilt import NINJA_subgraph as prebuilt_subgraph
30from yaks import NINJA_subgraph as yaks_subgraph
31
32from vendor import ninja_syntax
33
34# The file Ninja runs by default.
35BUILD_NINJA = 'build.ninja'
36
37
38def TarballManifest(cc_h_files):
39 names = []
40
41 # Code we know about
42 names.extend(cc_h_files)
43
44 names.extend([
45 # Text
46 'LICENSE.txt',
47 'README-native.txt',
48 'INSTALL.txt',
49 'configure',
50 'install',
51 'doc/osh.1',
52
53 # Build Scripts
54 'build/common.sh',
55 'build/native.sh',
56
57 # These 2 are used by build/ninja-rules-cpp.sh
58 'build/py2.sh',
59 'build/dev-shell.sh',
60 'build/ninja-rules-cpp.sh',
61
62 # Generated
63 '_build/oils.sh',
64
65 # These are in build/py.sh, not Ninja. Should probably put them in Ninja.
66 #'_gen/frontend/help_meta.h',
67 '_gen/frontend/match.re2c.h',
68 '_gen/frontend/id_kind.asdl_c.h',
69 '_gen/frontend/types.asdl_c.h',
70 ])
71
72 # For configure
73 names.extend(glob('build/detect-*.c'))
74
75 # TODO: crawl headers
76 # We can now use the headers=[] attribute
77 names.extend(glob('mycpp/*.h'))
78 names.extend(glob('cpp/*.h'))
79
80 # ONLY the headers
81 names.extend(glob('prebuilt/*/*.h'))
82
83 names.sort() # Pass them to tar sorted
84
85 # Check for dupes here
86 unique = sorted(set(names))
87 if names != unique:
88 dupes = [n for n in names if names.count(n) > 1]
89 raise AssertionError("Tarball manifest shouldn't have duplicates: %s" %
90 dupes)
91
92 for name in names:
93 print(name)
94
95
96def ShellFunctions(cc_sources, f, argv0):
97 """
98 Generate a shell script that invokes the same function that build.ninja does
99 """
100 print('''\
101#!/bin/sh
102#
103# _build/oils.sh - generated by %s
104#
105# For usage, run:
106#
107# _build/oils --help
108
109. build/ninja-rules-cpp.sh
110
111show_help() {
112 cat <<'EOF'
113Compile the oils-for-unix source into an executable.
114
115Usage:
116 _build/oils.sh COMPILER? VARIANT? TRANSLATOR? SKIP_REBUILD?
117
118 COMPILER: 'cxx' for system compiler, 'clang' or custom one [default cxx]
119 VARIANT: 'dbg' or 'opt' [default opt]
120 TRANSLATOR: 'mycpp' or 'mycpp-souffle' [default mycpp]
121 SKIP_REBUILD: if non-empty, checks if the output exists before building
122
123Environment variable respected:
124
125 OILS_PARALLEL_BUILD=
126 BASE_CXXFLAGS= # See build/ninja-rules-cpp.sh for details
127 CXXFLAGS=
128 OILS_CXX_VERBOSE=
129
130EOF
131}
132
133parse_flags() {
134 while true; do
135 # ${1:-} needed for set -u
136 case "${1:-}" in
137 '')
138 break
139 ;;
140 --help)
141 show_help
142 exit 0
143 ;;
144 *)
145 die "Invalid argument '$1'"
146 ;;
147 esac
148 shift
149 done
150}
151
152
153OILS_PARALLEL_BUILD=${OILS_PARALLEL_BUILD:-1}
154
155_compile_one() {
156 local src=$4
157
158 echo "CXX $src"
159
160 # Delegate to function in build/ninja-rules-cpp.sh
161 if test "${_do_fork:-}" = 1; then
162 compile_one "$@" & # we will wait later
163 else
164 compile_one "$@"
165 fi
166}
167
168main() {
169 ### Compile oils-for-unix into _bin/$compiler-$variant-sh/ (not with ninja)
170
171 parse_flags "$@"
172
173 local compiler=${1:-cxx} # default is system compiler
174 local variant=${2:-opt} # default is optimized build
175 local translator=${3:-mycpp} # default is the translator w/o optimizations
176 local skip_rebuild=${4:-} # if the output exists, skip build'
177''' % (argv0),
178 file=f)
179
180 print('''\
181 local out_dir
182 case $translator in
183 mycpp)
184 out_dir=_bin/$compiler-$variant-sh
185 ;;
186 *)
187 out_dir=_bin/$compiler-$variant-sh/$translator
188 ;;
189 esac
190 local out=$out_dir/oils-for-unix
191
192 if test -n "$skip_rebuild" && test -f "$out"; then
193 echo
194 echo "$0: SKIPPING build because $out exists"
195 echo
196 return
197 fi
198
199 echo
200 echo "$0: Building oils-for-unix: $out"
201 echo "$0: PWD = $PWD"
202 echo
203''',
204 file=f)
205
206 objects = []
207
208 in_out = [
209 ('_gen/bin/oils_for_unix.$translator.cc',
210 '_build/obj/$compiler-$variant-sh/_gen/bin/oils_for_unix.o'),
211 ]
212 for src in sorted(cc_sources):
213 # e.g. _build/obj/cxx-dbg-sh/posix.o
214 prefix, _ = os.path.splitext(src)
215 if prefix.startswith('_gen/bin/oils_for_unix'):
216 continue
217 obj = '_build/obj/$compiler-$variant-sh/%s.o' % prefix
218 in_out.append((src, obj))
219
220
221 bin_dir = '_bin/$compiler-$variant-sh/$translator'
222 obj_dirs = sorted(set(os.path.dirname(obj) for _, obj in in_out))
223
224 all_dirs = [bin_dir] + obj_dirs
225 # Double quote
226 all_dirs = ['"%s"' % d for d in all_dirs]
227
228 print(' mkdir -p \\', file=f)
229 print(' %s' % ' \\\n '.join(all_dirs), file=f)
230 print('', file=f)
231
232 do_fork = ''
233
234 for i, (src, obj) in enumerate(in_out):
235 obj_quoted = '"%s"' % obj
236 objects.append(obj_quoted)
237
238 # Only fork one translation unit that we know to be slow
239 if re.match('.*oils_for_unix\..*\.cc', src):
240 # There should only be one forked translation unit
241 # It can be turned off with OILS_PARALLEL_BUILD= _build/oils
242 assert do_fork == ''
243 do_fork = '_do_fork=$OILS_PARALLEL_BUILD'
244 else:
245 do_fork = ''
246
247 if do_fork:
248 print(' # Potentially fork this translation unit with &', file=f)
249 print(' %s _compile_one "$compiler" "$variant" "" \\' % do_fork,
250 file=f)
251 print(' %s %s' % (src, obj_quoted), file=f)
252 print('', file=f)
253
254 print(' # wait for the translation unit before linking', file=f)
255 print(' echo WAIT', file=f)
256 # time -p shows any excess parallelism on 2 cores
257 # example: oils_for_unix.mycpp.cc takes ~8 seconds longer to compile than all
258 # other translation units combined!
259
260 # Timing isn't POSIX
261 #print(' time -p wait', file=f)
262 print(' wait', file=f)
263 print('', file=f)
264
265 print(' echo "LINK $out"', file=f)
266 # note: can't have spaces in filenames
267 print(' link "$compiler" "$variant" "" "$out" \\', file=f)
268 # put each object on its own line, and indent by 4
269 print(' %s' % (' \\\n '.join(objects)), file=f)
270 print('', file=f)
271
272 # Strip opt binary
273 # TODO: provide a way for the user to get symbols?
274
275 print('''\
276 local out_name=oils-for-unix
277 if test "$variant" = opt; then
278 strip -o "$out.stripped" "$out"
279
280 # Symlink to unstripped binary for benchmarking
281 # out_name=$out_name.stripped
282 fi
283
284 cd $out_dir
285 for symlink in osh ysh; do
286 # like ln -v, which we can't use portably
287 echo " $symlink -> $out_name"
288 ln -s -f $out_name $symlink
289 done
290}
291
292main "$@"
293''',
294 file=f)
295
296
297def Preprocessed(n, cc_sources):
298 # See how much input we're feeding to the compiler. Test C++ template
299 # explosion, e.g. <unordered_map>
300 #
301 # Limit to {dbg,opt} so we don't generate useless rules. Invoked by
302 # metrics/source-code.sh
303
304 pre_matrix = [
305 ('cxx', 'dbg'),
306 ('cxx', 'opt'),
307 ('clang', 'dbg'),
308 ('clang', 'opt'),
309 ]
310 for compiler, variant in pre_matrix:
311 preprocessed = []
312 for src in cc_sources:
313 # e.g. mycpp/gc_heap.cc -> _build/preprocessed/cxx-dbg/mycpp/gc_heap.cc
314 pre = '_build/preprocessed/%s-%s/%s' % (compiler, variant, src)
315 preprocessed.append(pre)
316
317 # Summary file
318 n.build('_build/preprocessed/%s-%s.txt' % (compiler, variant),
319 'line_count', preprocessed)
320 n.newline()
321
322
323def InitSteps(n):
324 """Wrappers for build/ninja-rules-*.sh
325
326 Some of these are defined in mycpp/NINJA_subgraph.py. Could move them here.
327 """
328 #
329 # Compiling and linking
330 #
331
332 # Preprocess one translation unit
333 n.rule(
334 'preprocess',
335 # compile_one detects the _build/preprocessed path
336 command=
337 'build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out',
338 description='PP $compiler $variant $more_cxx_flags $in $out')
339 n.newline()
340
341 n.rule('line_count',
342 command='build/ninja-rules-cpp.sh line_count $out $in',
343 description='line_count $out $in')
344 n.newline()
345
346 # Compile one translation unit
347 n.rule(
348 'compile_one',
349 command=
350 'build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out $out.d',
351 depfile='$out.d',
352 # no prefix since the compiler is the first arg
353 description='$compiler $variant $more_cxx_flags $in $out')
354 n.newline()
355
356 # Link objects together
357 n.rule(
358 'link',
359 command=
360 'build/ninja-rules-cpp.sh link $compiler $variant $more_link_flags $out $in',
361 description='LINK $compiler $variant $more_link_flags $out $in')
362 n.newline()
363
364 # 1 input and 2 outputs
365 n.rule('strip',
366 command='build/ninja-rules-cpp.sh strip_ $in $out',
367 description='STRIP $in $out')
368 n.newline()
369
370 # cc_binary can have symliks
371 n.rule('symlink',
372 command='build/ninja-rules-cpp.sh symlink $dir $target $new',
373 description='SYMLINK $dir $target $new')
374 n.newline()
375
376 #
377 # Code generators
378 #
379
380 n.rule(
381 'write-shwrap',
382 # $in must start with main program
383 command='build/ninja-rules-py.sh write-shwrap $template $out $in',
384 description='make-pystub $out $in')
385 n.newline()
386
387 n.rule(
388 'gen-oils-for-unix',
389 command=
390 'build/ninja-rules-py.sh gen-oils-for-unix $main_name $translator $out_prefix $preamble $extra_mycpp_opts $in',
391 description='gen-oils-for-unix $main_name $translator $out_prefix $preamble $extra_mycpp_opts $in')
392 n.newline()
393
394
395def main(argv):
396 try:
397 action = argv[1]
398 except IndexError:
399 action = 'ninja'
400
401 if action == 'ninja':
402 f = open(BUILD_NINJA, 'w')
403 else:
404 f = cStringIO.StringIO() # thrown away
405
406 n = ninja_syntax.Writer(f)
407 ru = ninja_lib.Rules(n)
408
409 ru.comment('InitSteps()')
410 InitSteps(n)
411
412 #
413 # Create the graph.
414 #
415
416 asdl_subgraph.NinjaGraph(ru)
417 ru.comment('')
418
419 bin_subgraph.NinjaGraph(ru)
420 ru.comment('')
421
422 core_subgraph.NinjaGraph(ru)
423 ru.comment('')
424
425 cpp_subgraph.NinjaGraph(ru)
426 ru.comment('')
427
428 data_lang_subgraph.NinjaGraph(ru)
429 ru.comment('')
430
431 display_subgraph.NinjaGraph(ru)
432 ru.comment('')
433
434 frontend_subgraph.NinjaGraph(ru)
435 ru.comment('')
436
437 mycpp_subgraph.NinjaGraph(ru)
438 ru.comment('')
439
440 ysh_subgraph.NinjaGraph(ru)
441 ru.comment('')
442
443 osh_subgraph.NinjaGraph(ru)
444 ru.comment('')
445
446 pea_subgraph.NinjaGraph(ru)
447 ru.comment('')
448
449 prebuilt_subgraph.NinjaGraph(ru)
450 ru.comment('')
451
452 yaks_subgraph.NinjaGraph(ru)
453 ru.comment('')
454
455 # Materialize all the cc_binary() rules
456 ru.WriteRules()
457
458 # Collect sources for metrics, tarball, shell script
459 cc_sources = ru.SourcesForBinary('_gen/bin/oils_for_unix.mycpp.cc')
460
461 if 0:
462 from pprint import pprint
463 pprint(cc_sources)
464
465 # TODO: could thin these out, not generate for unit tests, etc.
466 Preprocessed(n, cc_sources)
467
468 ru.WritePhony()
469
470 n.default(['_bin/cxx-asan/osh', '_bin/cxx-asan/ysh'])
471
472 if action == 'ninja':
473 log(' (%s) -> %s (%d targets)', argv[0], BUILD_NINJA,
474 n.num_build_targets())
475
476 elif action == 'shell':
477 out = '_build/oils.sh'
478 with open(out, 'w') as f:
479 ShellFunctions(cc_sources, f, argv[0])
480 log(' (%s) -> %s', argv[0], out)
481
482 elif action == 'tarball-manifest':
483 h = ru.HeadersForBinary('_gen/bin/oils_for_unix.mycpp.cc')
484 tar_cc_sources = cc_sources + ['_gen/bin/oils_for_unix.mycpp-souffle.cc']
485 TarballManifest(tar_cc_sources + h)
486
487 else:
488 raise RuntimeError('Invalid action %r' % action)
489
490
491if __name__ == '__main__':
492 try:
493 main(sys.argv)
494 except RuntimeError as e:
495 print('FATAL: %s' % e, file=sys.stderr)
496 sys.exit(1)
497
498# vim: sw=2