OILS / build / ninja_main.py View on Github | oilshell.org

459 lines, 231 significant
1#!/usr/bin/env python2
2"""
3build/ninja_main.py - invoked by ./NINJA-config.sh
4
5See build/README.md for the code and data layout.
6
7"""
8from __future__ import print_function
9
10import cStringIO
11from glob import glob
12import os
13import re
14import sys
15
16from build import ninja_lib
17from build.ninja_lib import log
18
19from asdl import NINJA_subgraph as asdl_subgraph
20from bin import NINJA_subgraph as bin_subgraph
21from core import NINJA_subgraph as core_subgraph
22from cpp import NINJA_subgraph as cpp_subgraph
23from data_lang import NINJA_subgraph as data_lang_subgraph
24from display import NINJA_subgraph as display_subgraph
25from frontend import NINJA_subgraph as frontend_subgraph
26from ysh import NINJA_subgraph as ysh_subgraph
27from osh import NINJA_subgraph as osh_subgraph
28from mycpp import NINJA_subgraph as mycpp_subgraph
29from pea import NINJA_subgraph as pea_subgraph
30from prebuilt import NINJA_subgraph as prebuilt_subgraph
31from yaks import NINJA_subgraph as yaks_subgraph
32
33from vendor import ninja_syntax
34
35# The file Ninja runs by default.
36BUILD_NINJA = 'build.ninja'
37
38
39def TarballManifest(cc_h_files):
40 names = []
41
42 # Code we know about
43 names.extend(cc_h_files)
44
45 names.extend([
46 # Text
47 'LICENSE.txt',
48 'README-native.txt',
49 'INSTALL.txt',
50 'configure',
51 'install',
52 'doc/osh.1',
53
54 # Build Scripts
55 'build/common.sh',
56 'build/native.sh',
57
58 # These 2 are used by build/ninja-rules-cpp.sh
59 'build/py2.sh',
60 'build/dev-shell.sh',
61 'build/ninja-rules-cpp.sh',
62
63 # Generated
64 '_build/oils.sh',
65
66 # These are in build/py.sh, not Ninja. Should probably put them in Ninja.
67 #'_gen/frontend/help_meta.h',
68 '_gen/frontend/match.re2c.h',
69 '_gen/frontend/id_kind.asdl_c.h',
70 '_gen/frontend/types.asdl_c.h',
71 ])
72
73 # For configure
74 names.extend(glob('build/detect-*.c'))
75
76 # TODO: crawl headers
77 # We can now use the headers=[] attribute
78 names.extend(glob('mycpp/*.h'))
79 names.extend(glob('cpp/*.h'))
80
81 # ONLY the headers
82 names.extend(glob('prebuilt/*/*.h'))
83
84 names.sort() # Pass them to tar sorted
85
86 # Check for dupes here
87 unique = sorted(set(names))
88 if names != unique:
89 dupes = [n for n in names if names.count(n) > 1]
90 raise AssertionError("Tarball manifest shouldn't have duplicates: %s" %
91 dupes)
92
93 for name in names:
94 print(name)
95
96
97def ShellFunctions(cc_sources, f, argv0):
98 """
99 Generate a shell script that invokes the same function that build.ninja does
100 """
101 print('''\
102#!/bin/sh
103#
104# _build/oils.sh - generated by %s
105#
106# Usage:
107# _build/oils.sh COMPILER? VARIANT? SKIP_REBUILD?
108#
109# COMPILER: 'cxx' for system compiler, 'clang' or custom one [default cxx]
110# VARIANT: 'dbg' or 'opt' [default opt]
111# TRANSLATOR: 'mycpp' or 'mycpp-souffle' [default mycpp]
112# SKIP_REBUILD: if non-empty, checks if the output exists before building
113
114. build/ninja-rules-cpp.sh
115
116OILS_PARALLEL_BUILD=${OILS_PARALLEL_BUILD:-1}
117
118_compile_one() {
119 local src=$4
120
121 echo "CXX $src"
122
123 # Delegate to function in build/ninja-rules-cpp.sh
124 if test "${_do_fork:-}" = 1; then
125 compile_one "$@" & # we will wait later
126 else
127 compile_one "$@"
128 fi
129}
130
131main() {
132 ### Compile oils-for-unix into _bin/$compiler-$variant-sh/ (not with ninja)
133
134 local compiler=${1:-cxx} # default is system compiler
135 local variant=${2:-opt} # default is optimized build
136 local translator=${3:-mycpp} # default is the translator w/o optimizations
137 local skip_rebuild=${4:-} # if the output exists, skip build'
138''' % (argv0),
139 file=f)
140
141 print('''\
142 local out_dir
143 case $translator in
144 mycpp)
145 out_dir=_bin/$compiler-$variant-sh
146 ;;
147 *)
148 out_dir=_bin/$compiler-$variant-sh/$translator
149 ;;
150 esac
151 local out=$out_dir/oils-for-unix
152
153 if test -n "$skip_rebuild" && test -f "$out"; then
154 echo
155 echo "$0: SKIPPING build because $out exists"
156 echo
157 return
158 fi
159
160 echo
161 echo "$0: Building oils-for-unix: $out"
162 echo "$0: PWD = $PWD"
163 echo
164''',
165 file=f)
166
167 objects = []
168
169 in_out = [
170 ('_gen/bin/oils_for_unix.$translator.cc',
171 '_build/obj/$compiler-$variant-sh/_gen/bin/oils_for_unix.o'),
172 ]
173 for src in sorted(cc_sources):
174 # e.g. _build/obj/cxx-dbg-sh/posix.o
175 prefix, _ = os.path.splitext(src)
176 if prefix.startswith('_gen/bin/oils_for_unix'):
177 continue
178 obj = '_build/obj/$compiler-$variant-sh/%s.o' % prefix
179 in_out.append((src, obj))
180
181
182 bin_dir = '_bin/$compiler-$variant-sh/$translator'
183 obj_dirs = sorted(set(os.path.dirname(obj) for _, obj in in_out))
184
185 all_dirs = [bin_dir] + obj_dirs
186 # Double quote
187 all_dirs = ['"%s"' % d for d in all_dirs]
188
189 print(' mkdir -p \\', file=f)
190 print(' %s' % ' \\\n '.join(all_dirs), file=f)
191 print('', file=f)
192
193 do_fork = ''
194
195 for i, (src, obj) in enumerate(in_out):
196 obj_quoted = '"%s"' % obj
197 objects.append(obj_quoted)
198
199 # Only fork one translation unit that we know to be slow
200 if re.match('.*oils_for_unix\..*\.cc', src):
201 # There should only be one forked translation unit
202 # It can be turned off with OILS_PARALLEL_BUILD= _build/oils
203 assert do_fork == ''
204 do_fork = '_do_fork=$OILS_PARALLEL_BUILD'
205 else:
206 do_fork = ''
207
208 if do_fork:
209 print(' # Potentially fork this translation unit with &', file=f)
210 print(' %s _compile_one "$compiler" "$variant" "" \\' % do_fork,
211 file=f)
212 print(' %s %s' % (src, obj_quoted), file=f)
213 print('', file=f)
214
215 print(' # wait for the translation unit before linking', file=f)
216 print(' echo WAIT', file=f)
217 # time -p shows any excess parallelism on 2 cores
218 # example: oils_for_unix.mycpp.cc takes ~8 seconds longer to compile than all
219 # other translation units combined!
220
221 # Timing isn't POSIX
222 #print(' time -p wait', file=f)
223 print(' wait', file=f)
224 print('', file=f)
225
226 print(' echo "LINK $out"', file=f)
227 # note: can't have spaces in filenames
228 print(' link "$compiler" "$variant" "" "$out" \\', file=f)
229 # put each object on its own line, and indent by 4
230 print(' %s' % (' \\\n '.join(objects)), file=f)
231 print('', file=f)
232
233 # Strip opt binary
234 # TODO: provide a way for the user to get symbols?
235
236 print('''\
237 local out_name=oils-for-unix
238 if test "$variant" = opt; then
239 strip -o "$out.stripped" "$out"
240
241 # Symlink to unstripped binary for benchmarking
242 # out_name=$out_name.stripped
243 fi
244
245 cd $out_dir
246 for symlink in osh ysh; do
247 # like ln -v, which we can't use portably
248 echo " $symlink -> $out_name"
249 ln -s -f $out_name $symlink
250 done
251}
252
253main "$@"
254''',
255 file=f)
256
257
258def Preprocessed(n, cc_sources):
259 # See how much input we're feeding to the compiler. Test C++ template
260 # explosion, e.g. <unordered_map>
261 #
262 # Limit to {dbg,opt} so we don't generate useless rules. Invoked by
263 # metrics/source-code.sh
264
265 pre_matrix = [
266 ('cxx', 'dbg'),
267 ('cxx', 'opt'),
268 ('clang', 'dbg'),
269 ('clang', 'opt'),
270 ]
271 for compiler, variant in pre_matrix:
272 preprocessed = []
273 for src in cc_sources:
274 # e.g. mycpp/gc_heap.cc -> _build/preprocessed/cxx-dbg/mycpp/gc_heap.cc
275 pre = '_build/preprocessed/%s-%s/%s' % (compiler, variant, src)
276 preprocessed.append(pre)
277
278 # Summary file
279 n.build('_build/preprocessed/%s-%s.txt' % (compiler, variant),
280 'line_count', preprocessed)
281 n.newline()
282
283
284def InitSteps(n):
285 """Wrappers for build/ninja-rules-*.sh
286
287 Some of these are defined in mycpp/NINJA_subgraph.py. Could move them here.
288 """
289 #
290 # Compiling and linking
291 #
292
293 # Preprocess one translation unit
294 n.rule(
295 'preprocess',
296 # compile_one detects the _build/preprocessed path
297 command=
298 'build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out',
299 description='PP $compiler $variant $more_cxx_flags $in $out')
300 n.newline()
301
302 n.rule('line_count',
303 command='build/ninja-rules-cpp.sh line_count $out $in',
304 description='line_count $out $in')
305 n.newline()
306
307 # Compile one translation unit
308 n.rule(
309 'compile_one',
310 command=
311 'build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out $out.d',
312 depfile='$out.d',
313 # no prefix since the compiler is the first arg
314 description='$compiler $variant $more_cxx_flags $in $out')
315 n.newline()
316
317 # Link objects together
318 n.rule(
319 'link',
320 command=
321 'build/ninja-rules-cpp.sh link $compiler $variant $more_link_flags $out $in',
322 description='LINK $compiler $variant $more_link_flags $out $in')
323 n.newline()
324
325 # 1 input and 2 outputs
326 n.rule('strip',
327 command='build/ninja-rules-cpp.sh strip_ $in $out',
328 description='STRIP $in $out')
329 n.newline()
330
331 # cc_binary can have symliks
332 n.rule('symlink',
333 command='build/ninja-rules-cpp.sh symlink $dir $target $new',
334 description='SYMLINK $dir $target $new')
335 n.newline()
336
337 #
338 # Code generators
339 #
340
341 n.rule(
342 'write-shwrap',
343 # $in must start with main program
344 command='build/ninja-rules-py.sh write-shwrap $template $out $in',
345 description='make-pystub $out $in')
346 n.newline()
347
348 n.rule(
349 'gen-oils-for-unix',
350 command=
351 'build/ninja-rules-py.sh gen-oils-for-unix $main_name $translator $out_prefix $preamble $extra_mycpp_opts $in',
352 description='gen-oils-for-unix $main_name $translator $out_prefix $preamble $extra_mycpp_opts $in')
353 n.newline()
354
355
356def main(argv):
357 try:
358 action = argv[1]
359 except IndexError:
360 action = 'ninja'
361
362 if action == 'ninja':
363 f = open(BUILD_NINJA, 'w')
364 else:
365 f = cStringIO.StringIO() # thrown away
366
367 n = ninja_syntax.Writer(f)
368 ru = ninja_lib.Rules(n)
369
370 ru.comment('InitSteps()')
371 InitSteps(n)
372
373 #
374 # Create the graph.
375 #
376
377 asdl_subgraph.NinjaGraph(ru)
378 ru.comment('')
379
380 bin_subgraph.NinjaGraph(ru)
381 ru.comment('')
382
383 core_subgraph.NinjaGraph(ru)
384 ru.comment('')
385
386 cpp_subgraph.NinjaGraph(ru)
387 ru.comment('')
388
389 data_lang_subgraph.NinjaGraph(ru)
390 ru.comment('')
391
392 display_subgraph.NinjaGraph(ru)
393 ru.comment('')
394
395 frontend_subgraph.NinjaGraph(ru)
396 ru.comment('')
397
398 mycpp_subgraph.NinjaGraph(ru)
399 ru.comment('')
400
401 ysh_subgraph.NinjaGraph(ru)
402 ru.comment('')
403
404 osh_subgraph.NinjaGraph(ru)
405 ru.comment('')
406
407 pea_subgraph.NinjaGraph(ru)
408 ru.comment('')
409
410 prebuilt_subgraph.NinjaGraph(ru)
411 ru.comment('')
412
413 yaks_subgraph.NinjaGraph(ru)
414 ru.comment('')
415
416 # Materialize all the cc_binary() rules
417 ru.WriteRules()
418
419 # Collect sources for metrics, tarball, shell script
420 cc_sources = ru.SourcesForBinary('_gen/bin/oils_for_unix.mycpp.cc')
421
422 if 0:
423 from pprint import pprint
424 pprint(cc_sources)
425
426 # TODO: could thin these out, not generate for unit tests, etc.
427 Preprocessed(n, cc_sources)
428
429 ru.WritePhony()
430
431 n.default(['_bin/cxx-asan/osh', '_bin/cxx-asan/ysh'])
432
433 if action == 'ninja':
434 log(' (%s) -> %s (%d targets)', argv[0], BUILD_NINJA,
435 n.num_build_targets())
436
437 elif action == 'shell':
438 out = '_build/oils.sh'
439 with open(out, 'w') as f:
440 ShellFunctions(cc_sources, f, argv[0])
441 log(' (%s) -> %s', argv[0], out)
442
443 elif action == 'tarball-manifest':
444 h = ru.HeadersForBinary('_gen/bin/oils_for_unix.mycpp.cc')
445 tar_cc_sources = cc_sources + ['_gen/bin/oils_for_unix.mycpp-souffle.cc']
446 TarballManifest(tar_cc_sources + h)
447
448 else:
449 raise RuntimeError('Invalid action %r' % action)
450
451
452if __name__ == '__main__':
453 try:
454 main(sys.argv)
455 except RuntimeError as e:
456 print('FATAL: %s' % e, file=sys.stderr)
457 sys.exit(1)
458
459# vim: sw=2