OILS / build / ninja_main.py View on Github | oilshell.org

426 lines, 227 significant
1#!/usr/bin/env python2
2"""
3build/ninja_main.py - invoked by ./NINJA-config.sh
4
5See build/README.md for the code and data layout.
6"""
7from __future__ import print_function
8
9import cStringIO
10from glob import glob
11import os
12import re
13import sys
14
15from build import ninja_lib
16from build.ninja_lib import log
17
18from asdl import NINJA_subgraph as asdl_subgraph
19from bin import NINJA_subgraph as bin_subgraph
20from core import NINJA_subgraph as core_subgraph
21from cpp import NINJA_subgraph as cpp_subgraph
22from data_lang import NINJA_subgraph as data_lang_subgraph
23from display import NINJA_subgraph as display_subgraph
24from frontend import NINJA_subgraph as frontend_subgraph
25from ysh import NINJA_subgraph as ysh_subgraph
26from osh import NINJA_subgraph as osh_subgraph
27from mycpp import NINJA_subgraph as mycpp_subgraph
28from pea import NINJA_subgraph as pea_subgraph
29from prebuilt import NINJA_subgraph as prebuilt_subgraph
30from yaks import NINJA_subgraph as yaks_subgraph
31
32from vendor import ninja_syntax
33
34# The file Ninja runs by default.
35BUILD_NINJA = 'build.ninja'
36
37
38def TarballManifest(cc_h_files):
39 names = []
40
41 # Code we know about
42 names.extend(cc_h_files)
43
44 names.extend([
45 # Text
46 'LICENSE.txt',
47 'README-native.txt',
48 'INSTALL.txt',
49 'configure',
50 'install',
51 'doc/osh.1',
52
53 # Build Scripts
54 'build/common.sh',
55 'build/native.sh',
56
57 # These 2 are used by build/ninja-rules-cpp.sh
58 'build/py2.sh',
59 'build/dev-shell.sh',
60 'build/ninja-rules-cpp.sh',
61
62 # Generated
63 '_build/oils.sh',
64
65 # These are in build/py.sh, not Ninja. Should probably put them in Ninja.
66 #'_gen/frontend/help_meta.h',
67 '_gen/frontend/match.re2c.h',
68 '_gen/frontend/id_kind.asdl_c.h',
69 '_gen/frontend/types.asdl_c.h',
70 ])
71
72 # For configure
73 names.extend(glob('build/detect-*.c'))
74
75 # TODO: crawl headers
76 # We can now use the headers=[] attribute
77 names.extend(glob('mycpp/*.h'))
78 names.extend(glob('cpp/*.h'))
79
80 # ONLY the headers
81 names.extend(glob('prebuilt/*/*.h'))
82
83 names.sort() # Pass them to tar sorted
84
85 # Check for dupes here
86 unique = sorted(set(names))
87 if names != unique:
88 dupes = [n for n in names if names.count(n) > 1]
89 raise AssertionError("Tarball manifest shouldn't have duplicates: %s" %
90 dupes)
91
92 for name in names:
93 print(name)
94
95
96def ShellFunctions(cc_sources, f, argv0):
97 """
98 Generate a shell fragment that invokes the same function that build.ninja
99 does
100 """
101 print('''\
102main() {
103 ### Compile oils-for-unix into _bin/$compiler-$variant-sh/ (not with ninja)
104
105 parse_flags "$@"
106
107 local compiler=${1:-cxx} # default is system compiler
108 local variant=${2:-opt} # default is optimized build
109 local translator=${3:-mycpp} # default is the translator w/o optimizations
110 local skip_rebuild=${4:-} # if the output exists, skip build'
111
112 local out_dir
113 case $translator in
114 mycpp)
115 out_dir=_bin/$compiler-$variant-sh
116 ;;
117 *)
118 out_dir=_bin/$compiler-$variant-sh/$translator
119 ;;
120 esac
121 local out=$out_dir/oils-for-unix
122
123 if test -n "$skip_rebuild" && test -f "$out"; then
124 echo
125 echo "$0: SKIPPING build because $out exists"
126 echo
127 return
128 fi
129
130 echo
131 echo "$0: Building oils-for-unix: $out"
132 echo "$0: PWD = $PWD"
133 echo
134''',
135 file=f)
136
137 objects = []
138
139 in_out = [
140 ('_gen/bin/oils_for_unix.$translator.cc',
141 '_build/obj/$compiler-$variant-sh/_gen/bin/oils_for_unix.o'),
142 ]
143 for src in sorted(cc_sources):
144 # e.g. _build/obj/cxx-dbg-sh/posix.o
145 prefix, _ = os.path.splitext(src)
146 if prefix.startswith('_gen/bin/oils_for_unix'):
147 continue
148 obj = '_build/obj/$compiler-$variant-sh/%s.o' % prefix
149 in_out.append((src, obj))
150
151
152 bin_dir = '_bin/$compiler-$variant-sh/$translator'
153 obj_dirs = sorted(set(os.path.dirname(obj) for _, obj in in_out))
154
155 all_dirs = [bin_dir] + obj_dirs
156 # Double quote
157 all_dirs = ['"%s"' % d for d in all_dirs]
158
159 print(' mkdir -p \\', file=f)
160 print(' %s' % ' \\\n '.join(all_dirs), file=f)
161 print('', file=f)
162
163 do_fork = ''
164
165 for i, (src, obj) in enumerate(in_out):
166 obj_quoted = '"%s"' % obj
167 objects.append(obj_quoted)
168
169 # Only fork one translation unit that we know to be slow
170 if re.match('.*oils_for_unix\..*\.cc', src):
171 # There should only be one forked translation unit
172 # It can be turned off with OILS_PARALLEL_BUILD= _build/oils
173 assert do_fork == ''
174 do_fork = '_do_fork=$OILS_PARALLEL_BUILD'
175 else:
176 do_fork = ''
177
178 if do_fork:
179 print(' # Potentially fork this translation unit with &', file=f)
180 print(' %s _compile_one "$compiler" "$variant" "" \\' % do_fork,
181 file=f)
182 print(' %s %s' % (src, obj_quoted), file=f)
183 if do_fork:
184 print(' _do_fork= # work around bug in some versions of the dash shell', file=f)
185 print('', file=f)
186
187 print(' # wait for the translation unit before linking', file=f)
188 print(' echo WAIT', file=f)
189 # time -p shows any excess parallelism on 2 cores
190 # example: oils_for_unix.mycpp.cc takes ~8 seconds longer to compile than all
191 # other translation units combined!
192
193 # Timing isn't POSIX
194 #print(' time -p wait', file=f)
195 print(' wait', file=f)
196 print('', file=f)
197
198 print(' echo "LINK $out"', file=f)
199 # note: can't have spaces in filenames
200 print(' link "$compiler" "$variant" "" "$out" \\', file=f)
201 # put each object on its own line, and indent by 4
202 print(' %s' % (' \\\n '.join(objects)), file=f)
203 print('', file=f)
204
205 # Strip opt binary
206 # TODO: provide a way for the user to get symbols?
207
208 print('''\
209 local out_name=oils-for-unix
210 if test "$variant" = opt; then
211 strip -o "$out.stripped" "$out"
212
213 # Symlink to unstripped binary for benchmarking
214 # out_name=$out_name.stripped
215 fi
216
217 cd $out_dir
218 for symlink in osh ysh; do
219 # like ln -v, which we can't use portably
220 echo " $symlink -> $out_name"
221 ln -s -f $out_name $symlink
222 done
223}
224
225main "$@"
226''',
227 file=f)
228
229
230def Preprocessed(n, cc_sources):
231 # See how much input we're feeding to the compiler. Test C++ template
232 # explosion, e.g. <unordered_map>
233 #
234 # Limit to {dbg,opt} so we don't generate useless rules. Invoked by
235 # metrics/source-code.sh
236
237 pre_matrix = [
238 ('cxx', 'dbg'),
239 ('cxx', 'opt'),
240 ('clang', 'dbg'),
241 ('clang', 'opt'),
242 ]
243 for compiler, variant in pre_matrix:
244 preprocessed = []
245 for src in cc_sources:
246 # e.g. mycpp/gc_heap.cc -> _build/preprocessed/cxx-dbg/mycpp/gc_heap.cc
247 pre = '_build/preprocessed/%s-%s/%s' % (compiler, variant, src)
248 preprocessed.append(pre)
249
250 # Summary file
251 n.build('_build/preprocessed/%s-%s.txt' % (compiler, variant),
252 'line_count', preprocessed)
253 n.newline()
254
255
256def InitSteps(n):
257 """Wrappers for build/ninja-rules-*.sh
258
259 Some of these are defined in mycpp/NINJA_subgraph.py. Could move them here.
260 """
261 #
262 # Compiling and linking
263 #
264
265 # Preprocess one translation unit
266 n.rule(
267 'preprocess',
268 # compile_one detects the _build/preprocessed path
269 command=
270 'build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out',
271 description='PP $compiler $variant $more_cxx_flags $in $out')
272 n.newline()
273
274 n.rule('line_count',
275 command='build/ninja-rules-cpp.sh line_count $out $in',
276 description='line_count $out $in')
277 n.newline()
278
279 # Compile one translation unit
280 n.rule(
281 'compile_one',
282 command=
283 'build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out $out.d',
284 depfile='$out.d',
285 # no prefix since the compiler is the first arg
286 description='$compiler $variant $more_cxx_flags $in $out')
287 n.newline()
288
289 # Link objects together
290 n.rule(
291 'link',
292 command=
293 'build/ninja-rules-cpp.sh link $compiler $variant $more_link_flags $out $in',
294 description='LINK $compiler $variant $more_link_flags $out $in')
295 n.newline()
296
297 # 1 input and 2 outputs
298 n.rule('strip',
299 command='build/ninja-rules-cpp.sh strip_ $in $out',
300 description='STRIP $in $out')
301 n.newline()
302
303 # cc_binary can have symliks
304 n.rule('symlink',
305 command='build/ninja-rules-cpp.sh symlink $dir $target $new',
306 description='SYMLINK $dir $target $new')
307 n.newline()
308
309 #
310 # Code generators
311 #
312
313 n.rule(
314 'write-shwrap',
315 # $in must start with main program
316 command='build/ninja-rules-py.sh write-shwrap $template $out $in',
317 description='make-pystub $out $in')
318 n.newline()
319
320 n.rule(
321 'gen-oils-for-unix',
322 command=
323 'build/ninja-rules-py.sh gen-oils-for-unix $main_name $translator $out_prefix $preamble $extra_mycpp_opts $in',
324 description='gen-oils-for-unix $main_name $translator $out_prefix $preamble $extra_mycpp_opts $in')
325 n.newline()
326
327
328def main(argv):
329 try:
330 action = argv[1]
331 except IndexError:
332 action = 'ninja'
333
334 if action == 'ninja':
335 f = open(BUILD_NINJA, 'w')
336 else:
337 f = cStringIO.StringIO() # thrown away
338
339 n = ninja_syntax.Writer(f)
340 ru = ninja_lib.Rules(n)
341
342 ru.comment('InitSteps()')
343 InitSteps(n)
344
345 #
346 # Create the graph.
347 #
348
349 asdl_subgraph.NinjaGraph(ru)
350 ru.comment('')
351
352 bin_subgraph.NinjaGraph(ru)
353 ru.comment('')
354
355 core_subgraph.NinjaGraph(ru)
356 ru.comment('')
357
358 cpp_subgraph.NinjaGraph(ru)
359 ru.comment('')
360
361 data_lang_subgraph.NinjaGraph(ru)
362 ru.comment('')
363
364 display_subgraph.NinjaGraph(ru)
365 ru.comment('')
366
367 frontend_subgraph.NinjaGraph(ru)
368 ru.comment('')
369
370 mycpp_subgraph.NinjaGraph(ru)
371 ru.comment('')
372
373 ysh_subgraph.NinjaGraph(ru)
374 ru.comment('')
375
376 osh_subgraph.NinjaGraph(ru)
377 ru.comment('')
378
379 pea_subgraph.NinjaGraph(ru)
380 ru.comment('')
381
382 prebuilt_subgraph.NinjaGraph(ru)
383 ru.comment('')
384
385 yaks_subgraph.NinjaGraph(ru)
386 ru.comment('')
387
388 # Materialize all the cc_binary() rules
389 ru.WriteRules()
390
391 # Collect sources for metrics, tarball, shell script
392 cc_sources = ru.SourcesForBinary('_gen/bin/oils_for_unix.mycpp.cc')
393
394 if 0:
395 from pprint import pprint
396 pprint(cc_sources)
397
398 # TODO: could thin these out, not generate for unit tests, etc.
399 Preprocessed(n, cc_sources)
400
401 ru.WritePhony()
402
403 n.default(['_bin/cxx-asan/osh', '_bin/cxx-asan/ysh'])
404
405 if action == 'ninja':
406 log(' (%s) -> %s (%d targets)', argv[0], BUILD_NINJA,
407 n.num_build_targets())
408
409 elif action == 'shell':
410 ShellFunctions(cc_sources, sys.stdout, argv[0])
411
412 elif action == 'tarball-manifest':
413 h = ru.HeadersForBinary('_gen/bin/oils_for_unix.mycpp.cc')
414 tar_cc_sources = cc_sources + ['_gen/bin/oils_for_unix.mycpp-souffle.cc']
415 TarballManifest(tar_cc_sources + h)
416
417 else:
418 raise RuntimeError('Invalid action %r' % action)
419
420
421if __name__ == '__main__':
422 try:
423 main(sys.argv)
424 except RuntimeError as e:
425 print('FATAL: %s' % e, file=sys.stderr)
426 sys.exit(1)