OILS / build / ninja_main.py View on Github | oilshell.org

445 lines, 235 significant
1#!/usr/bin/env python2
2"""
3build/ninja_main.py - invoked by ./NINJA-config.sh
4
5See build/README.md for the code and data layout.
6"""
7from __future__ import print_function
8
9import cStringIO
10from glob import glob
11import os
12import re
13import sys
14
15from build import ninja_lib
16from build.ninja_lib import log
17
18from asdl import NINJA_subgraph as asdl_subgraph
19from bin import NINJA_subgraph as bin_subgraph
20from core import NINJA_subgraph as core_subgraph
21from cpp import NINJA_subgraph as cpp_subgraph
22from data_lang import NINJA_subgraph as data_lang_subgraph
23from display import NINJA_subgraph as display_subgraph
24from frontend import NINJA_subgraph as frontend_subgraph
25from ysh import NINJA_subgraph as ysh_subgraph
26from osh import NINJA_subgraph as osh_subgraph
27from mycpp import NINJA_subgraph as mycpp_subgraph
28from pea import NINJA_subgraph as pea_subgraph
29from prebuilt import NINJA_subgraph as prebuilt_subgraph
30from yaks import NINJA_subgraph as yaks_subgraph
31
32from vendor import ninja_syntax
33
34# The file Ninja runs by default.
35BUILD_NINJA = 'build.ninja'
36
37
38def TarballManifest(cc_h_files):
39 names = []
40
41 # Code we know about
42 names.extend(cc_h_files)
43
44 names.extend([
45 # Text
46 'LICENSE.txt',
47 'README-native.txt',
48 'INSTALL.txt',
49 'configure',
50 'install',
51 'doc/osh.1',
52
53 # Build Scripts
54 'build/common.sh',
55 'build/native.sh',
56
57 # These 2 are used by build/ninja-rules-cpp.sh
58 'build/py2.sh',
59 'build/dev-shell.sh',
60 'build/ninja-rules-cpp.sh',
61
62 # Generated
63 '_build/oils.sh',
64
65 # These are in build/py.sh, not Ninja. Should probably put them in Ninja.
66 #'_gen/frontend/help_meta.h',
67 '_gen/frontend/match.re2c.h',
68 '_gen/frontend/id_kind.asdl_c.h',
69 '_gen/frontend/types.asdl_c.h',
70 ])
71
72 # For configure
73 names.extend(glob('build/detect-*.c'))
74
75 # TODO: crawl headers
76 # We can now use the headers=[] attribute
77 names.extend(glob('cpp/*.h'))
78 names.extend(glob('mycpp/*.h'))
79 names.extend(glob('asdl/*.h'))
80
81 # ONLY the headers
82 names.extend(glob('prebuilt/*/*.h'))
83
84 names.sort() # Pass them to tar sorted
85
86 # Check for dupes here
87 unique = sorted(set(names))
88 if names != unique:
89 dupes = [n for n in names if names.count(n) > 1]
90 raise AssertionError("Tarball manifest shouldn't have duplicates: %s" %
91 dupes)
92
93 for name in names:
94 print(name)
95
96
97def ShellFunctions(cc_sources, f, argv0):
98 """
99 Generate a shell fragment that invokes the same function that build.ninja
100 does
101 """
102 print('''\
103main() {
104 ### Compile oils-for-unix into _bin/$compiler-$variant-sh/ (not with ninja)
105
106 parse_flags "$@"
107
108 # Copy into locals
109 local compiler=$FLAG_cxx
110 local variant=$FLAG_variant
111 local translator=$FLAG_translator
112 local skip_rebuild=$FLAG_skip_rebuild
113
114 local out_dir
115 case $translator in
116 mycpp)
117 out_dir=_bin/$compiler-$variant-sh
118 ;;
119 *)
120 out_dir=_bin/$compiler-$variant-sh/$translator
121 ;;
122 esac
123 local out=$out_dir/oils-for-unix
124
125 echo
126 echo "$0: Building oils-for-unix: $out"
127 echo " PWD = $PWD"
128 echo " cxx = $compiler"
129 echo " variant = $variant"
130 echo " translator = $translator"
131 if test -n "$skip_rebuild"; then
132 echo " skip_rebuild = $skip_rebuild"
133 fi
134
135 if test -n "$skip_rebuild" && test -f "$out"; then
136 echo
137 echo "$0: SKIPPING build because $out exists"
138 echo
139 return
140 fi
141
142 echo
143''',
144 file=f)
145
146 objects = []
147
148 in_out = [
149 ('_gen/bin/oils_for_unix.$translator.cc',
150 '_build/obj/$compiler-$variant-sh/_gen/bin/oils_for_unix.o'),
151 ]
152 for src in sorted(cc_sources):
153 # e.g. _build/obj/cxx-dbg-sh/posix.o
154 prefix, _ = os.path.splitext(src)
155 if prefix.startswith('_gen/bin/oils_for_unix'):
156 continue
157 obj = '_build/obj/$compiler-$variant-sh/%s.o' % prefix
158 in_out.append((src, obj))
159
160 bin_dir = '_bin/$compiler-$variant-sh/$translator'
161 obj_dirs = sorted(set(os.path.dirname(obj) for _, obj in in_out))
162
163 all_dirs = [bin_dir] + obj_dirs
164 # Double quote
165 all_dirs = ['"%s"' % d for d in all_dirs]
166
167 print(' mkdir -p \\', file=f)
168 print(' %s' % ' \\\n '.join(all_dirs), file=f)
169 print('', file=f)
170
171 do_fork = ''
172
173 for i, (src, obj) in enumerate(in_out):
174 obj_quoted = '"%s"' % obj
175 objects.append(obj_quoted)
176
177 # Only fork one translation unit that we know to be slow
178 if re.match('.*oils_for_unix\..*\.cc', src):
179 # There should only be one forked translation unit
180 # It can be turned off with OILS_PARALLEL_BUILD= _build/oils
181 assert do_fork == ''
182 do_fork = '_do_fork=$OILS_PARALLEL_BUILD'
183 else:
184 do_fork = ''
185
186 if do_fork:
187 print(' # Potentially fork this translation unit with &', file=f)
188 print(' %s \\' % do_fork, file=f)
189 indent = ' ' if do_fork else ''
190 print(' %s_compile_one "$compiler" "$variant" "" \\' % indent, file=f)
191 print(' %s %s' % (src, obj_quoted), file=f)
192 if do_fork:
193 print(
194 ' _do_fork= # work around bug in some versions of the dash shell',
195 file=f)
196 print('', file=f)
197
198 print(' # wait for the translation unit before linking', file=f)
199 print(' echo WAIT', file=f)
200 # time -p shows any excess parallelism on 2 cores
201 # example: oils_for_unix.mycpp.cc takes ~8 seconds longer to compile than all
202 # other translation units combined!
203
204 # Timing isn't POSIX
205 #print(' time -p wait', file=f)
206 print(' wait', file=f)
207 print('', file=f)
208
209 print(' echo "LINK $out"', file=f)
210 # note: can't have spaces in filenames
211 print(' link "$compiler" "$variant" "" "$out" \\', file=f)
212 # put each object on its own line, and indent by 4
213 print(' %s' % (' \\\n '.join(objects)), file=f)
214 print('', file=f)
215
216 # Strip opt binary
217 # TODO: provide a way for the user to get symbols?
218
219 print('''\
220 local out_name=oils-for-unix
221 if test "$variant" = opt; then
222 strip -o "$out.stripped" "$out"
223
224 # Symlink to unstripped binary for benchmarking
225 # out_name=$out_name.stripped
226 fi
227
228 cd $out_dir
229 for symlink in osh ysh; do
230 # like ln -v, which we can't use portably
231 echo " $symlink -> $out_name"
232 ln -s -f $out_name $symlink
233 done
234}
235
236main "$@"
237''',
238 file=f)
239
240
241def Preprocessed(n, cc_sources):
242 # See how much input we're feeding to the compiler. Test C++ template
243 # explosion, e.g. <unordered_map>
244 #
245 # Limit to {dbg,opt} so we don't generate useless rules. Invoked by
246 # metrics/source-code.sh
247
248 pre_matrix = [
249 ('cxx', 'dbg'),
250 ('cxx', 'opt'),
251 ('clang', 'dbg'),
252 ('clang', 'opt'),
253 ]
254 for compiler, variant in pre_matrix:
255 preprocessed = []
256 for src in cc_sources:
257 # e.g. mycpp/gc_heap.cc -> _build/preprocessed/cxx-dbg/mycpp/gc_heap.cc
258 pre = '_build/preprocessed/%s-%s/%s' % (compiler, variant, src)
259 preprocessed.append(pre)
260
261 # Summary file
262 n.build('_build/preprocessed/%s-%s.txt' % (compiler, variant),
263 'line_count', preprocessed)
264 n.newline()
265
266
267def InitSteps(n):
268 """Wrappers for build/ninja-rules-*.sh
269
270 Some of these are defined in mycpp/NINJA_subgraph.py. Could move them here.
271 """
272 #
273 # Compiling and linking
274 #
275
276 # Preprocess one translation unit
277 n.rule(
278 'preprocess',
279 # compile_one detects the _build/preprocessed path
280 command=
281 'build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out',
282 description='PP $compiler $variant $more_cxx_flags $in $out')
283 n.newline()
284
285 n.rule('line_count',
286 command='build/ninja-rules-cpp.sh line_count $out $in',
287 description='line_count $out $in')
288 n.newline()
289
290 # Compile one translation unit
291 n.rule(
292 'compile_one',
293 command=
294 'build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out $out.d',
295 depfile='$out.d',
296 # no prefix since the compiler is the first arg
297 description='$compiler $variant $more_cxx_flags $in $out')
298 n.newline()
299
300 # Link objects together
301 n.rule(
302 'link',
303 command=
304 'build/ninja-rules-cpp.sh link $compiler $variant $more_link_flags $out $in',
305 description='LINK $compiler $variant $more_link_flags $out $in')
306 n.newline()
307
308 # 1 input and 2 outputs
309 n.rule('strip',
310 command='build/ninja-rules-cpp.sh strip_ $in $out',
311 description='STRIP $in $out')
312 n.newline()
313
314 # cc_binary can have symliks
315 n.rule('symlink',
316 command='build/ninja-rules-cpp.sh symlink $dir $target $new',
317 description='SYMLINK $dir $target $new')
318 n.newline()
319
320 #
321 # Code generators
322 #
323
324 n.rule(
325 'write-shwrap',
326 # $in must start with main program
327 command='build/ninja-rules-py.sh write-shwrap $template $out $in',
328 description='make-pystub $out $in')
329 n.newline()
330
331 # Trivial build rule, for bin/mycpp_main -> _bin/shwrap/mycpp_souffle
332 # while adding implicit deps
333 n.rule('cp', command='cp $in $out', description='cp $in $out')
334 n.newline()
335
336 n.rule(
337 'gen-oils-for-unix',
338 command=
339 'build/ninja-rules-py.sh gen-oils-for-unix $main_name $shwrap_path $out_prefix $preamble $in',
340 description=
341 'gen-oils-for-unix $main_name $shwarp_path $out_prefix $preamble $in')
342 n.newline()
343
344
345def main(argv):
346 try:
347 action = argv[1]
348 except IndexError:
349 action = 'ninja'
350
351 if action == 'ninja':
352 f = open(BUILD_NINJA, 'w')
353 else:
354 f = cStringIO.StringIO() # thrown away
355
356 n = ninja_syntax.Writer(f)
357 ru = ninja_lib.Rules(n)
358
359 ru.comment('InitSteps()')
360 InitSteps(n)
361
362 #
363 # Create the graph.
364 #
365
366 asdl_subgraph.NinjaGraph(ru)
367 ru.comment('')
368
369 bin_subgraph.NinjaGraph(ru)
370 ru.comment('')
371
372 core_subgraph.NinjaGraph(ru)
373 ru.comment('')
374
375 cpp_subgraph.NinjaGraph(ru)
376 ru.comment('')
377
378 data_lang_subgraph.NinjaGraph(ru)
379 ru.comment('')
380
381 display_subgraph.NinjaGraph(ru)
382 ru.comment('')
383
384 frontend_subgraph.NinjaGraph(ru)
385 ru.comment('')
386
387 mycpp_subgraph.NinjaGraph(ru)
388 ru.comment('')
389
390 ysh_subgraph.NinjaGraph(ru)
391 ru.comment('')
392
393 osh_subgraph.NinjaGraph(ru)
394 ru.comment('')
395
396 pea_subgraph.NinjaGraph(ru)
397 ru.comment('')
398
399 prebuilt_subgraph.NinjaGraph(ru)
400 ru.comment('')
401
402 yaks_subgraph.NinjaGraph(ru)
403 ru.comment('')
404
405 # Materialize all the cc_binary() rules
406 ru.WriteRules()
407
408 # Collect sources for metrics, tarball, shell script
409 cc_sources = ru.SourcesForBinary('_gen/bin/oils_for_unix.mycpp.cc')
410
411 if 0:
412 from pprint import pprint
413 pprint(cc_sources)
414
415 # TODO: could thin these out, not generate for unit tests, etc.
416 Preprocessed(n, cc_sources)
417
418 ru.WritePhony()
419
420 n.default(['_bin/cxx-asan/osh', '_bin/cxx-asan/ysh'])
421
422 if action == 'ninja':
423 log(' (%s) -> %s (%d targets)', argv[0], BUILD_NINJA,
424 n.num_build_targets())
425
426 elif action == 'shell':
427 ShellFunctions(cc_sources, sys.stdout, argv[0])
428
429 elif action == 'tarball-manifest':
430 h = ru.HeadersForBinary('_gen/bin/oils_for_unix.mycpp.cc')
431 tar_cc_sources = cc_sources + [
432 '_gen/bin/oils_for_unix.mycpp-souffle.cc'
433 ]
434 TarballManifest(tar_cc_sources + h)
435
436 else:
437 raise RuntimeError('Invalid action %r' % action)
438
439
440if __name__ == '__main__':
441 try:
442 main(sys.argv)
443 except RuntimeError as e:
444 print('FATAL: %s' % e, file=sys.stderr)
445 sys.exit(1)