| 1 | #!/usr/bin/env python2
|
| 2 | """
|
| 3 | build/ninja_main.py - invoked by ./NINJA-config.sh
|
| 4 |
|
| 5 | See build/README.md for the code and data layout.
|
| 6 | """
|
| 7 | from __future__ import print_function
|
| 8 |
|
| 9 | import cStringIO
|
| 10 | from glob import glob
|
| 11 | import os
|
| 12 | import re
|
| 13 | import sys
|
| 14 |
|
| 15 | from build import ninja_lib
|
| 16 | from build.ninja_lib import log
|
| 17 |
|
| 18 | from asdl import NINJA_subgraph as asdl_subgraph
|
| 19 | from bin import NINJA_subgraph as bin_subgraph
|
| 20 | from core import NINJA_subgraph as core_subgraph
|
| 21 | from cpp import NINJA_subgraph as cpp_subgraph
|
| 22 | from data_lang import NINJA_subgraph as data_lang_subgraph
|
| 23 | from display import NINJA_subgraph as display_subgraph
|
| 24 | from frontend import NINJA_subgraph as frontend_subgraph
|
| 25 | from osh import NINJA_subgraph as osh_subgraph
|
| 26 | from mycpp import NINJA_subgraph as mycpp_subgraph
|
| 27 | from pea import NINJA_subgraph as pea_subgraph
|
| 28 | from prebuilt import NINJA_subgraph as prebuilt_subgraph
|
| 29 | from yaks import NINJA_subgraph as yaks_subgraph
|
| 30 | from ysh import NINJA_subgraph as ysh_subgraph
|
| 31 |
|
| 32 | from vendor import ninja_syntax
|
| 33 |
|
| 34 | # The file Ninja runs by default.
|
| 35 | BUILD_NINJA = 'build.ninja'
|
| 36 |
|
| 37 |
|
| 38 | def TarballManifest(cc_h_files):
|
| 39 | names = []
|
| 40 |
|
| 41 | # Code we know about
|
| 42 | names.extend(cc_h_files)
|
| 43 |
|
| 44 | names.extend([
|
| 45 | # Text
|
| 46 | 'LICENSE.txt',
|
| 47 | 'README-native.txt',
|
| 48 | 'INSTALL.txt',
|
| 49 | 'configure',
|
| 50 | 'install',
|
| 51 | 'doc/osh.1',
|
| 52 |
|
| 53 | # Build Scripts
|
| 54 | 'build/common.sh',
|
| 55 | 'build/native.sh',
|
| 56 |
|
| 57 | # These 2 are used by build/ninja-rules-cpp.sh
|
| 58 | 'build/py2.sh',
|
| 59 | 'build/dev-shell.sh',
|
| 60 | 'build/ninja-rules-cpp.sh',
|
| 61 |
|
| 62 | # Generated
|
| 63 | '_build/oils.sh',
|
| 64 |
|
| 65 | # These are in build/py.sh, not Ninja. Should probably put them in Ninja.
|
| 66 | #'_gen/frontend/help_meta.h',
|
| 67 | '_gen/frontend/match.re2c.h',
|
| 68 | '_gen/frontend/id_kind.asdl_c.h',
|
| 69 | '_gen/frontend/types.asdl_c.h',
|
| 70 | ])
|
| 71 |
|
| 72 | # For configure
|
| 73 | names.extend(glob('build/detect-*.c'))
|
| 74 |
|
| 75 | # TODO: crawl headers
|
| 76 | # We can now use the headers=[] attribute
|
| 77 | names.extend(glob('bin/oils_for_unix_preamble.h'))
|
| 78 |
|
| 79 | # TODO: Parameterize by app
|
| 80 | names.extend(glob('bin/hello_preamble.h'))
|
| 81 |
|
| 82 | names.extend(glob('cpp/*.h'))
|
| 83 | names.extend(glob('mycpp/*.h'))
|
| 84 | names.extend(glob('asdl/*.h'))
|
| 85 |
|
| 86 | # ONLY the headers
|
| 87 | names.extend(glob('prebuilt/*/*.h'))
|
| 88 |
|
| 89 | names.sort() # Pass them to tar sorted
|
| 90 |
|
| 91 | # Check for dupes here
|
| 92 | unique = sorted(set(names))
|
| 93 | if names != unique:
|
| 94 | dupes = [n for n in names if names.count(n) > 1]
|
| 95 | raise AssertionError("Tarball manifest shouldn't have duplicates: %s" %
|
| 96 | dupes)
|
| 97 |
|
| 98 | for name in names:
|
| 99 | print(name)
|
| 100 |
|
| 101 |
|
| 102 | def ShellFunctions(app_name, cc_sources, f, argv0):
|
| 103 | """
|
| 104 | Generate a shell fragment that invokes the same function that build.ninja
|
| 105 | does
|
| 106 | """
|
| 107 | # oils_for_unix -> oils-for-unix
|
| 108 | app_name_hyphens = app_name.replace('_', '-')
|
| 109 |
|
| 110 | print('''\
|
| 111 | main() {
|
| 112 | ### Compile oils-for-unix into _bin/$compiler-$variant-sh/ (not with ninja)
|
| 113 |
|
| 114 | parse_flags "$@"
|
| 115 |
|
| 116 | # Copy into locals
|
| 117 | local compiler=$FLAG_cxx
|
| 118 | local variant=$FLAG_variant
|
| 119 | local translator=$FLAG_translator
|
| 120 | local skip_rebuild=$FLAG_skip_rebuild
|
| 121 |
|
| 122 | local out_name=%s
|
| 123 |
|
| 124 | local out_dir
|
| 125 | case $translator in
|
| 126 | mycpp)
|
| 127 | out_dir=_bin/$compiler-$variant-sh
|
| 128 | ;;
|
| 129 | *)
|
| 130 | out_dir=_bin/$compiler-$variant-sh/$translator
|
| 131 | ;;
|
| 132 | esac
|
| 133 | local out=$out_dir/$out_name
|
| 134 |
|
| 135 | echo
|
| 136 | echo "$0: Building $out_name: $out"
|
| 137 | echo " PWD = $PWD"
|
| 138 | echo " cxx = $compiler"
|
| 139 | echo " variant = $variant"
|
| 140 | echo " translator = $translator"
|
| 141 | if test -n "$skip_rebuild"; then
|
| 142 | echo " skip_rebuild = $skip_rebuild"
|
| 143 | fi
|
| 144 |
|
| 145 | if test -n "$skip_rebuild" && test -f "$out"; then
|
| 146 | echo
|
| 147 | echo "$0: SKIPPING build because $out exists"
|
| 148 | echo
|
| 149 | return
|
| 150 | fi
|
| 151 |
|
| 152 | echo
|
| 153 | ''' % app_name_hyphens,
|
| 154 | file=f)
|
| 155 |
|
| 156 | objects = []
|
| 157 |
|
| 158 | # Filenames that depend on the translator
|
| 159 | in_out = [
|
| 160 | ('_gen/bin/%s.$translator.cc' % app_name,
|
| 161 | '_build/obj/$compiler-$variant-sh/_gen/bin/%s.$translator.o' %
|
| 162 | app_name),
|
| 163 | ('_gen/bin/%s.$translator-main.cc' % app_name,
|
| 164 | '_build/obj/$compiler-$variant-sh/_gen/bin/%s.$translator-main.o' %
|
| 165 | app_name),
|
| 166 | ]
|
| 167 | for src in sorted(cc_sources):
|
| 168 | # e.g. _build/obj/cxx-dbg-sh/posix.o
|
| 169 | prefix, _ = os.path.splitext(src)
|
| 170 |
|
| 171 | # HACK to skip the special ones above
|
| 172 | if re.match(src, r'.mycpp.*\.cc$'):
|
| 173 | continue
|
| 174 |
|
| 175 | obj = '_build/obj/$compiler-$variant-sh/%s.o' % prefix
|
| 176 | in_out.append((src, obj))
|
| 177 |
|
| 178 | bin_dir = '_bin/$compiler-$variant-sh/$translator'
|
| 179 | obj_dirs = sorted(set(os.path.dirname(obj) for _, obj in in_out))
|
| 180 |
|
| 181 | all_dirs = [bin_dir] + obj_dirs
|
| 182 | # Double quote
|
| 183 | all_dirs = ['"%s"' % d for d in all_dirs]
|
| 184 |
|
| 185 | print(' mkdir -p \\', file=f)
|
| 186 | print(' %s' % ' \\\n '.join(all_dirs), file=f)
|
| 187 | print('', file=f)
|
| 188 |
|
| 189 | do_fork = ''
|
| 190 |
|
| 191 | for i, (src, obj) in enumerate(in_out):
|
| 192 | obj_quoted = '"%s"' % obj
|
| 193 | objects.append(obj_quoted)
|
| 194 |
|
| 195 | # Only fork one translation unit that we know to be slow
|
| 196 | if src.endswith('.$translator.cc'):
|
| 197 | # There should only be one forked translation unit
|
| 198 | # It can be turned off with OILS_PARALLEL_BUILD= _build/oils
|
| 199 | assert do_fork == ''
|
| 200 | do_fork = '_do_fork=$OILS_PARALLEL_BUILD'
|
| 201 | else:
|
| 202 | do_fork = ''
|
| 203 |
|
| 204 | if do_fork:
|
| 205 | print(' # Potentially fork this translation unit with &', file=f)
|
| 206 | print(' %s \\' % do_fork, file=f)
|
| 207 | indent = ' ' if do_fork else ''
|
| 208 | print(' %s_compile_one "$compiler" "$variant" "" \\' % indent, file=f)
|
| 209 | print(' %s %s' % (src, obj_quoted), file=f)
|
| 210 | if do_fork:
|
| 211 | print(
|
| 212 | ' _do_fork= # work around bug in some versions of the dash shell',
|
| 213 | file=f)
|
| 214 | print('', file=f)
|
| 215 |
|
| 216 | print(' # wait for the translation unit before linking', file=f)
|
| 217 | print(' echo WAIT', file=f)
|
| 218 | # time -p shows any excess parallelism on 2 cores
|
| 219 | # example: oils_for_unix.mycpp.cc takes ~8 seconds longer to compile than all
|
| 220 | # other translation units combined!
|
| 221 |
|
| 222 | # Timing isn't POSIX
|
| 223 | #print(' time -p wait', file=f)
|
| 224 | print(' wait', file=f)
|
| 225 | print('', file=f)
|
| 226 |
|
| 227 | print(' echo "LINK $out"', file=f)
|
| 228 | # note: can't have spaces in filenames
|
| 229 | print(' link "$compiler" "$variant" "" "$out" \\', file=f)
|
| 230 | # put each object on its own line, and indent by 4
|
| 231 | print(' %s' % (' \\\n '.join(objects)), file=f)
|
| 232 | print('', file=f)
|
| 233 |
|
| 234 | # Strip opt binary
|
| 235 | # TODO: provide a way for the user to get symbols?
|
| 236 |
|
| 237 | print('''\
|
| 238 | if test "$variant" = opt; then
|
| 239 | strip -o "$out.stripped" "$out"
|
| 240 |
|
| 241 | # Symlink to unstripped binary for benchmarking
|
| 242 | # out_name=$out_name.stripped
|
| 243 | fi
|
| 244 |
|
| 245 | cd "$out_dir" # dir may have spaces
|
| 246 | for symlink in osh ysh; do
|
| 247 | # like ln -v, which we can't use portably
|
| 248 | echo " $symlink -> $out_name"
|
| 249 | ln -s -f $out_name $symlink
|
| 250 | done
|
| 251 | }
|
| 252 |
|
| 253 | main "$@"
|
| 254 | ''',
|
| 255 | file=f)
|
| 256 |
|
| 257 |
|
| 258 | def WritePreprocessed(n, cc_sources):
|
| 259 | # See how much input we're feeding to the compiler. Test C++ template
|
| 260 | # explosion, e.g. <unordered_map>
|
| 261 | #
|
| 262 | # Limit to {dbg,opt} so we don't generate useless rules. Invoked by
|
| 263 | # metrics/source-code.sh
|
| 264 |
|
| 265 | pre_matrix = [
|
| 266 | ('cxx', 'dbg'),
|
| 267 | ('cxx', 'opt'),
|
| 268 | ('clang', 'dbg'),
|
| 269 | ('clang', 'opt'),
|
| 270 | ]
|
| 271 | for compiler, variant in pre_matrix:
|
| 272 | preprocessed = []
|
| 273 | for src in cc_sources:
|
| 274 | # e.g. mycpp/gc_heap.cc -> _build/preprocessed/cxx-dbg/mycpp/gc_heap.cc
|
| 275 | pre = '_build/preprocessed/%s-%s/%s' % (compiler, variant, src)
|
| 276 | preprocessed.append(pre)
|
| 277 |
|
| 278 | # Summary file
|
| 279 | n.build('_build/preprocessed/%s-%s.txt' % (compiler, variant),
|
| 280 | 'line_count', preprocessed)
|
| 281 | n.newline()
|
| 282 |
|
| 283 |
|
| 284 | def InitSteps(n):
|
| 285 | """Wrappers for build/ninja-rules-*.sh
|
| 286 |
|
| 287 | Some of these are defined in mycpp/NINJA_subgraph.py. Could move them here.
|
| 288 | """
|
| 289 | #
|
| 290 | # Compiling and linking
|
| 291 | #
|
| 292 |
|
| 293 | # Preprocess one translation unit
|
| 294 | n.rule(
|
| 295 | 'preprocess',
|
| 296 | # compile_one detects the _build/preprocessed path
|
| 297 | command=
|
| 298 | 'build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out',
|
| 299 | description='PP $compiler $variant $more_cxx_flags $in $out')
|
| 300 | n.newline()
|
| 301 |
|
| 302 | n.rule('line_count',
|
| 303 | command='build/ninja-rules-cpp.sh line_count $out $in',
|
| 304 | description='line_count $out $in')
|
| 305 | n.newline()
|
| 306 |
|
| 307 | # Compile one translation unit
|
| 308 | n.rule(
|
| 309 | 'compile_one',
|
| 310 | command=
|
| 311 | 'build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out $out.d',
|
| 312 | depfile='$out.d',
|
| 313 | # no prefix since the compiler is the first arg
|
| 314 | description='$compiler $variant $more_cxx_flags $in $out')
|
| 315 | n.newline()
|
| 316 |
|
| 317 | # Link objects together
|
| 318 | n.rule(
|
| 319 | 'link',
|
| 320 | command=
|
| 321 | 'build/ninja-rules-cpp.sh link $compiler $variant $more_link_flags $out $in',
|
| 322 | description='LINK $compiler $variant $more_link_flags $out $in')
|
| 323 | n.newline()
|
| 324 |
|
| 325 | # 1 input and 2 outputs
|
| 326 | n.rule('strip',
|
| 327 | command='build/ninja-rules-cpp.sh strip_ $in $out',
|
| 328 | description='STRIP $in $out')
|
| 329 | n.newline()
|
| 330 |
|
| 331 | # cc_binary can have symliks
|
| 332 | n.rule('symlink',
|
| 333 | command='build/ninja-rules-cpp.sh symlink $dir $target $new',
|
| 334 | description='SYMLINK $dir $target $new')
|
| 335 | n.newline()
|
| 336 |
|
| 337 | #
|
| 338 | # Code generators
|
| 339 | #
|
| 340 |
|
| 341 | n.rule(
|
| 342 | 'write-shwrap',
|
| 343 | # $in must start with main program
|
| 344 | command='build/ninja-rules-py.sh write-shwrap $template $out $in',
|
| 345 | description='make-shwrap $template $out $in')
|
| 346 | n.newline()
|
| 347 |
|
| 348 | n.rule('write-main',
|
| 349 | command=
|
| 350 | 'build/ninja-rules-py.sh write-main $template $out $main_namespace',
|
| 351 | description='write-main $out $main_namespace')
|
| 352 | n.newline()
|
| 353 |
|
| 354 | # Trivial build rule, for bin/mycpp_main -> _bin/shwrap/mycpp_souffle
|
| 355 | # while adding implicit deps
|
| 356 | n.rule('cp', command='cp $in $out', description='cp $in $out')
|
| 357 | n.newline()
|
| 358 |
|
| 359 |
|
| 360 | def main(argv):
|
| 361 | try:
|
| 362 | action = argv[1]
|
| 363 | except IndexError:
|
| 364 | action = 'ninja'
|
| 365 |
|
| 366 | try:
|
| 367 | app_name = argv[2]
|
| 368 | except IndexError:
|
| 369 | app_name = 'oils_for_unix'
|
| 370 |
|
| 371 | if action == 'ninja':
|
| 372 | f = open(BUILD_NINJA, 'w')
|
| 373 | else:
|
| 374 | f = cStringIO.StringIO() # thrown away
|
| 375 |
|
| 376 | n = ninja_syntax.Writer(f)
|
| 377 | ru = ninja_lib.Rules(n)
|
| 378 |
|
| 379 | ru.comment('InitSteps()')
|
| 380 | InitSteps(n)
|
| 381 |
|
| 382 | #
|
| 383 | # Create the graph.
|
| 384 | #
|
| 385 |
|
| 386 | asdl_subgraph.NinjaGraph(ru)
|
| 387 | ru.comment('')
|
| 388 |
|
| 389 | # translate-mycpp rule
|
| 390 | mycpp_subgraph.NinjaGraph(ru)
|
| 391 | ru.comment('')
|
| 392 |
|
| 393 | bin_subgraph.NinjaGraph(ru)
|
| 394 | ru.comment('')
|
| 395 |
|
| 396 | core_subgraph.NinjaGraph(ru)
|
| 397 | ru.comment('')
|
| 398 |
|
| 399 | cpp_subgraph.NinjaGraph(ru)
|
| 400 | ru.comment('')
|
| 401 |
|
| 402 | data_lang_subgraph.NinjaGraph(ru)
|
| 403 | ru.comment('')
|
| 404 |
|
| 405 | display_subgraph.NinjaGraph(ru)
|
| 406 | ru.comment('')
|
| 407 |
|
| 408 | frontend_subgraph.NinjaGraph(ru)
|
| 409 | ru.comment('')
|
| 410 |
|
| 411 | ysh_subgraph.NinjaGraph(ru)
|
| 412 | ru.comment('')
|
| 413 |
|
| 414 | osh_subgraph.NinjaGraph(ru)
|
| 415 | ru.comment('')
|
| 416 |
|
| 417 | pea_subgraph.NinjaGraph(ru)
|
| 418 | ru.comment('')
|
| 419 |
|
| 420 | prebuilt_subgraph.NinjaGraph(ru)
|
| 421 | ru.comment('')
|
| 422 |
|
| 423 | yaks_subgraph.NinjaGraph(ru)
|
| 424 | ru.comment('')
|
| 425 |
|
| 426 | deps = ninja_lib.Deps(ru)
|
| 427 | # Materialize all the cc_binary() rules
|
| 428 | deps.WriteRules()
|
| 429 |
|
| 430 | # Collect sources for metrics, tarball, shell script
|
| 431 | cc_sources = deps.SourcesForBinary('_gen/bin/%s.mycpp-main.cc' % app_name)
|
| 432 | log('cc_sources = %s', cc_sources)
|
| 433 |
|
| 434 | if 0:
|
| 435 | from pprint import pprint
|
| 436 | pprint(cc_sources)
|
| 437 |
|
| 438 | # TODO: could thin these out, not generate for unit tests, etc.
|
| 439 | WritePreprocessed(n, cc_sources)
|
| 440 |
|
| 441 | ru.WritePhony()
|
| 442 |
|
| 443 | n.default(['_bin/cxx-asan/osh', '_bin/cxx-asan/ysh'])
|
| 444 |
|
| 445 | if action == 'ninja':
|
| 446 | log(' (%s) -> %s (%d targets)', argv[0], BUILD_NINJA,
|
| 447 | n.num_build_targets())
|
| 448 |
|
| 449 | elif action == 'shell':
|
| 450 | ShellFunctions(app_name, cc_sources, sys.stdout, argv[0])
|
| 451 |
|
| 452 | elif action == 'tarball-manifest':
|
| 453 | h = deps.HeadersForBinary('_gen/bin/%s.mycpp-main.cc' % app_name)
|
| 454 | souffle = [
|
| 455 | '_gen/bin/%s.mycpp-souffle.cc' % app_name,
|
| 456 | '_gen/bin/%s.mycpp-souffle-main.cc' % app_name,
|
| 457 | ]
|
| 458 | TarballManifest(souffle + cc_sources + h)
|
| 459 |
|
| 460 | else:
|
| 461 | raise RuntimeError('Invalid action %r' % action)
|
| 462 |
|
| 463 |
|
| 464 | if __name__ == '__main__':
|
| 465 | try:
|
| 466 | main(sys.argv)
|
| 467 | except RuntimeError as e:
|
| 468 | print('FATAL: %s' % e, file=sys.stderr)
|
| 469 | sys.exit(1)
|