| 1 | #!/usr/bin/env python2
|
| 2 | """
|
| 3 | build/ninja_main.py - invoked by ./NINJA-config.sh
|
| 4 |
|
| 5 | See build/README.md for the code and data layout.
|
| 6 | """
|
| 7 | from __future__ import print_function
|
| 8 |
|
| 9 | import cStringIO
|
| 10 | from glob import glob
|
| 11 | import os
|
| 12 | import re
|
| 13 | import sys
|
| 14 |
|
| 15 | from build import ninja_lib
|
| 16 | from build.ninja_lib import log
|
| 17 |
|
| 18 | from asdl import NINJA_subgraph as asdl_subgraph
|
| 19 | from bin import NINJA_subgraph as bin_subgraph
|
| 20 | from core import NINJA_subgraph as core_subgraph
|
| 21 | from cpp import NINJA_subgraph as cpp_subgraph
|
| 22 | from data_lang import NINJA_subgraph as data_lang_subgraph
|
| 23 | from display import NINJA_subgraph as display_subgraph
|
| 24 | from frontend import NINJA_subgraph as frontend_subgraph
|
| 25 | from osh import NINJA_subgraph as osh_subgraph
|
| 26 | from mycpp import NINJA_subgraph as mycpp_subgraph
|
| 27 | from pea import NINJA_subgraph as pea_subgraph
|
| 28 | from prebuilt import NINJA_subgraph as prebuilt_subgraph
|
| 29 | from yaks import NINJA_subgraph as yaks_subgraph
|
| 30 | from ysh import NINJA_subgraph as ysh_subgraph
|
| 31 |
|
| 32 | from vendor import ninja_syntax
|
| 33 |
|
| 34 | # The file Ninja runs by default.
|
| 35 | BUILD_NINJA = 'build.ninja'
|
| 36 |
|
| 37 |
|
| 38 | def TarballManifest(some_files):
|
| 39 | names = []
|
| 40 |
|
| 41 | # Code we know about
|
| 42 | names.extend(some_files)
|
| 43 |
|
| 44 | names.extend([
|
| 45 | # Text
|
| 46 | 'LICENSE.txt',
|
| 47 | 'README-native.txt',
|
| 48 | 'INSTALL.txt',
|
| 49 | 'configure',
|
| 50 | 'install',
|
| 51 | 'doc/osh.1',
|
| 52 |
|
| 53 | # Build Scripts
|
| 54 | 'build/common.sh',
|
| 55 | 'build/native.sh',
|
| 56 |
|
| 57 | # These 2 are used by build/ninja-rules-cpp.sh
|
| 58 | 'build/py2.sh',
|
| 59 | 'build/dev-shell.sh',
|
| 60 | 'build/ninja-rules-cpp.sh',
|
| 61 |
|
| 62 | # These are in build/py.sh, not Ninja. Should probably put them in Ninja.
|
| 63 | #'_gen/frontend/help_meta.h',
|
| 64 | '_gen/frontend/match.re2c.h',
|
| 65 | '_gen/frontend/id_kind.asdl_c.h',
|
| 66 | '_gen/frontend/types.asdl_c.h',
|
| 67 | ])
|
| 68 |
|
| 69 | # For configure
|
| 70 | names.extend(glob('build/detect-*.c'))
|
| 71 |
|
| 72 | names.extend(glob('cpp/*.h'))
|
| 73 | names.extend(glob('mycpp/*.h'))
|
| 74 | names.extend(glob('asdl/*.h'))
|
| 75 |
|
| 76 | # ONLY the headers
|
| 77 | names.extend(glob('prebuilt/*/*.h'))
|
| 78 |
|
| 79 | names.sort() # Pass them to tar sorted
|
| 80 |
|
| 81 | # Check for dupes here
|
| 82 | unique = sorted(set(names))
|
| 83 | if names != unique:
|
| 84 | dupes = [n for n in names if names.count(n) > 1]
|
| 85 | raise AssertionError("Tarball manifest shouldn't have duplicates: %s" %
|
| 86 | dupes)
|
| 87 |
|
| 88 | for name in names:
|
| 89 | print(name)
|
| 90 |
|
| 91 |
|
| 92 | def _WinPath(s):
|
| 93 | return s.replace('/', '\\')
|
| 94 |
|
| 95 |
|
| 96 | def BatchFunctions(app_name, cc_sources, f, argv0):
|
| 97 | objects = []
|
| 98 | in_out = []
|
| 99 | for src in sorted(cc_sources):
|
| 100 | # e.g. _build/obj/cxx-dbg-sh/posix.o
|
| 101 | prefix, _ = os.path.splitext(src)
|
| 102 |
|
| 103 | #obj = '_build/obj/$compiler-$variant-sh/%s.o' % prefix
|
| 104 | obj = '_build/obj/cxx-dbg/%s.o' % prefix
|
| 105 | in_out.append((src, obj))
|
| 106 |
|
| 107 | obj_dirs = sorted(set(os.path.dirname(obj) for _, obj in in_out))
|
| 108 | for d in obj_dirs:
|
| 109 | print('mkdir %s' % _WinPath(d), file=f)
|
| 110 |
|
| 111 | print('', file=f)
|
| 112 |
|
| 113 | for i, (src, obj) in enumerate(in_out):
|
| 114 | #obj_quoted = '"%s"' % obj
|
| 115 | objects.append(_WinPath(obj))
|
| 116 |
|
| 117 | # TODO: OILS_WIN32 is a hack, should have ./configure for dprintf()
|
| 118 | print('g++ -I . -D OILS_WIN32 -D MARK_SWEEP -c -o %s %s' %
|
| 119 | (_WinPath(obj), _WinPath(src)),
|
| 120 | file=f)
|
| 121 | print('if %ERRORLEVEL% neq 0 goto :error', file=f)
|
| 122 |
|
| 123 | print('', file=f)
|
| 124 |
|
| 125 | # Link
|
| 126 |
|
| 127 | out_dir = '_bin/cxx-dbg'
|
| 128 | out = '%s/%s' % (out_dir, app_name)
|
| 129 | print('mkdir %s' % _WinPath(out_dir), file=f)
|
| 130 | print('g++ -o %s %s' % (_WinPath(out), ' '.join(objects)), file=f)
|
| 131 |
|
| 132 | print('', file=f)
|
| 133 | print(':error', file=f)
|
| 134 | print(' echo ERROR %ERRORLEVEL%', file=f)
|
| 135 | print(' exit /b %ERRORLEVEL%', file=f)
|
| 136 |
|
| 137 |
|
| 138 | def ShellFunctions(app_name, cc_sources, f, argv0):
|
| 139 | """
|
| 140 | Generate a shell fragment that invokes the same function that build.ninja
|
| 141 | does
|
| 142 | """
|
| 143 | # oils_for_unix -> oils-for-unix
|
| 144 | app_name_hyphens = app_name.replace('_', '-')
|
| 145 |
|
| 146 | print('''\
|
| 147 | main() {
|
| 148 | ### Compile oils-for-unix into _bin/$compiler-$variant-sh/ (not with ninja)
|
| 149 |
|
| 150 | parse_flags "$@"
|
| 151 |
|
| 152 | # Copy into locals
|
| 153 | local compiler=$FLAG_cxx
|
| 154 | local variant=$FLAG_variant
|
| 155 | local translator=$FLAG_translator
|
| 156 | local skip_rebuild=$FLAG_skip_rebuild
|
| 157 |
|
| 158 | local out_name=%s
|
| 159 |
|
| 160 | local out_dir
|
| 161 | case $translator in
|
| 162 | mycpp)
|
| 163 | out_dir=_bin/$compiler-$variant-sh
|
| 164 | ;;
|
| 165 | *)
|
| 166 | out_dir=_bin/$compiler-$variant-sh/$translator
|
| 167 | ;;
|
| 168 | esac
|
| 169 | local out=$out_dir/$out_name
|
| 170 |
|
| 171 | echo
|
| 172 | echo "$0: Building $out_name: $out"
|
| 173 | echo " PWD = $PWD"
|
| 174 | echo " cxx = $compiler"
|
| 175 | echo " variant = $variant"
|
| 176 | echo " translator = $translator"
|
| 177 | if test -n "$skip_rebuild"; then
|
| 178 | echo " skip_rebuild = $skip_rebuild"
|
| 179 | fi
|
| 180 |
|
| 181 | if test -n "$skip_rebuild" && test -f "$out"; then
|
| 182 | echo
|
| 183 | echo "$0: SKIPPING build because $out exists"
|
| 184 | echo
|
| 185 | return
|
| 186 | fi
|
| 187 |
|
| 188 | echo
|
| 189 | ''' % app_name_hyphens,
|
| 190 | file=f)
|
| 191 |
|
| 192 | objects = []
|
| 193 |
|
| 194 | # Filenames that depend on the translator
|
| 195 | in_out = [
|
| 196 | ('_gen/bin/%s.$translator.cc' % app_name,
|
| 197 | '_build/obj/$compiler-$variant-sh/_gen/bin/%s.$translator.o' %
|
| 198 | app_name),
|
| 199 | ('_gen/bin/%s.$translator-main.cc' % app_name,
|
| 200 | '_build/obj/$compiler-$variant-sh/_gen/bin/%s.$translator-main.o' %
|
| 201 | app_name),
|
| 202 | ]
|
| 203 | for src in sorted(cc_sources):
|
| 204 | # e.g. _build/obj/cxx-dbg-sh/posix.o
|
| 205 | prefix, _ = os.path.splitext(src)
|
| 206 |
|
| 207 | # HACK to skip the special ones above
|
| 208 | if re.match(r'.*\.mycpp.*\.cc$', src):
|
| 209 | continue
|
| 210 |
|
| 211 | obj = '_build/obj/$compiler-$variant-sh/%s.o' % prefix
|
| 212 | in_out.append((src, obj))
|
| 213 |
|
| 214 | if 0:
|
| 215 | from pprint import pformat
|
| 216 | log('cc_sources = %s', pformat(cc_sources))
|
| 217 | log('in_out = %s', pformat(in_out))
|
| 218 |
|
| 219 | bin_dir = '_bin/$compiler-$variant-sh/$translator'
|
| 220 | obj_dirs = sorted(set(os.path.dirname(obj) for _, obj in in_out))
|
| 221 |
|
| 222 | all_dirs = [bin_dir] + obj_dirs
|
| 223 | # Double quote
|
| 224 | all_dirs = ['"%s"' % d for d in all_dirs]
|
| 225 |
|
| 226 | print(' mkdir -p \\', file=f)
|
| 227 | print(' %s' % ' \\\n '.join(all_dirs), file=f)
|
| 228 | print('', file=f)
|
| 229 |
|
| 230 | do_fork = ''
|
| 231 |
|
| 232 | for i, (src, obj) in enumerate(in_out):
|
| 233 | obj_quoted = '"%s"' % obj
|
| 234 | objects.append(obj_quoted)
|
| 235 |
|
| 236 | # Only fork one translation unit that we know to be slow
|
| 237 | if src.endswith('.$translator.cc'):
|
| 238 | # There should only be one forked translation unit
|
| 239 | # It can be turned off with OILS_PARALLEL_BUILD= _build/oils
|
| 240 | assert do_fork == ''
|
| 241 | do_fork = '_do_fork=$OILS_PARALLEL_BUILD'
|
| 242 | else:
|
| 243 | do_fork = ''
|
| 244 |
|
| 245 | if do_fork:
|
| 246 | print(' # Potentially fork this translation unit with &', file=f)
|
| 247 | print(' %s \\' % do_fork, file=f)
|
| 248 | indent = ' ' if do_fork else ''
|
| 249 | print(' %s_compile_one "$compiler" "$variant" "" \\' % indent, file=f)
|
| 250 | print(' %s %s' % (src, obj_quoted), file=f)
|
| 251 | if do_fork:
|
| 252 | print(
|
| 253 | ' _do_fork= # work around bug in some versions of the dash shell',
|
| 254 | file=f)
|
| 255 | print('', file=f)
|
| 256 |
|
| 257 | print(' # wait for the translation unit before linking', file=f)
|
| 258 | print(' echo WAIT', file=f)
|
| 259 | # time -p shows any excess parallelism on 2 cores
|
| 260 | # example: oils_for_unix.mycpp.cc takes ~8 seconds longer to compile than all
|
| 261 | # other translation units combined!
|
| 262 |
|
| 263 | # Timing isn't POSIX
|
| 264 | #print(' time -p wait', file=f)
|
| 265 | print(' wait', file=f)
|
| 266 | print('', file=f)
|
| 267 |
|
| 268 | print(' echo "LINK $out"', file=f)
|
| 269 | # note: can't have spaces in filenames
|
| 270 | print(' link "$compiler" "$variant" "" "$out" \\', file=f)
|
| 271 | # put each object on its own line, and indent by 4
|
| 272 | print(' %s' % (' \\\n '.join(objects)), file=f)
|
| 273 | print('', file=f)
|
| 274 |
|
| 275 | # Strip opt binary
|
| 276 | # TODO: provide a way for the user to get symbols?
|
| 277 |
|
| 278 | print('''\
|
| 279 | if test "$variant" = opt; then
|
| 280 | strip -o "$out.stripped" "$out"
|
| 281 |
|
| 282 | # Symlink to unstripped binary for benchmarking
|
| 283 | # out_name=$out_name.stripped
|
| 284 | fi
|
| 285 |
|
| 286 | cd "$out_dir" # dir may have spaces
|
| 287 | for symlink in osh ysh; do
|
| 288 | # like ln -v, which we can't use portably
|
| 289 | echo " $symlink -> $out_name"
|
| 290 | ln -s -f $out_name $symlink
|
| 291 | done
|
| 292 | }
|
| 293 |
|
| 294 | main "$@"
|
| 295 | ''',
|
| 296 | file=f)
|
| 297 |
|
| 298 |
|
| 299 | def WritePreprocessed(n, cc_sources):
|
| 300 | # See how much input we're feeding to the compiler. Test C++ template
|
| 301 | # explosion, e.g. <unordered_map>
|
| 302 | #
|
| 303 | # Limit to {dbg,opt} so we don't generate useless rules. Invoked by
|
| 304 | # metrics/source-code.sh
|
| 305 |
|
| 306 | pre_matrix = [
|
| 307 | ('cxx', 'dbg'),
|
| 308 | ('cxx', 'opt'),
|
| 309 | ('clang', 'dbg'),
|
| 310 | ('clang', 'opt'),
|
| 311 | ]
|
| 312 | for compiler, variant in pre_matrix:
|
| 313 | preprocessed = []
|
| 314 | for src in cc_sources:
|
| 315 | # e.g. mycpp/gc_heap.cc -> _build/preprocessed/cxx-dbg/mycpp/gc_heap.cc
|
| 316 | pre = '_build/preprocessed/%s-%s/%s' % (compiler, variant, src)
|
| 317 | preprocessed.append(pre)
|
| 318 |
|
| 319 | # Summary file
|
| 320 | n.build('_build/preprocessed/%s-%s.txt' % (compiler, variant),
|
| 321 | 'line_count', preprocessed)
|
| 322 | n.newline()
|
| 323 |
|
| 324 |
|
| 325 | def InitSteps(n):
|
| 326 | """Wrappers for build/ninja-rules-*.sh
|
| 327 |
|
| 328 | Some of these are defined in mycpp/NINJA_subgraph.py. Could move them here.
|
| 329 | """
|
| 330 | #
|
| 331 | # Compiling and linking
|
| 332 | #
|
| 333 |
|
| 334 | # Preprocess one translation unit
|
| 335 | n.rule(
|
| 336 | 'preprocess',
|
| 337 | # compile_one detects the _build/preprocessed path
|
| 338 | command=
|
| 339 | 'build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out',
|
| 340 | description='PP $compiler $variant $more_cxx_flags $in $out')
|
| 341 | n.newline()
|
| 342 |
|
| 343 | n.rule('line_count',
|
| 344 | command='build/ninja-rules-cpp.sh line_count $out $in',
|
| 345 | description='line_count $out $in')
|
| 346 | n.newline()
|
| 347 |
|
| 348 | # Compile one translation unit
|
| 349 | n.rule(
|
| 350 | 'compile_one',
|
| 351 | command=
|
| 352 | 'build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out $out.d',
|
| 353 | depfile='$out.d',
|
| 354 | # no prefix since the compiler is the first arg
|
| 355 | description='$compiler $variant $more_cxx_flags $in $out')
|
| 356 | n.newline()
|
| 357 |
|
| 358 | # Link objects together
|
| 359 | n.rule(
|
| 360 | 'link',
|
| 361 | command=
|
| 362 | 'build/ninja-rules-cpp.sh link $compiler $variant $more_link_flags $out $in',
|
| 363 | description='LINK $compiler $variant $more_link_flags $out $in')
|
| 364 | n.newline()
|
| 365 |
|
| 366 | # 1 input and 2 outputs
|
| 367 | n.rule('strip',
|
| 368 | command='build/ninja-rules-cpp.sh strip_ $in $out',
|
| 369 | description='STRIP $in $out')
|
| 370 | n.newline()
|
| 371 |
|
| 372 | # cc_binary can have symliks
|
| 373 | n.rule('symlink',
|
| 374 | command='build/ninja-rules-cpp.sh symlink $dir $target $new',
|
| 375 | description='SYMLINK $dir $target $new')
|
| 376 | n.newline()
|
| 377 |
|
| 378 | #
|
| 379 | # Code generators
|
| 380 | #
|
| 381 |
|
| 382 | n.rule(
|
| 383 | 'write-shwrap',
|
| 384 | # $in must start with main program
|
| 385 | command='build/ninja-rules-py.sh write-shwrap $template $out $in',
|
| 386 | description='make-shwrap $template $out $in')
|
| 387 | n.newline()
|
| 388 |
|
| 389 | n.rule('write-main',
|
| 390 | command=
|
| 391 | 'build/ninja-rules-py.sh write-main $template $out $main_namespace',
|
| 392 | description='write-main $out $main_namespace')
|
| 393 | n.newline()
|
| 394 |
|
| 395 | # Trivial build rule, for bin/mycpp_main -> _bin/shwrap/mycpp_souffle
|
| 396 | # while adding implicit deps
|
| 397 | n.rule('cp', command='cp $in $out', description='cp $in $out')
|
| 398 | n.newline()
|
| 399 |
|
| 400 |
|
| 401 | def main(argv):
|
| 402 | try:
|
| 403 | action = argv[1]
|
| 404 | except IndexError:
|
| 405 | action = 'ninja'
|
| 406 |
|
| 407 | try:
|
| 408 | app_name = argv[2]
|
| 409 | except IndexError:
|
| 410 | app_name = 'oils_for_unix'
|
| 411 |
|
| 412 | if action == 'ninja':
|
| 413 | f = open(BUILD_NINJA, 'w')
|
| 414 | else:
|
| 415 | f = cStringIO.StringIO() # thrown away
|
| 416 |
|
| 417 | n = ninja_syntax.Writer(f)
|
| 418 | ru = ninja_lib.Rules(n)
|
| 419 |
|
| 420 | ru.comment('InitSteps()')
|
| 421 | InitSteps(n)
|
| 422 |
|
| 423 | #
|
| 424 | # Create the graph.
|
| 425 | #
|
| 426 |
|
| 427 | asdl_subgraph.NinjaGraph(ru)
|
| 428 | ru.comment('')
|
| 429 |
|
| 430 | # translate-mycpp rule
|
| 431 | mycpp_subgraph.NinjaGraph(ru)
|
| 432 | ru.comment('')
|
| 433 |
|
| 434 | bin_subgraph.NinjaGraph(ru)
|
| 435 | ru.comment('')
|
| 436 |
|
| 437 | core_subgraph.NinjaGraph(ru)
|
| 438 | ru.comment('')
|
| 439 |
|
| 440 | cpp_subgraph.NinjaGraph(ru)
|
| 441 | ru.comment('')
|
| 442 |
|
| 443 | data_lang_subgraph.NinjaGraph(ru)
|
| 444 | ru.comment('')
|
| 445 |
|
| 446 | display_subgraph.NinjaGraph(ru)
|
| 447 | ru.comment('')
|
| 448 |
|
| 449 | frontend_subgraph.NinjaGraph(ru)
|
| 450 | ru.comment('')
|
| 451 |
|
| 452 | ysh_subgraph.NinjaGraph(ru)
|
| 453 | ru.comment('')
|
| 454 |
|
| 455 | osh_subgraph.NinjaGraph(ru)
|
| 456 | ru.comment('')
|
| 457 |
|
| 458 | pea_subgraph.NinjaGraph(ru)
|
| 459 | ru.comment('')
|
| 460 |
|
| 461 | prebuilt_subgraph.NinjaGraph(ru)
|
| 462 | ru.comment('')
|
| 463 |
|
| 464 | yaks_subgraph.NinjaGraph(ru)
|
| 465 | ru.comment('')
|
| 466 |
|
| 467 | deps = ninja_lib.Deps(ru)
|
| 468 | # Materialize all the cc_binary() rules
|
| 469 | deps.WriteRules()
|
| 470 |
|
| 471 | main_cc = '_gen/bin/%s.mycpp-main.cc' % app_name
|
| 472 |
|
| 473 | # Collect sources for metrics, tarball, shell script
|
| 474 | cc_sources = deps.SourcesForBinary(main_cc)
|
| 475 |
|
| 476 | if 0:
|
| 477 | from pprint import pprint
|
| 478 | pprint(cc_sources)
|
| 479 |
|
| 480 | # TODO: could thin these out, not generate for unit tests, etc.
|
| 481 | WritePreprocessed(n, cc_sources)
|
| 482 |
|
| 483 | ru.WritePhony()
|
| 484 |
|
| 485 | n.default(['_bin/cxx-asan/osh', '_bin/cxx-asan/ysh'])
|
| 486 |
|
| 487 | if action == 'ninja':
|
| 488 | log(' (%s) -> %s (%d targets)', argv[0], BUILD_NINJA,
|
| 489 | n.num_build_targets())
|
| 490 |
|
| 491 | elif action == 'shell':
|
| 492 | ShellFunctions(app_name, cc_sources, sys.stdout, argv[0])
|
| 493 |
|
| 494 | elif action == 'batch':
|
| 495 | # TODO: write batch file
|
| 496 | BatchFunctions(app_name, cc_sources, sys.stdout, argv[0])
|
| 497 |
|
| 498 | elif action == 'tarball-manifest':
|
| 499 | names = list(cc_sources) # copy
|
| 500 |
|
| 501 | h = deps.HeadersForBinary(main_cc)
|
| 502 | names.extend(h)
|
| 503 |
|
| 504 | if app_name == 'oils_for_unix':
|
| 505 | # TODO: SourcesForBinary() and HeadersForBinary can take MULTIPLE
|
| 506 | # -main.cc
|
| 507 | names.extend([
|
| 508 | '_gen/bin/%s.mycpp-souffle.cc' % app_name,
|
| 509 | '_gen/bin/%s.mycpp-souffle-main.cc' % app_name,
|
| 510 | ])
|
| 511 | names.append('_build/oils.sh')
|
| 512 | else:
|
| 513 | names.append('_build/%s.sh' % app_name)
|
| 514 | names.append('_build/%s.bat' % app_name)
|
| 515 |
|
| 516 | TarballManifest(names)
|
| 517 |
|
| 518 | elif action == 'app-deps':
|
| 519 | for name in cc_sources:
|
| 520 | print(name)
|
| 521 | print('---')
|
| 522 | h_sources = deps.HeadersForBinary(main_cc)
|
| 523 | for name in h_sources:
|
| 524 | print(name)
|
| 525 |
|
| 526 | else:
|
| 527 | raise RuntimeError('Invalid action %r' % action)
|
| 528 |
|
| 529 |
|
| 530 | if __name__ == '__main__':
|
| 531 | try:
|
| 532 | main(sys.argv)
|
| 533 | except RuntimeError as e:
|
| 534 | print('FATAL: %s' % e, file=sys.stderr)
|
| 535 | sys.exit(1)
|