1 | #!/usr/bin/env python2
|
2 | """
|
3 | ninja_lib.py
|
4 |
|
5 | Runtime options:
|
6 |
|
7 | CXXFLAGS Additional flags to pass to the C++ compiler
|
8 |
|
9 | Notes on ninja_syntax.py:
|
10 |
|
11 | - escape_path() seems wrong?
|
12 | - It should really take $ to $$.
|
13 | - It doesn't escape newlines
|
14 |
|
15 | return word.replace('$ ', '$$ ').replace(' ', '$ ').replace(':', '$:')
|
16 |
|
17 | Ninja shouldn't have used $ and ALSO used shell commands (sh -c)! Better
|
18 | solutions:
|
19 |
|
20 | - Spawn a process with environment variables.
|
21 | - use % for substitution instead
|
22 |
|
23 | - Another problem: Ninja doesn't escape the # comment character like $#, so
|
24 | how can you write a string with a # as the first char on a line?
|
25 | """
|
26 | from __future__ import print_function
|
27 |
|
28 | import collections
|
29 | import os
|
30 | import sys
|
31 |
|
32 |
|
33 | def log(msg, *args):
|
34 | if args:
|
35 | msg = msg % args
|
36 | print(msg, file=sys.stderr)
|
37 |
|
38 |
|
39 | # Matrix of configurations
|
40 |
|
41 | COMPILERS_VARIANTS = [
|
42 | ('cxx', 'dbg'),
|
43 | ('cxx', 'opt'),
|
44 | ('cxx', 'asan'),
|
45 | ('cxx', 'asan+gcalways'),
|
46 | ('cxx', 'asan32+gcalways'),
|
47 | ('cxx', 'ubsan'),
|
48 |
|
49 | #('clang', 'asan'),
|
50 | ('clang', 'dbg'), # compile-quickly
|
51 | ('clang', 'opt'), # for comparisons
|
52 | ('clang', 'ubsan'), # finds different bugs
|
53 | ('clang', 'coverage'),
|
54 | ]
|
55 |
|
56 | GC_PERF_VARIANTS = [
|
57 | ('cxx', 'opt+bumpleak'),
|
58 | ('cxx', 'opt+bumproot'),
|
59 | ('cxx', 'opt+bumpsmall'),
|
60 | ('cxx', 'asan+bumpsmall'),
|
61 | ('cxx', 'opt+nopool'),
|
62 |
|
63 | # TODO: should be binary with different files
|
64 | ('cxx', 'opt+tcmalloc'),
|
65 |
|
66 | # For tracing allocations, or debugging
|
67 | ('cxx', 'uftrace'),
|
68 |
|
69 | # Test performance of 32-bit build. (It uses less memory usage, but can be
|
70 | # slower.)
|
71 | ('cxx', 'opt32'),
|
72 | ]
|
73 |
|
74 | OTHER_VARIANTS = [
|
75 | ('cxx', 'opt+bigint'),
|
76 | ('cxx', 'opt+souffle'),
|
77 | ('cxx', 'asan+bigint'),
|
78 | ]
|
79 |
|
80 | SMALL_TEST_MATRIX = [
|
81 | ('cxx', 'asan'),
|
82 | ('cxx', 'ubsan'),
|
83 | ('clang', 'coverage'),
|
84 | ]
|
85 |
|
86 |
|
87 | def ConfigDir(config):
|
88 | compiler, variant, more_cxx_flags = config
|
89 | if more_cxx_flags is None:
|
90 | return '%s-%s' % (compiler, variant)
|
91 | else:
|
92 | # -D CPP_UNIT_TEST -> D_CPP_UNIT_TEST
|
93 | flags_str = more_cxx_flags.replace('-', '').replace(' ', '_')
|
94 | return '%s-%s-%s' % (compiler, variant, flags_str)
|
95 |
|
96 |
|
97 | def ObjPath(src_path, config):
|
98 | rel_path, _ = os.path.splitext(src_path)
|
99 | return '_build/obj/%s/%s.o' % (ConfigDir(config), rel_path)
|
100 |
|
101 |
|
102 | # Used namedtuple since it doesn't have any state
|
103 | CcBinary = collections.namedtuple(
|
104 | 'CcBinary',
|
105 | 'main_cc symlinks implicit deps matrix phony_prefix preprocessed bin_path')
|
106 |
|
107 |
|
108 | class CcLibrary(object):
|
109 | """
|
110 | Life cycle:
|
111 |
|
112 | 1. A cc_library is first created
|
113 | 2. A cc_binary can depend on it
|
114 | - maybe writing rules, and ensuring uniques per configuration
|
115 | 3. The link step needs the list of objects
|
116 | 4. The tarball needs the list of sources for binary
|
117 | """
|
118 |
|
119 | def __init__(self, label, srcs, implicit, deps, headers,
|
120 | generated_headers):
|
121 | self.label = label
|
122 | self.srcs = srcs # queried by SourcesForBinary
|
123 | self.implicit = implicit
|
124 | self.deps = deps
|
125 | self.headers = headers
|
126 | # TODO: asdl() rule should add to this.
|
127 | # Generated headers are different than regular headers. The former need an
|
128 | # implicit dep in Ninja, while the latter can rely on the .d mechanism.
|
129 | self.generated_headers = generated_headers
|
130 |
|
131 | self.obj_lookup = {} # config -> list of objects
|
132 | self.preprocessed_lookup = {} # config -> boolean
|
133 |
|
134 | def _CalculateImplicit(self, ru):
|
135 | """ Compile actions for cc_library() also need implicit deps on generated headers"""
|
136 |
|
137 | out_deps = set()
|
138 | ru._TransitiveClosure(self.label, self.deps, out_deps)
|
139 | unique_deps = sorted(out_deps)
|
140 |
|
141 | implicit = list(self.implicit) # copy
|
142 | for label in unique_deps:
|
143 | cc_lib = ru.cc_libs[label]
|
144 | implicit.extend(cc_lib.generated_headers)
|
145 | return implicit
|
146 |
|
147 | def MaybeWrite(self, ru, config, preprocessed):
|
148 | if config not in self.obj_lookup: # already written by some other cc_binary()
|
149 | implicit = self._CalculateImplicit(ru)
|
150 |
|
151 | objects = []
|
152 | for src in self.srcs:
|
153 | obj = ObjPath(src, config)
|
154 | ru.compile(obj, src, self.deps, config, implicit=implicit)
|
155 | objects.append(obj)
|
156 |
|
157 | self.obj_lookup[config] = objects
|
158 |
|
159 | if preprocessed and config not in self.preprocessed_lookup:
|
160 | implicit = self._CalculateImplicit(ru)
|
161 |
|
162 | for src in self.srcs:
|
163 | # no output needed
|
164 | ru.compile('',
|
165 | src,
|
166 | self.deps,
|
167 | config,
|
168 | implicit=implicit,
|
169 | maybe_preprocess=True)
|
170 | self.preprocessed_lookup[config] = True
|
171 |
|
172 |
|
173 | class Rules(object):
|
174 | """High-level wrapper for NinjaWriter
|
175 |
|
176 | What should it handle?
|
177 |
|
178 | - The (compiler, variant) matrix loop
|
179 | - Implicit deps for generated code
|
180 | - Phony convenience targets
|
181 |
|
182 | Maybe: exporting data to test runner
|
183 |
|
184 | Terminology:
|
185 |
|
186 | Ninja has
|
187 | - rules, which are like Bazel "actions"
|
188 | - build targets
|
189 |
|
190 | Our library has:
|
191 | - Build config: (compiler, variant), and more later
|
192 |
|
193 | - Labels: identifiers starting with //, which are higher level than Ninja
|
194 | "targets"
|
195 | cc_library:
|
196 | //mycpp/runtime
|
197 |
|
198 | //mycpp/examples/expr.asdl
|
199 | //frontend/syntax.asdl
|
200 |
|
201 | - Deps are lists of labels, and have a transitive closure
|
202 |
|
203 | - H Rules / High level rules? B rules / Boil?
|
204 | cc_binary, cc_library, asdl, etc.
|
205 | """
|
206 |
|
207 | def __init__(self, n):
|
208 | self.n = n # direct ninja writer
|
209 |
|
210 | self.cc_bins = [] # list of CcBinary() objects to write
|
211 | self.cc_libs = {} # label -> CcLibrary object
|
212 | self.cc_binary_deps = {} # main_cc -> list of LABELS
|
213 | self.phony = {} # list of phony targets
|
214 |
|
215 | def AddPhony(self, phony_to_add):
|
216 | self.phony.update(phony_to_add)
|
217 |
|
218 | def WritePhony(self):
|
219 | for name in sorted(self.phony):
|
220 | targets = self.phony[name]
|
221 | if targets:
|
222 | self.n.build([name], 'phony', targets)
|
223 | self.n.newline()
|
224 |
|
225 | def WriteRules(self):
|
226 | for cc_bin in self.cc_bins:
|
227 | self.WriteCcBinary(cc_bin)
|
228 |
|
229 | def compile(self,
|
230 | out_obj,
|
231 | in_cc,
|
232 | deps,
|
233 | config,
|
234 | implicit=None,
|
235 | maybe_preprocess=False):
|
236 | """ .cc -> compiler -> .o """
|
237 |
|
238 | implicit = implicit or []
|
239 |
|
240 | compiler, variant, more_cxx_flags = config
|
241 | if more_cxx_flags is None:
|
242 | flags_str = "''"
|
243 | else:
|
244 | assert "'" not in more_cxx_flags, more_cxx_flags # can't handle single quotes
|
245 | flags_str = "'%s'" % more_cxx_flags
|
246 |
|
247 | v = [('compiler', compiler), ('variant', variant),
|
248 | ('more_cxx_flags', flags_str)]
|
249 | if maybe_preprocess:
|
250 | # Limit it to certain configs
|
251 | if more_cxx_flags is None and variant in ('dbg', 'opt'):
|
252 | pre = '_build/preprocessed/%s-%s/%s' % (compiler, variant,
|
253 | in_cc)
|
254 | self.n.build(pre,
|
255 | 'preprocess', [in_cc],
|
256 | implicit=implicit,
|
257 | variables=v)
|
258 | else:
|
259 | self.n.build([out_obj],
|
260 | 'compile_one', [in_cc],
|
261 | implicit=implicit,
|
262 | variables=v)
|
263 |
|
264 | self.n.newline()
|
265 |
|
266 | def link(self, out_bin, main_obj, deps, config):
|
267 | """ list of .o -> linker -> executable, along with stripped version """
|
268 | compiler, variant, _ = config
|
269 |
|
270 | assert isinstance(out_bin, str), out_bin
|
271 | assert isinstance(main_obj, str), main_obj
|
272 |
|
273 | objects = [main_obj]
|
274 | for label in deps:
|
275 | key = (label, compiler, variant)
|
276 | try:
|
277 | cc_lib = self.cc_libs[label]
|
278 | except KeyError:
|
279 | raise RuntimeError("Couldn't resolve label %r" % label)
|
280 |
|
281 | o = cc_lib.obj_lookup[config]
|
282 | objects.extend(o)
|
283 |
|
284 | v = [('compiler', compiler), ('variant', variant),
|
285 | ('more_link_flags', "''")]
|
286 | self.n.build([out_bin], 'link', objects, variables=v)
|
287 | self.n.newline()
|
288 |
|
289 | # Strip any .opt binaries
|
290 | if variant.startswith('opt') or variant.startswith('opt32'):
|
291 | stripped = out_bin + '.stripped'
|
292 | symbols = out_bin + '.symbols'
|
293 | self.n.build([stripped, symbols], 'strip', [out_bin])
|
294 | self.n.newline()
|
295 |
|
296 | def comment(self, s):
|
297 | self.n.comment(s)
|
298 | self.n.newline()
|
299 |
|
300 | def cc_library(
|
301 | self,
|
302 | label,
|
303 | srcs=None,
|
304 | implicit=None,
|
305 | deps=None,
|
306 | # note: headers is only used for tarball manifest, not compiler command line
|
307 | headers=None,
|
308 | generated_headers=None):
|
309 |
|
310 | # srcs = [] is allowed for _gen/asdl/hnode.asdl.h
|
311 | if srcs is None:
|
312 | raise RuntimeError('cc_library %r requires srcs' % label)
|
313 |
|
314 | implicit = implicit or []
|
315 | deps = deps or []
|
316 | headers = headers or []
|
317 | generated_headers = generated_headers or []
|
318 |
|
319 | if label in self.cc_libs:
|
320 | raise RuntimeError('%s was already defined' % label)
|
321 |
|
322 | self.cc_libs[label] = CcLibrary(label, srcs, implicit, deps, headers,
|
323 | generated_headers)
|
324 |
|
325 | def _TransitiveClosure(self, name, deps, unique_out):
|
326 | """
|
327 | Args:
|
328 | name: for error messages
|
329 | """
|
330 | for label in deps:
|
331 | if label in unique_out:
|
332 | continue
|
333 | unique_out.add(label)
|
334 |
|
335 | try:
|
336 | cc_lib = self.cc_libs[label]
|
337 | except KeyError:
|
338 | raise RuntimeError('Undefined label %s in %s' % (label, name))
|
339 |
|
340 | self._TransitiveClosure(cc_lib.label, cc_lib.deps, unique_out)
|
341 |
|
342 | def cc_binary(
|
343 | self,
|
344 | main_cc,
|
345 | symlinks=None,
|
346 | implicit=None, # for COMPILE action, not link action
|
347 | deps=None,
|
348 | matrix=None, # $compiler $variant
|
349 | phony_prefix=None,
|
350 | preprocessed=False,
|
351 | bin_path=None, # default is _bin/$compiler-$variant/rel/path
|
352 | ):
|
353 | symlinks = symlinks or []
|
354 | implicit = implicit or []
|
355 | deps = deps or []
|
356 | if not matrix:
|
357 | raise RuntimeError("Config matrix required")
|
358 |
|
359 | cc_bin = CcBinary(main_cc, symlinks, implicit, deps, matrix,
|
360 | phony_prefix, preprocessed, bin_path)
|
361 |
|
362 | self.cc_bins.append(cc_bin)
|
363 |
|
364 | def WriteCcBinary(self, cc_bin):
|
365 | c = cc_bin
|
366 |
|
367 | out_deps = set()
|
368 | self._TransitiveClosure(c.main_cc, c.deps, out_deps)
|
369 | unique_deps = sorted(out_deps)
|
370 |
|
371 | # save for SourcesForBinary()
|
372 | self.cc_binary_deps[c.main_cc] = unique_deps
|
373 |
|
374 | compile_imp = list(c.implicit)
|
375 | for label in unique_deps:
|
376 | cc_lib = self.cc_libs[label] # should exit
|
377 | # compile actions of binaries that have ASDL label deps need the
|
378 | # generated header as implicit dep
|
379 | compile_imp.extend(cc_lib.generated_headers)
|
380 |
|
381 | for config in c.matrix:
|
382 | if len(config) == 2:
|
383 | config = (config[0], config[1], None)
|
384 |
|
385 | for label in unique_deps:
|
386 | cc_lib = self.cc_libs[label] # should exit
|
387 |
|
388 | cc_lib.MaybeWrite(self, config, c.preprocessed)
|
389 |
|
390 | # Compile main object, maybe with IMPLICIT headers deps
|
391 | main_obj = ObjPath(c.main_cc, config)
|
392 | self.compile(main_obj,
|
393 | c.main_cc,
|
394 | c.deps,
|
395 | config,
|
396 | implicit=compile_imp)
|
397 | if c.preprocessed:
|
398 | self.compile('',
|
399 | c.main_cc,
|
400 | c.deps,
|
401 | config,
|
402 | implicit=compile_imp,
|
403 | maybe_preprocess=True)
|
404 |
|
405 | config_dir = ConfigDir(config)
|
406 | bin_dir = '_bin/%s' % config_dir
|
407 |
|
408 | if c.bin_path:
|
409 | # e.g. _bin/cxx-dbg/oils_for_unix
|
410 | bin_ = '%s/%s' % (bin_dir, c.bin_path)
|
411 | bin_subdir, _, bin_name = c.bin_path.rpartition('/')
|
412 | if bin_subdir:
|
413 | bin_dir = '%s/%s' % (bin_dir, bin_subdir)
|
414 | else:
|
415 | bin_name = c.bin_path
|
416 |
|
417 | else:
|
418 | # e.g. _gen/mycpp/examples/classes.mycpp
|
419 | rel_path, _ = os.path.splitext(c.main_cc)
|
420 |
|
421 | # Put binary in _bin/cxx-dbg/mycpp/examples, not _bin/cxx-dbg/_gen/mycpp/examples
|
422 | if rel_path.startswith('_gen/'):
|
423 | rel_path = rel_path[len('_gen/'):]
|
424 |
|
425 | bin_ = '%s/%s' % (bin_dir, rel_path)
|
426 |
|
427 | # Link with OBJECT deps
|
428 | self.link(bin_, main_obj, unique_deps, config)
|
429 |
|
430 | # Make symlinks
|
431 | for symlink in c.symlinks:
|
432 | # Must explicitly specify bin_path to have a symlink, for now
|
433 | assert c.bin_path is not None
|
434 | self.n.build(['%s/%s' % (bin_dir, symlink)],
|
435 | 'symlink', [bin_],
|
436 | variables=[('dir', bin_dir), ('target', bin_name),
|
437 | ('new', symlink)])
|
438 | self.n.newline()
|
439 |
|
440 | if c.phony_prefix:
|
441 | key = '%s-%s' % (c.phony_prefix, config_dir)
|
442 | if key not in self.phony:
|
443 | self.phony[key] = []
|
444 | self.phony[key].append(bin_)
|
445 |
|
446 | def SourcesForBinary(self, main_cc):
|
447 | """
|
448 | Used for preprocessed metrics, release tarball, _build/oils.sh, etc.
|
449 | """
|
450 | deps = self.cc_binary_deps[main_cc]
|
451 | sources = [main_cc]
|
452 | for label in deps:
|
453 | sources.extend(self.cc_libs[label].srcs)
|
454 | return sources
|
455 |
|
456 | def HeadersForBinary(self, main_cc):
|
457 | deps = self.cc_binary_deps[main_cc]
|
458 | headers = []
|
459 | for label in deps:
|
460 | headers.extend(self.cc_libs[label].headers)
|
461 | headers.extend(self.cc_libs[label].generated_headers)
|
462 | return headers
|
463 |
|
464 | def asdl_library(self, asdl_path, deps=None, pretty_print_methods=True):
|
465 |
|
466 | deps = deps or []
|
467 |
|
468 | # SYSTEM header, _gen/asdl/hnode.asdl.h
|
469 | deps.append('//asdl/hnode.asdl')
|
470 | deps.append('//display/pretty.asdl')
|
471 |
|
472 | # to create _gen/mycpp/examples/expr.asdl.h
|
473 | prefix = '_gen/%s' % asdl_path
|
474 |
|
475 | out_cc = prefix + '.cc'
|
476 | out_header = prefix + '.h'
|
477 |
|
478 | asdl_flags = ''
|
479 |
|
480 | if pretty_print_methods:
|
481 | outputs = [out_cc, out_header]
|
482 | else:
|
483 | outputs = [out_header]
|
484 | asdl_flags += '--no-pretty-print-methods'
|
485 |
|
486 | debug_mod = prefix + '_debug.py'
|
487 | outputs.append(debug_mod)
|
488 |
|
489 | # Generating syntax_asdl.h does NOT depend on hnode_asdl.h existing ...
|
490 | self.n.build(outputs,
|
491 | 'asdl-cpp', [asdl_path],
|
492 | implicit=['_bin/shwrap/asdl_main'],
|
493 | variables=[
|
494 | ('action', 'cpp'),
|
495 | ('out_prefix', prefix),
|
496 | ('asdl_flags', asdl_flags),
|
497 | ('debug_mod', debug_mod),
|
498 | ])
|
499 | self.n.newline()
|
500 |
|
501 | # ... But COMPILING anything that #includes it does.
|
502 | # Note: assumes there's a build rule for this "system" ASDL schema
|
503 |
|
504 | srcs = [out_cc] if pretty_print_methods else []
|
505 | # Define lazy CC library
|
506 | self.cc_library(
|
507 | '//' + asdl_path,
|
508 | srcs=srcs,
|
509 | deps=deps,
|
510 | # For compile_one steps of files that #include this ASDL file
|
511 | generated_headers=[out_header],
|
512 | )
|
513 |
|
514 | def py_binary(self, main_py, deps_base_dir='_build/NINJA', template='py'):
|
515 | """
|
516 | Wrapper for Python script with dynamically discovered deps
|
517 | """
|
518 | rel_path, _ = os.path.splitext(main_py)
|
519 | py_module = rel_path.replace(
|
520 | '/', '.') # asdl/asdl_main.py -> asdl.asdl_main
|
521 |
|
522 | deps_path = os.path.join(deps_base_dir, py_module, 'deps.txt')
|
523 | with open(deps_path) as f:
|
524 | deps = [line.strip() for line in f]
|
525 |
|
526 | deps.remove(main_py) # raises ValueError if it's not there
|
527 |
|
528 | basename = os.path.basename(rel_path)
|
529 | self.n.build('_bin/shwrap/%s' % basename,
|
530 | 'write-shwrap', [main_py] + deps,
|
531 | variables=[('template', template)])
|
532 | self.n.newline()
|
533 |
|
534 | def souffle_binary(self, souffle_cpp):
|
535 | """
|
536 | Compile souffle C++ into a native executable.
|
537 | """
|
538 | rel_path, _ = os.path.splitext(souffle_cpp)
|
539 | basename = os.path.basename(rel_path)
|
540 |
|
541 | souffle_obj = '_build/obj/datalog/%s.o' % basename
|
542 | self.n.build([souffle_obj],
|
543 | 'compile_one',
|
544 | souffle_cpp,
|
545 | variables=[('compiler', 'cxx'), ('variant', 'opt'),
|
546 | ('more_cxx_flags', "'-Ivendor -std=c++17'")])
|
547 |
|
548 | souffle_bin = '_bin/datalog/%s' % basename
|
549 | self.n.build([souffle_bin],
|
550 | 'link',
|
551 | souffle_obj,
|
552 | variables=[('compiler', 'cxx'), ('variant', 'opt'),
|
553 | ('more_link_flags', "'-lstdc++fs'")])
|
554 |
|
555 | self.n.newline()
|