OILS / build / ninja_lib.py View on Github | oils.pub

568 lines, 346 significant
1#!/usr/bin/env python2
2"""
3ninja_lib.py
4
5Runtime options:
6
7 CXXFLAGS Additional flags to pass to the C++ compiler
8
9Notes on ninja_syntax.py:
10
11- escape_path() seems wrong?
12 - It should really take $ to $$.
13 - It doesn't escape newlines
14
15 return word.replace('$ ', '$$ ').replace(' ', '$ ').replace(':', '$:')
16
17 Ninja shouldn't have used $ and ALSO used shell commands (sh -c)! Better
18 solutions:
19
20 - Spawn a process with environment variables.
21 - use % for substitution instead
22
23- Another problem: Ninja doesn't escape the # comment character like $#, so
24 how can you write a string with a # as the first char on a line?
25"""
26from __future__ import print_function
27
28import collections
29import os
30import sys
31
32
33def log(msg, *args):
34 if args:
35 msg = msg % args
36 print(msg, file=sys.stderr)
37
38
39# Matrix of configurations
40
41COMPILERS_VARIANTS = [
42 ('cxx', 'dbg'),
43 ('cxx', 'opt'),
44 ('cxx', 'asan'),
45 ('cxx', 'asan+gcalways'),
46 ('cxx', 'asan32+gcalways'),
47 ('cxx', 'ubsan'),
48
49 #('clang', 'asan'),
50 ('clang', 'dbg'), # compile-quickly
51 ('clang', 'opt'), # for comparisons
52 ('clang', 'ubsan'), # finds different bugs
53 ('clang', 'coverage'),
54]
55
56GC_PERF_VARIANTS = [
57 ('cxx', 'opt+bumpleak'),
58 ('cxx', 'opt+bumproot'),
59 ('cxx', 'opt+bumpsmall'),
60 ('cxx', 'asan+bumpsmall'),
61 ('cxx', 'opt+nopool'),
62
63 # TODO: should be binary with different files
64 ('cxx', 'opt+tcmalloc'),
65
66 # For tracing allocations, or debugging
67 ('cxx', 'uftrace'),
68
69 # Test performance of 32-bit build. (It uses less memory usage, but can be
70 # slower.)
71 ('cxx', 'opt32'),
72]
73
74OTHER_VARIANTS = [
75 ('cxx', 'opt+bigint'),
76 ('cxx', 'opt+souffle'),
77 ('cxx', 'asan+bigint'),
78]
79
80SMALL_TEST_MATRIX = [
81 ('cxx', 'asan'),
82 ('cxx', 'ubsan'),
83 ('clang', 'coverage'),
84]
85
86
87def ConfigDir(config):
88 compiler, variant, more_cxx_flags = config
89 if more_cxx_flags is None:
90 return '%s-%s' % (compiler, variant)
91 else:
92 # -D CPP_UNIT_TEST -> D_CPP_UNIT_TEST
93 flags_str = more_cxx_flags.replace('-', '').replace(' ', '_')
94 return '%s-%s-%s' % (compiler, variant, flags_str)
95
96
97def ObjPath(src_path, config):
98 rel_path, _ = os.path.splitext(src_path)
99 return '_build/obj/%s/%s.o' % (ConfigDir(config), rel_path)
100
101
102# Used namedtuple since it doesn't have any state
103CcBinary = collections.namedtuple(
104 'CcBinary',
105 'main_cc symlinks implicit deps matrix phony_prefix preprocessed bin_path')
106
107
108class CcLibrary(object):
109 """
110 Life cycle:
111
112 1. A cc_library is first created
113 2. A cc_binary can depend on it
114 - maybe writing rules, and ensuring uniques per configuration
115 3. The link step needs the list of objects
116 4. The tarball needs the list of sources for binary
117 """
118
119 def __init__(self, label, srcs, implicit, deps, headers,
120 generated_headers):
121 self.label = label
122 self.srcs = srcs # queried by SourcesForBinary
123 self.implicit = implicit
124 self.deps = deps
125 self.headers = headers
126 # TODO: asdl() rule should add to this.
127 # Generated headers are different than regular headers. The former need an
128 # implicit dep in Ninja, while the latter can rely on the .d mechanism.
129 self.generated_headers = generated_headers
130
131 self.obj_lookup = {} # config -> list of objects
132 self.preprocessed_lookup = {} # config -> boolean
133
134 def _CalculateImplicit(self, ru):
135 """ Compile actions for cc_library() also need implicit deps on generated headers"""
136
137 out_deps = set()
138 ru._TransitiveClosure(self.label, self.deps, out_deps)
139 unique_deps = sorted(out_deps)
140
141 implicit = list(self.implicit) # copy
142 for label in unique_deps:
143 cc_lib = ru.cc_libs[label]
144 implicit.extend(cc_lib.generated_headers)
145 return implicit
146
147 def MaybeWrite(self, ru, config, preprocessed):
148 if config not in self.obj_lookup: # already written by some other cc_binary()
149 implicit = self._CalculateImplicit(ru)
150
151 objects = []
152 for src in self.srcs:
153 obj = ObjPath(src, config)
154 ru.compile(obj, src, self.deps, config, implicit=implicit)
155 objects.append(obj)
156
157 self.obj_lookup[config] = objects
158
159 if preprocessed and config not in self.preprocessed_lookup:
160 implicit = self._CalculateImplicit(ru)
161
162 for src in self.srcs:
163 # no output needed
164 ru.compile('',
165 src,
166 self.deps,
167 config,
168 implicit=implicit,
169 maybe_preprocess=True)
170 self.preprocessed_lookup[config] = True
171
172
173class Rules(object):
174 """High-level wrapper for NinjaWriter
175
176 What should it handle?
177
178 - The (compiler, variant) matrix loop
179 - Implicit deps for generated code
180 - Phony convenience targets
181
182 Maybe: exporting data to test runner
183
184 Terminology:
185
186 Ninja has
187 - rules, which are like Bazel "actions"
188 - build targets
189
190 Our library has:
191 - Build config: (compiler, variant), and more later
192
193 - Labels: identifiers starting with //, which are higher level than Ninja
194 "targets"
195 cc_library:
196 //mycpp/runtime
197
198 //mycpp/examples/expr.asdl
199 //frontend/syntax.asdl
200
201 - Deps are lists of labels, and have a transitive closure
202
203 - H Rules / High level rules? B rules / Boil?
204 cc_binary, cc_library, asdl, etc.
205 """
206
207 def __init__(self, n):
208 self.n = n # direct ninja writer
209
210 self.cc_bins = [] # list of CcBinary() objects to write
211 self.cc_libs = {} # label -> CcLibrary object
212 self.cc_binary_deps = {} # main_cc -> list of LABELS
213 self.phony = {} # list of phony targets
214
215 def AddPhony(self, phony_to_add):
216 self.phony.update(phony_to_add)
217
218 def WritePhony(self):
219 for name in sorted(self.phony):
220 targets = self.phony[name]
221 if targets:
222 self.n.build([name], 'phony', targets)
223 self.n.newline()
224
225 def WriteRules(self):
226 for cc_bin in self.cc_bins:
227 self.WriteCcBinary(cc_bin)
228
229 def compile(self,
230 out_obj,
231 in_cc,
232 deps,
233 config,
234 implicit=None,
235 maybe_preprocess=False):
236 """ .cc -> compiler -> .o """
237
238 implicit = implicit or []
239
240 compiler, variant, more_cxx_flags = config
241 if more_cxx_flags is None:
242 flags_str = "''"
243 else:
244 assert "'" not in more_cxx_flags, more_cxx_flags # can't handle single quotes
245 flags_str = "'%s'" % more_cxx_flags
246
247 v = [('compiler', compiler), ('variant', variant),
248 ('more_cxx_flags', flags_str)]
249 if maybe_preprocess:
250 # Limit it to certain configs
251 if more_cxx_flags is None and variant in ('dbg', 'opt'):
252 pre = '_build/preprocessed/%s-%s/%s' % (compiler, variant,
253 in_cc)
254 self.n.build(pre,
255 'preprocess', [in_cc],
256 implicit=implicit,
257 variables=v)
258 else:
259 self.n.build([out_obj],
260 'compile_one', [in_cc],
261 implicit=implicit,
262 variables=v)
263
264 self.n.newline()
265
266 def link(self, out_bin, main_obj, deps, config):
267 """ list of .o -> linker -> executable, along with stripped version """
268 compiler, variant, _ = config
269
270 assert isinstance(out_bin, str), out_bin
271 assert isinstance(main_obj, str), main_obj
272
273 objects = [main_obj]
274 for label in deps:
275 try:
276 cc_lib = self.cc_libs[label]
277 except KeyError:
278 raise RuntimeError("Couldn't resolve label %r" % label)
279
280 o = cc_lib.obj_lookup[config]
281 objects.extend(o)
282
283 v = [('compiler', compiler), ('variant', variant),
284 ('more_link_flags', "''")]
285 self.n.build([out_bin], 'link', objects, variables=v)
286 self.n.newline()
287
288 # Strip any .opt binaries
289 if variant.startswith('opt') or variant.startswith('opt32'):
290 stripped = out_bin + '.stripped'
291 symbols = out_bin + '.symbols'
292 self.n.build([stripped, symbols], 'strip', [out_bin])
293 self.n.newline()
294
295 def comment(self, s):
296 self.n.comment(s)
297 self.n.newline()
298
299 def cc_library(
300 self,
301 label,
302 srcs=None,
303 implicit=None,
304 deps=None,
305 # note: headers is only used for tarball manifest, not compiler command line
306 headers=None,
307 generated_headers=None):
308
309 # srcs = [] is allowed for _gen/asdl/hnode.asdl.h
310 if srcs is None:
311 raise RuntimeError('cc_library %r requires srcs' % label)
312
313 implicit = implicit or []
314 deps = deps or []
315 headers = headers or []
316 generated_headers = generated_headers or []
317
318 if label in self.cc_libs:
319 raise RuntimeError('%s was already defined' % label)
320
321 self.cc_libs[label] = CcLibrary(label, srcs, implicit, deps, headers,
322 generated_headers)
323
324 def _TransitiveClosure(self, name, deps, unique_out):
325 """
326 Args:
327 name: for error messages
328 """
329 for label in deps:
330 if label in unique_out:
331 continue
332 unique_out.add(label)
333
334 try:
335 cc_lib = self.cc_libs[label]
336 except KeyError:
337 raise RuntimeError('Undefined label %s in %s' % (label, name))
338
339 self._TransitiveClosure(cc_lib.label, cc_lib.deps, unique_out)
340
341 def cc_binary(
342 self,
343 main_cc,
344 symlinks=None,
345 implicit=None, # for COMPILE action, not link action
346 deps=None,
347 matrix=None, # $compiler $variant
348 phony_prefix=None,
349 preprocessed=False,
350 bin_path=None, # default is _bin/$compiler-$variant/rel/path
351 ):
352 symlinks = symlinks or []
353 implicit = implicit or []
354 deps = deps or []
355 if not matrix:
356 raise RuntimeError("Config matrix required")
357
358 cc_bin = CcBinary(main_cc, symlinks, implicit, deps, matrix,
359 phony_prefix, preprocessed, bin_path)
360
361 self.cc_bins.append(cc_bin)
362
363 def WriteCcBinary(self, cc_bin):
364 c = cc_bin
365
366 out_deps = set()
367 self._TransitiveClosure(c.main_cc, c.deps, out_deps)
368 unique_deps = sorted(out_deps)
369
370 # save for SourcesForBinary()
371 self.cc_binary_deps[c.main_cc] = unique_deps
372
373 compile_imp = list(c.implicit)
374 for label in unique_deps:
375 cc_lib = self.cc_libs[label] # should exit
376 # compile actions of binaries that have ASDL label deps need the
377 # generated header as implicit dep
378 compile_imp.extend(cc_lib.generated_headers)
379
380 for config in c.matrix:
381 if len(config) == 2:
382 config = (config[0], config[1], None)
383
384 for label in unique_deps:
385 cc_lib = self.cc_libs[label] # should exit
386
387 cc_lib.MaybeWrite(self, config, c.preprocessed)
388
389 # Compile main object, maybe with IMPLICIT headers deps
390 main_obj = ObjPath(c.main_cc, config)
391 self.compile(main_obj,
392 c.main_cc,
393 c.deps,
394 config,
395 implicit=compile_imp)
396 if c.preprocessed:
397 self.compile('',
398 c.main_cc,
399 c.deps,
400 config,
401 implicit=compile_imp,
402 maybe_preprocess=True)
403
404 config_dir = ConfigDir(config)
405 bin_dir = '_bin/%s' % config_dir
406
407 if c.bin_path:
408 # e.g. _bin/cxx-dbg/oils_for_unix
409 bin_ = '%s/%s' % (bin_dir, c.bin_path)
410 bin_subdir, _, bin_name = c.bin_path.rpartition('/')
411 if bin_subdir:
412 bin_dir = '%s/%s' % (bin_dir, bin_subdir)
413 else:
414 bin_name = c.bin_path
415
416 else:
417 # e.g. _gen/mycpp/examples/classes.mycpp
418 rel_path, _ = os.path.splitext(c.main_cc)
419
420 # Put binary in _bin/cxx-dbg/mycpp/examples, not _bin/cxx-dbg/_gen/mycpp/examples
421 if rel_path.startswith('_gen/'):
422 rel_path = rel_path[len('_gen/'):]
423
424 bin_ = '%s/%s' % (bin_dir, rel_path)
425
426 # Link with OBJECT deps
427 self.link(bin_, main_obj, unique_deps, config)
428
429 # Make symlinks
430 for symlink in c.symlinks:
431 # Must explicitly specify bin_path to have a symlink, for now
432 assert c.bin_path is not None
433 self.n.build(['%s/%s' % (bin_dir, symlink)],
434 'symlink', [bin_],
435 variables=[('dir', bin_dir), ('target', bin_name),
436 ('new', symlink)])
437 self.n.newline()
438
439 if c.phony_prefix:
440 key = '%s-%s' % (c.phony_prefix, config_dir)
441 if key not in self.phony:
442 self.phony[key] = []
443 self.phony[key].append(bin_)
444
445 def SourcesForBinary(self, main_cc):
446 """
447 Used for preprocessed metrics, release tarball, _build/oils.sh, etc.
448 """
449 deps = self.cc_binary_deps[main_cc]
450 sources = [main_cc]
451 for label in deps:
452 sources.extend(self.cc_libs[label].srcs)
453 return sources
454
455 def HeadersForBinary(self, main_cc):
456 deps = self.cc_binary_deps[main_cc]
457 headers = []
458 for label in deps:
459 headers.extend(self.cc_libs[label].headers)
460 headers.extend(self.cc_libs[label].generated_headers)
461 return headers
462
463 def asdl_library(self,
464 asdl_path,
465 deps=None,
466 pretty_print_methods=True,
467 abbrev_module=None):
468
469 deps = deps or []
470
471 # SYSTEM header, _gen/asdl/hnode.asdl.h
472 deps.append('//asdl/hnode.asdl')
473 deps.append('//display/pretty.asdl')
474
475 # to create _gen/mycpp/examples/expr.asdl.h
476 prefix = '_gen/%s' % asdl_path
477
478 out_cc = prefix + '.cc'
479 out_header = prefix + '.h'
480
481 asdl_flags = []
482
483 if pretty_print_methods:
484 outputs = [out_cc, out_header]
485 else:
486 outputs = [out_header]
487 asdl_flags.append('--no-pretty-print-methods')
488
489 if abbrev_module:
490 asdl_flags.append('--abbrev-module=%s' % abbrev_module)
491
492 debug_mod = prefix + '_debug.py'
493 outputs.append(debug_mod)
494
495 # Generating syntax_asdl.h does NOT depend on hnode_asdl.h existing ...
496 self.n.build(outputs,
497 'asdl-cpp', [asdl_path],
498 implicit=['_bin/shwrap/asdl_main'],
499 variables=[
500 ('action', 'cpp'),
501 ('out_prefix', prefix),
502 ('asdl_flags', ' '.join(asdl_flags)),
503 ('debug_mod', debug_mod),
504 ])
505 self.n.newline()
506
507 # ... But COMPILING anything that #includes it does.
508 # Note: assumes there's a build rule for this "system" ASDL schema
509
510 srcs = [out_cc] if pretty_print_methods else []
511 # Define lazy CC library
512 self.cc_library(
513 '//' + asdl_path,
514 srcs=srcs,
515 deps=deps,
516 # For compile_one steps of files that #include this ASDL file
517 generated_headers=[out_header],
518 )
519
520 def py_binary(self, main_py, deps_base_dir='_build/NINJA', template='py'):
521 """Wrapper for Python script with dynamically discovered deps
522
523 Args:
524 template: py, mycpp, or pea
525
526 Example:
527 _bin/shwrap/mycpp_main wraps mycpp/mycpp_main.py
528 - using dependencies from prebuilt/ninja/mycpp.mycpp_main/deps.txt
529 - with the 'shwrap-mycpp' template defined in build/ninja-lib.sh
530 """
531 rel_path, _ = os.path.splitext(main_py)
532 # asdl/asdl_main.py -> asdl.asdl_main
533 py_module = rel_path.replace('/', '.')
534
535 deps_path = os.path.join(deps_base_dir, py_module, 'deps.txt')
536 with open(deps_path) as f:
537 deps = [line.strip() for line in f]
538
539 deps.remove(main_py) # raises ValueError if it's not there
540
541 shwrap_name = os.path.basename(rel_path)
542 self.n.build('_bin/shwrap/%s' % shwrap_name,
543 'write-shwrap', [main_py] + deps,
544 variables=[('template', template)])
545 self.n.newline()
546
547 def souffle_binary(self, souffle_cpp):
548 """
549 Compile souffle C++ into a native executable.
550 """
551 rel_path, _ = os.path.splitext(souffle_cpp)
552 basename = os.path.basename(rel_path)
553
554 souffle_obj = '_build/obj/datalog/%s.o' % basename
555 self.n.build([souffle_obj],
556 'compile_one',
557 souffle_cpp,
558 variables=[('compiler', 'cxx'), ('variant', 'opt'),
559 ('more_cxx_flags', "'-Ivendor -std=c++17'")])
560
561 souffle_bin = '_bin/datalog/%s' % basename
562 self.n.build([souffle_bin],
563 'link',
564 souffle_obj,
565 variables=[('compiler', 'cxx'), ('variant', 'opt'),
566 ('more_link_flags', "'-lstdc++fs'")])
567
568 self.n.newline()