OILS / build / ninja_lib.py View on Github | oilshell.org

555 lines, 342 significant
1#!/usr/bin/env python2
2"""
3ninja_lib.py
4
5Runtime options:
6
7 CXXFLAGS Additional flags to pass to the C++ compiler
8
9Notes on ninja_syntax.py:
10
11- escape_path() seems wrong?
12 - It should really take $ to $$.
13 - It doesn't escape newlines
14
15 return word.replace('$ ', '$$ ').replace(' ', '$ ').replace(':', '$:')
16
17 Ninja shouldn't have used $ and ALSO used shell commands (sh -c)! Better
18 solutions:
19
20 - Spawn a process with environment variables.
21 - use % for substitution instead
22
23- Another problem: Ninja doesn't escape the # comment character like $#, so
24 how can you write a string with a # as the first char on a line?
25"""
26from __future__ import print_function
27
28import collections
29import os
30import sys
31
32
33def log(msg, *args):
34 if args:
35 msg = msg % args
36 print(msg, file=sys.stderr)
37
38
39# Matrix of configurations
40
41COMPILERS_VARIANTS = [
42 ('cxx', 'dbg'),
43 ('cxx', 'opt'),
44 ('cxx', 'asan'),
45 ('cxx', 'asan+gcalways'),
46 ('cxx', 'asan32+gcalways'),
47 ('cxx', 'ubsan'),
48
49 #('clang', 'asan'),
50 ('clang', 'dbg'), # compile-quickly
51 ('clang', 'opt'), # for comparisons
52 ('clang', 'ubsan'), # finds different bugs
53 ('clang', 'coverage'),
54]
55
56GC_PERF_VARIANTS = [
57 ('cxx', 'opt+bumpleak'),
58 ('cxx', 'opt+bumproot'),
59 ('cxx', 'opt+bumpsmall'),
60 ('cxx', 'asan+bumpsmall'),
61 ('cxx', 'opt+nopool'),
62
63 # TODO: should be binary with different files
64 ('cxx', 'opt+tcmalloc'),
65
66 # For tracing allocations, or debugging
67 ('cxx', 'uftrace'),
68
69 # Test performance of 32-bit build. (It uses less memory usage, but can be
70 # slower.)
71 ('cxx', 'opt32'),
72]
73
74OTHER_VARIANTS = [
75 ('cxx', 'opt+bigint'),
76 ('cxx', 'opt+souffle'),
77 ('cxx', 'asan+bigint'),
78]
79
80SMALL_TEST_MATRIX = [
81 ('cxx', 'asan'),
82 ('cxx', 'ubsan'),
83 ('clang', 'coverage'),
84]
85
86
87def ConfigDir(config):
88 compiler, variant, more_cxx_flags = config
89 if more_cxx_flags is None:
90 return '%s-%s' % (compiler, variant)
91 else:
92 # -D CPP_UNIT_TEST -> D_CPP_UNIT_TEST
93 flags_str = more_cxx_flags.replace('-', '').replace(' ', '_')
94 return '%s-%s-%s' % (compiler, variant, flags_str)
95
96
97def ObjPath(src_path, config):
98 rel_path, _ = os.path.splitext(src_path)
99 return '_build/obj/%s/%s.o' % (ConfigDir(config), rel_path)
100
101
102# Used namedtuple since it doesn't have any state
103CcBinary = collections.namedtuple(
104 'CcBinary',
105 'main_cc symlinks implicit deps matrix phony_prefix preprocessed bin_path')
106
107
108class CcLibrary(object):
109 """
110 Life cycle:
111
112 1. A cc_library is first created
113 2. A cc_binary can depend on it
114 - maybe writing rules, and ensuring uniques per configuration
115 3. The link step needs the list of objects
116 4. The tarball needs the list of sources for binary
117 """
118
119 def __init__(self, label, srcs, implicit, deps, headers,
120 generated_headers):
121 self.label = label
122 self.srcs = srcs # queried by SourcesForBinary
123 self.implicit = implicit
124 self.deps = deps
125 self.headers = headers
126 # TODO: asdl() rule should add to this.
127 # Generated headers are different than regular headers. The former need an
128 # implicit dep in Ninja, while the latter can rely on the .d mechanism.
129 self.generated_headers = generated_headers
130
131 self.obj_lookup = {} # config -> list of objects
132 self.preprocessed_lookup = {} # config -> boolean
133
134 def _CalculateImplicit(self, ru):
135 """ Compile actions for cc_library() also need implicit deps on generated headers"""
136
137 out_deps = set()
138 ru._TransitiveClosure(self.label, self.deps, out_deps)
139 unique_deps = sorted(out_deps)
140
141 implicit = list(self.implicit) # copy
142 for label in unique_deps:
143 cc_lib = ru.cc_libs[label]
144 implicit.extend(cc_lib.generated_headers)
145 return implicit
146
147 def MaybeWrite(self, ru, config, preprocessed):
148 if config not in self.obj_lookup: # already written by some other cc_binary()
149 implicit = self._CalculateImplicit(ru)
150
151 objects = []
152 for src in self.srcs:
153 obj = ObjPath(src, config)
154 ru.compile(obj, src, self.deps, config, implicit=implicit)
155 objects.append(obj)
156
157 self.obj_lookup[config] = objects
158
159 if preprocessed and config not in self.preprocessed_lookup:
160 implicit = self._CalculateImplicit(ru)
161
162 for src in self.srcs:
163 # no output needed
164 ru.compile('',
165 src,
166 self.deps,
167 config,
168 implicit=implicit,
169 maybe_preprocess=True)
170 self.preprocessed_lookup[config] = True
171
172
173class Rules(object):
174 """High-level wrapper for NinjaWriter
175
176 What should it handle?
177
178 - The (compiler, variant) matrix loop
179 - Implicit deps for generated code
180 - Phony convenience targets
181
182 Maybe: exporting data to test runner
183
184 Terminology:
185
186 Ninja has
187 - rules, which are like Bazel "actions"
188 - build targets
189
190 Our library has:
191 - Build config: (compiler, variant), and more later
192
193 - Labels: identifiers starting with //, which are higher level than Ninja
194 "targets"
195 cc_library:
196 //mycpp/runtime
197
198 //mycpp/examples/expr.asdl
199 //frontend/syntax.asdl
200
201 - Deps are lists of labels, and have a transitive closure
202
203 - H Rules / High level rules? B rules / Boil?
204 cc_binary, cc_library, asdl, etc.
205 """
206
207 def __init__(self, n):
208 self.n = n # direct ninja writer
209
210 self.cc_bins = [] # list of CcBinary() objects to write
211 self.cc_libs = {} # label -> CcLibrary object
212 self.cc_binary_deps = {} # main_cc -> list of LABELS
213 self.phony = {} # list of phony targets
214
215 def AddPhony(self, phony_to_add):
216 self.phony.update(phony_to_add)
217
218 def WritePhony(self):
219 for name in sorted(self.phony):
220 targets = self.phony[name]
221 if targets:
222 self.n.build([name], 'phony', targets)
223 self.n.newline()
224
225 def WriteRules(self):
226 for cc_bin in self.cc_bins:
227 self.WriteCcBinary(cc_bin)
228
229 def compile(self,
230 out_obj,
231 in_cc,
232 deps,
233 config,
234 implicit=None,
235 maybe_preprocess=False):
236 """ .cc -> compiler -> .o """
237
238 implicit = implicit or []
239
240 compiler, variant, more_cxx_flags = config
241 if more_cxx_flags is None:
242 flags_str = "''"
243 else:
244 assert "'" not in more_cxx_flags, more_cxx_flags # can't handle single quotes
245 flags_str = "'%s'" % more_cxx_flags
246
247 v = [('compiler', compiler), ('variant', variant),
248 ('more_cxx_flags', flags_str)]
249 if maybe_preprocess:
250 # Limit it to certain configs
251 if more_cxx_flags is None and variant in ('dbg', 'opt'):
252 pre = '_build/preprocessed/%s-%s/%s' % (compiler, variant,
253 in_cc)
254 self.n.build(pre,
255 'preprocess', [in_cc],
256 implicit=implicit,
257 variables=v)
258 else:
259 self.n.build([out_obj],
260 'compile_one', [in_cc],
261 implicit=implicit,
262 variables=v)
263
264 self.n.newline()
265
266 def link(self, out_bin, main_obj, deps, config):
267 """ list of .o -> linker -> executable, along with stripped version """
268 compiler, variant, _ = config
269
270 assert isinstance(out_bin, str), out_bin
271 assert isinstance(main_obj, str), main_obj
272
273 objects = [main_obj]
274 for label in deps:
275 key = (label, compiler, variant)
276 try:
277 cc_lib = self.cc_libs[label]
278 except KeyError:
279 raise RuntimeError("Couldn't resolve label %r" % label)
280
281 o = cc_lib.obj_lookup[config]
282 objects.extend(o)
283
284 v = [('compiler', compiler), ('variant', variant),
285 ('more_link_flags', "''")]
286 self.n.build([out_bin], 'link', objects, variables=v)
287 self.n.newline()
288
289 # Strip any .opt binaries
290 if variant.startswith('opt') or variant.startswith('opt32'):
291 stripped = out_bin + '.stripped'
292 symbols = out_bin + '.symbols'
293 self.n.build([stripped, symbols], 'strip', [out_bin])
294 self.n.newline()
295
296 def comment(self, s):
297 self.n.comment(s)
298 self.n.newline()
299
300 def cc_library(
301 self,
302 label,
303 srcs=None,
304 implicit=None,
305 deps=None,
306 # note: headers is only used for tarball manifest, not compiler command line
307 headers=None,
308 generated_headers=None):
309
310 # srcs = [] is allowed for _gen/asdl/hnode.asdl.h
311 if srcs is None:
312 raise RuntimeError('cc_library %r requires srcs' % label)
313
314 implicit = implicit or []
315 deps = deps or []
316 headers = headers or []
317 generated_headers = generated_headers or []
318
319 if label in self.cc_libs:
320 raise RuntimeError('%s was already defined' % label)
321
322 self.cc_libs[label] = CcLibrary(label, srcs, implicit, deps, headers,
323 generated_headers)
324
325 def _TransitiveClosure(self, name, deps, unique_out):
326 """
327 Args:
328 name: for error messages
329 """
330 for label in deps:
331 if label in unique_out:
332 continue
333 unique_out.add(label)
334
335 try:
336 cc_lib = self.cc_libs[label]
337 except KeyError:
338 raise RuntimeError('Undefined label %s in %s' % (label, name))
339
340 self._TransitiveClosure(cc_lib.label, cc_lib.deps, unique_out)
341
342 def cc_binary(
343 self,
344 main_cc,
345 symlinks=None,
346 implicit=None, # for COMPILE action, not link action
347 deps=None,
348 matrix=None, # $compiler $variant
349 phony_prefix=None,
350 preprocessed=False,
351 bin_path=None, # default is _bin/$compiler-$variant/rel/path
352 ):
353 symlinks = symlinks or []
354 implicit = implicit or []
355 deps = deps or []
356 if not matrix:
357 raise RuntimeError("Config matrix required")
358
359 cc_bin = CcBinary(main_cc, symlinks, implicit, deps, matrix,
360 phony_prefix, preprocessed, bin_path)
361
362 self.cc_bins.append(cc_bin)
363
364 def WriteCcBinary(self, cc_bin):
365 c = cc_bin
366
367 out_deps = set()
368 self._TransitiveClosure(c.main_cc, c.deps, out_deps)
369 unique_deps = sorted(out_deps)
370
371 # save for SourcesForBinary()
372 self.cc_binary_deps[c.main_cc] = unique_deps
373
374 compile_imp = list(c.implicit)
375 for label in unique_deps:
376 cc_lib = self.cc_libs[label] # should exit
377 # compile actions of binaries that have ASDL label deps need the
378 # generated header as implicit dep
379 compile_imp.extend(cc_lib.generated_headers)
380
381 for config in c.matrix:
382 if len(config) == 2:
383 config = (config[0], config[1], None)
384
385 for label in unique_deps:
386 cc_lib = self.cc_libs[label] # should exit
387
388 cc_lib.MaybeWrite(self, config, c.preprocessed)
389
390 # Compile main object, maybe with IMPLICIT headers deps
391 main_obj = ObjPath(c.main_cc, config)
392 self.compile(main_obj,
393 c.main_cc,
394 c.deps,
395 config,
396 implicit=compile_imp)
397 if c.preprocessed:
398 self.compile('',
399 c.main_cc,
400 c.deps,
401 config,
402 implicit=compile_imp,
403 maybe_preprocess=True)
404
405 config_dir = ConfigDir(config)
406 bin_dir = '_bin/%s' % config_dir
407
408 if c.bin_path:
409 # e.g. _bin/cxx-dbg/oils_for_unix
410 bin_ = '%s/%s' % (bin_dir, c.bin_path)
411 bin_subdir, _, bin_name = c.bin_path.rpartition('/')
412 if bin_subdir:
413 bin_dir = '%s/%s' % (bin_dir, bin_subdir)
414 else:
415 bin_name = c.bin_path
416
417 else:
418 # e.g. _gen/mycpp/examples/classes.mycpp
419 rel_path, _ = os.path.splitext(c.main_cc)
420
421 # Put binary in _bin/cxx-dbg/mycpp/examples, not _bin/cxx-dbg/_gen/mycpp/examples
422 if rel_path.startswith('_gen/'):
423 rel_path = rel_path[len('_gen/'):]
424
425 bin_ = '%s/%s' % (bin_dir, rel_path)
426
427 # Link with OBJECT deps
428 self.link(bin_, main_obj, unique_deps, config)
429
430 # Make symlinks
431 for symlink in c.symlinks:
432 # Must explicitly specify bin_path to have a symlink, for now
433 assert c.bin_path is not None
434 self.n.build(['%s/%s' % (bin_dir, symlink)],
435 'symlink', [bin_],
436 variables=[('dir', bin_dir), ('target', bin_name),
437 ('new', symlink)])
438 self.n.newline()
439
440 if c.phony_prefix:
441 key = '%s-%s' % (c.phony_prefix, config_dir)
442 if key not in self.phony:
443 self.phony[key] = []
444 self.phony[key].append(bin_)
445
446 def SourcesForBinary(self, main_cc):
447 """
448 Used for preprocessed metrics, release tarball, _build/oils.sh, etc.
449 """
450 deps = self.cc_binary_deps[main_cc]
451 sources = [main_cc]
452 for label in deps:
453 sources.extend(self.cc_libs[label].srcs)
454 return sources
455
456 def HeadersForBinary(self, main_cc):
457 deps = self.cc_binary_deps[main_cc]
458 headers = []
459 for label in deps:
460 headers.extend(self.cc_libs[label].headers)
461 headers.extend(self.cc_libs[label].generated_headers)
462 return headers
463
464 def asdl_library(self, asdl_path, deps=None, pretty_print_methods=True):
465
466 deps = deps or []
467
468 # SYSTEM header, _gen/asdl/hnode.asdl.h
469 deps.append('//asdl/hnode.asdl')
470 deps.append('//display/pretty.asdl')
471
472 # to create _gen/mycpp/examples/expr.asdl.h
473 prefix = '_gen/%s' % asdl_path
474
475 out_cc = prefix + '.cc'
476 out_header = prefix + '.h'
477
478 asdl_flags = ''
479
480 if pretty_print_methods:
481 outputs = [out_cc, out_header]
482 else:
483 outputs = [out_header]
484 asdl_flags += '--no-pretty-print-methods'
485
486 debug_mod = prefix + '_debug.py'
487 outputs.append(debug_mod)
488
489 # Generating syntax_asdl.h does NOT depend on hnode_asdl.h existing ...
490 self.n.build(outputs,
491 'asdl-cpp', [asdl_path],
492 implicit=['_bin/shwrap/asdl_main'],
493 variables=[
494 ('action', 'cpp'),
495 ('out_prefix', prefix),
496 ('asdl_flags', asdl_flags),
497 ('debug_mod', debug_mod),
498 ])
499 self.n.newline()
500
501 # ... But COMPILING anything that #includes it does.
502 # Note: assumes there's a build rule for this "system" ASDL schema
503
504 srcs = [out_cc] if pretty_print_methods else []
505 # Define lazy CC library
506 self.cc_library(
507 '//' + asdl_path,
508 srcs=srcs,
509 deps=deps,
510 # For compile_one steps of files that #include this ASDL file
511 generated_headers=[out_header],
512 )
513
514 def py_binary(self, main_py, deps_base_dir='_build/NINJA', template='py'):
515 """
516 Wrapper for Python script with dynamically discovered deps
517 """
518 rel_path, _ = os.path.splitext(main_py)
519 py_module = rel_path.replace(
520 '/', '.') # asdl/asdl_main.py -> asdl.asdl_main
521
522 deps_path = os.path.join(deps_base_dir, py_module, 'deps.txt')
523 with open(deps_path) as f:
524 deps = [line.strip() for line in f]
525
526 deps.remove(main_py) # raises ValueError if it's not there
527
528 basename = os.path.basename(rel_path)
529 self.n.build('_bin/shwrap/%s' % basename,
530 'write-shwrap', [main_py] + deps,
531 variables=[('template', template)])
532 self.n.newline()
533
534 def souffle_binary(self, souffle_cpp):
535 """
536 Compile souffle C++ into a native executable.
537 """
538 rel_path, _ = os.path.splitext(souffle_cpp)
539 basename = os.path.basename(rel_path)
540
541 souffle_obj = '_build/obj/datalog/%s.o' % basename
542 self.n.build([souffle_obj],
543 'compile_one',
544 souffle_cpp,
545 variables=[('compiler', 'cxx'), ('variant', 'opt'),
546 ('more_cxx_flags', "'-Ivendor -std=c++17'")])
547
548 souffle_bin = '_bin/datalog/%s' % basename
549 self.n.build([souffle_bin],
550 'link',
551 souffle_obj,
552 variables=[('compiler', 'cxx'), ('variant', 'opt'),
553 ('more_link_flags', "'-lstdc++fs'")])
554
555 self.n.newline()