test/sh_spec.py

OILS / test / sh_spec.py View on Github | oils.pub

1668 lines, 1042 significant

1	#!/usr/bin/env python2
2	from __future__ import print_function
3	"""
4	sh_spec.py -- Test framework to compare shells.
5
6	Assertion help:
7	stdout: A single line of expected stdout. Newline is implicit.
8	stdout-json: JSON-encoded string. Use for the empty string (no newline),
9	for unicode chars, etc.
10
11	stderr: Ditto for stderr stream.
12	status: Expected shell return code. If not specified, the case must exit 0.
13
14	Results:
15	PASS - we got the ideal, expected value
16	OK - we got a value that was not ideal, but expected
17	For OSH this is behavior that was defined to be different?
18	N-I - Not implemented (e.g. $''). Assertions still checked (in case it
19	starts working)
20	BUG - we verified the value of a known bug
21	FAIL - we got an unexpected value. If the implementation can't be changed,
22	it should be converted to BUG or OK. Otherwise it should be made to
23	PASS.
24
25	NOTE: The difference between OK and BUG is a matter of judgement. If the ideal
26	behavior is a compile time error (code 2), a runtime error is generally OK.
27
28	If ALL shells agree on a broken behavior, they are all marked OK (but our
29	implementation will be PASS.) But if the behavior is NOT POSIX compliant, then
30	it will be a BUG.
31
32	If one shell disagrees with others, that is generally a BUG.
33
34	Example test case:
35
36	#### hello and fail
37	echo hello
38	echo world
39	exit 1
40	## status: 1
41	#
42	# ignored comment
43	#
44	## STDOUT:
45	hello
46	world
47	## END
48
49	"""
50
51	import collections
52	import cgi
53	import cStringIO
54	import errno
55	import json
56	import optparse
57	import os
58	import pprint
59	import re
60	import shutil
61	import signal
62	import subprocess
63	import sys
64
65	from test import spec_lib
66	from doctools import html_head
67
68	log = spec_lib.log
69
70	# Magic strings for other variants of OSH.
71
72	# NOTE: osh_ALT is usually _bin/osh -- the release binary.
73	# It would be better to rename these osh-cpython and osh-ovm. Have the concept
74	# of a suffix?
75
76	OSH_CPYTHON = ('osh', 'osh-dbg')
77	OTHER_OSH = ('osh_ALT', )
78
79	YSH_CPYTHON = ('ysh', 'ysh-dbg')
80	OTHER_YSH = ('oil_ALT', )
81
82	# For now, only count the Oils CPython failures. TODO: the spec-cpp job should
83	# assert the osh-cpp and ysh-cpp deltas.
84	OTHER_OILS = OTHER_OSH + OTHER_YSH + ('osh-cpp', 'ysh-cpp')
85
86
87	class ParseError(Exception):
88	pass
89
90
91	# EXAMPLES:
92	## stdout: foo
93	## stdout-json: ""
94	#
95	# In other words, it could be (name, value) or (qualifier, name, value)
96
97	KEY_VALUE_RE = re.compile(
98	r'''
99	[#][#] \s+
100	# optional prefix with qualifier and shells
101	(?: (OK(?:-\d)? \| BUG(?:-\d)? \| N-I) \s+ ([\w+/]+) \s+ )?
102	([\w\-]+) # key
103	:
104	\s* (.*) # value
105	''', re.VERBOSE)
106
107	END_MULTILINE_RE = re.compile(r'''
108	[#][#] \s+ END
109	''', re.VERBOSE)
110
111	# Line types
112	TEST_CASE_BEGIN = 0 # Starts with ####
113	KEY_VALUE = 1 # Metadata
114	KEY_VALUE_MULTILINE = 2 # STDOUT STDERR
115	END_MULTILINE = 3 # STDOUT STDERR
116	PLAIN_LINE = 4 # Uncommented
117	EOF = 5
118
119	LEX_OUTER = 0 # Ignore blank lines, e.g. for separating cases
120	LEX_RAW = 1 # Blank lines are significant
121
122
123	class Tokenizer(object):
124	"""Modal lexer!"""
125
126	def __init__(self, f):
127	self.f = f
128
129	self.cursor = None
130	self.line_num = 0
131
132	self.next()
133
134	def _ClassifyLine(self, line, lex_mode):
135	if not line: # empty
136	return self.line_num, EOF, ''
137
138	if lex_mode == LEX_OUTER and not line.strip():
139	return None
140
141	if line.startswith('####'):
142	desc = line[4:].strip()
143	return self.line_num, TEST_CASE_BEGIN, desc
144
145	m = KEY_VALUE_RE.match(line)
146	if m:
147	qualifier, shells, name, value = m.groups()
148	# HACK: Expected data should have the newline.
149	if name in ('stdout', 'stderr'):
150	value += '\n'
151
152	if name in ('STDOUT', 'STDERR'):
153	token_type = KEY_VALUE_MULTILINE
154	else:
155	token_type = KEY_VALUE
156	return self.line_num, token_type, (qualifier, shells, name, value)
157
158	m = END_MULTILINE_RE.match(line)
159	if m:
160	return self.line_num, END_MULTILINE, None
161
162	# If it starts with ##, it should be metadata. This finds some typos.
163	if line.startswith('##'):
164	raise RuntimeError('Invalid ## line %r' % line)
165
166	# TODO: comments should not be ignored in code or STDOUT blocks!
167	if line.lstrip().startswith('#'): # Ignore comments
168	return None # try again
169
170	# Non-empty line that doesn't start with '#'
171	# NOTE: We need the original line to test the whitespace sensitive <<-.
172	# And we need rstrip because we add newlines back below.
173	return self.line_num, PLAIN_LINE, line
174
175	def next(self, lex_mode=LEX_OUTER):
176	"""Raises StopIteration when exhausted."""
177	while True:
178	line = self.f.readline()
179	self.line_num += 1
180
181	tok = self._ClassifyLine(line, lex_mode)
182	if tok is not None:
183	break
184
185	self.cursor = tok
186	return self.cursor
187
188	def peek(self):
189	return self.cursor
190
191
192	def AddMetadataToCase(case, qualifier, shells, name, value, line_num):
193	shells = shells.split('/') # bash/dash/mksh
194	for shell in shells:
195	if shell not in case:
196	case[shell] = {}
197
198	# Check a duplicate specification
199	name_without_type = re.sub(r'-json$', '', name)
200	if (name_without_type in case[shell] or
201	name_without_type + '-json' in case[shell]):
202	raise ParseError('Line %d: duplicate spec %r for %r' %
203	(line_num, name, shell))
204
205	# Check inconsistent qualifier
206	if 'qualifier' in case[shell] and qualifier != case[shell]['qualifier']:
207	raise ParseError(
208	'Line %d: inconsistent qualifier %r is specified for %r, '
209	'but %r was previously specified. ' %
210	(line_num, qualifier, shell, case[shell]['qualifier']))
211
212	case[shell][name] = value
213	case[shell]['qualifier'] = qualifier
214
215
216	# Format of a test script.
217	#
218	# -- Code is either literal lines, or a commented out code: value.
219	# code = PLAIN_LINE*
220	# \| '## code:' VALUE
221	#
222	# -- Key value pairs can be single- or multi-line
223	# key_value = '##' KEY ':' VALUE
224	# \| KEY_VALUE_MULTILINE PLAIN_LINE* END_MULTILINE
225	#
226	# -- Description, then key-value pairs surrounding code.
227	# test_case = '####' DESC
228	# key_value*
229	# code
230	# key_value*
231	#
232	# -- Should be a blank line after each test case. Leading comments and code
233	# -- are OK.
234	#
235	# test_file =
236	# key_value* -- file level metadata
237	# (test_case '\n')*
238
239
240	def ParseKeyValue(tokens, case):
241	"""Parse commented-out metadata in a test case.
242
243	The metadata must be contiguous.
244
245	Args:
246	tokens: Tokenizer
247	case: dictionary to add to
248	"""
249	while True:
250	line_num, kind, item = tokens.peek()
251
252	if kind == KEY_VALUE_MULTILINE:
253	qualifier, shells, name, empty_value = item
254	if empty_value:
255	raise ParseError(
256	'Line %d: got value %r for %r, but the value should be on the '
257	'following lines' % (line_num, empty_value, name))
258
259	value_lines = []
260	while True:
261	tokens.next(lex_mode=LEX_RAW) # empty lines aren't skipped
262	_, kind2, item2 = tokens.peek()
263	if kind2 != PLAIN_LINE:
264	break
265	value_lines.append(item2)
266
267	value = ''.join(value_lines)
268
269	name = name.lower() # STDOUT -> stdout
270	if qualifier:
271	AddMetadataToCase(case, qualifier, shells, name, value,
272	line_num)
273	else:
274	case[name] = value
275
276	# END token is optional.
277	if kind2 == END_MULTILINE:
278	tokens.next()
279
280	elif kind == KEY_VALUE:
281	qualifier, shells, name, value = item
282
283	if qualifier:
284	AddMetadataToCase(case, qualifier, shells, name, value,
285	line_num)
286	else:
287	case[name] = value
288
289	tokens.next()
290
291	else: # Unknown token type
292	break
293
294
295	def ParseCodeLines(tokens, case):
296	"""Parse uncommented code in a test case."""
297	_, kind, item = tokens.peek()
298	if kind != PLAIN_LINE:
299	raise ParseError('Expected a line of code (got %r, %r)' % (kind, item))
300	code_lines = []
301	while True:
302	_, kind, item = tokens.peek()
303	if kind != PLAIN_LINE:
304	case['code'] = ''.join(code_lines)
305	return
306	code_lines.append(item)
307	tokens.next(lex_mode=LEX_RAW)
308
309
310	def ParseTestCase(tokens):
311	"""Parse a single test case and return it.
312
313	If at EOF, return None.
314	"""
315	line_num, kind, item = tokens.peek()
316	if kind == EOF:
317	return None
318
319	if kind != TEST_CASE_BEGIN:
320	raise RuntimeError("line %d: Expected TEST_CASE_BEGIN, got %r" %
321	(line_num, [kind, item]))
322
323	tokens.next()
324
325	case = {'desc': item, 'line_num': line_num}
326
327	ParseKeyValue(tokens, case)
328
329	# For broken code
330	if 'code' in case: # Got it through a key value pair
331	return case
332
333	ParseCodeLines(tokens, case)
334	ParseKeyValue(tokens, case)
335
336	return case
337
338
339	_META_FIELDS = [
340	'our_shell',
341	'compare_shells',
342	'suite',
343	'tags',
344	'oils_failures_allowed',
345	'oils_cpp_failures_allowed',
346	'legacy_tmp_dir',
347	]
348
349
350	def ParseTestFile(test_file, tokens):
351	"""
352	test_file: Only for error message
353	"""
354	file_metadata = {}
355	test_cases = []
356
357	try:
358	# Skip over the header. Setup code can go here, although would we have to
359	# execute it on every case?
360	while True:
361	line_num, kind, item = tokens.peek()
362	if kind != KEY_VALUE:
363	break
364
365	qualifier, shells, name, value = item
366	if qualifier is not None:
367	raise RuntimeError('Invalid qualifier in spec file metadata')
368	if shells is not None:
369	raise RuntimeError('Invalid shells in spec file metadata')
370
371	file_metadata[name] = value
372
373	tokens.next()
374
375	while True: # Loop over cases
376	test_case = ParseTestCase(tokens)
377	if test_case is None:
378	break
379	test_cases.append(test_case)
380
381	except StopIteration:
382	raise RuntimeError('Unexpected EOF parsing test cases')
383
384	for name in file_metadata:
385	if name not in _META_FIELDS:
386	raise RuntimeError('Invalid file metadata %r in %r' %
387	(name, test_file))
388
389	return file_metadata, test_cases
390
391
392	def CreateStringAssertion(d, key, assertions, qualifier=False):
393	found = False
394
395	exp = d.get(key)
396	if exp is not None:
397	a = EqualAssertion(key, exp, qualifier=qualifier)
398	assertions.append(a)
399	found = True
400
401	exp_json = d.get(key + '-json')
402	if exp_json is not None:
403	exp = json.loads(exp_json, encoding='utf-8')
404	a = EqualAssertion(key, exp, qualifier=qualifier)
405	assertions.append(a)
406	found = True
407
408	return found
409
410
411	def CreateIntAssertion(d, key, assertions, qualifier=False):
412	exp = d.get(key) # expected
413	if exp is not None:
414	# For now, turn it into int
415	a = EqualAssertion(key, int(exp), qualifier=qualifier)
416	assertions.append(a)
417	return True
418	return False
419
420
421	def CreateAssertions(case, sh_label):
422	"""Given a raw test case and a shell label, create EqualAssertion instances
423	to run."""
424	assertions = []
425
426	# Whether we found assertions
427	stdout = False
428	stderr = False
429	status = False
430
431	# So the assertion are exactly the same for osh and osh_ALT
432
433	if sh_label.startswith('osh'):
434	case_sh = 'osh'
435	elif sh_label.startswith('bash'):
436	case_sh = 'bash'
437	else:
438	case_sh = sh_label
439
440	if case_sh in case:
441	q = case[case_sh]['qualifier']
442	if CreateStringAssertion(case[case_sh],
443	'stdout',
444	assertions,
445	qualifier=q):
446	stdout = True
447	if CreateStringAssertion(case[case_sh],
448	'stderr',
449	assertions,
450	qualifier=q):
451	stderr = True
452	if CreateIntAssertion(case[case_sh], 'status', assertions,
453	qualifier=q):
454	status = True
455
456	if not stdout:
457	CreateStringAssertion(case, 'stdout', assertions)
458	if not stderr:
459	CreateStringAssertion(case, 'stderr', assertions)
460	if not status:
461	if 'status' in case:
462	CreateIntAssertion(case, 'status', assertions)
463	else:
464	# If the user didn't specify a 'status' assertion, assert that the exit
465	# code is 0.
466	a = EqualAssertion('status', 0)
467	assertions.append(a)
468
469	no_traceback = SubstringAssertion('stderr', 'Traceback (most recent')
470	assertions.append(no_traceback)
471
472	#print 'SHELL', shell
473	#pprint.pprint(case)
474	#print(assertions)
475	return assertions
476
477
478	class Result(object):
479	"""Result of an stdout/stderr/status assertion or of a (case, shell) cell.
480
481	Order is important: the result of a cell is the minimum of the results of
482	each assertion.
483	"""
484	TIMEOUT = 0 # ONLY a cell result, not an assertion result
485	FAIL = 1
486	BUG = 2
487	BUG_2 = 3
488	NI = 4
489	OK = 5
490	OK_2 = 6
491	OK_3 = 7
492	OK_4 = 8
493	PASS = 9
494
495	length = 10 # for loops
496
497
498	def QualifierToResult(qualifier):
499	# type: (str) -> Result
500	if qualifier == 'BUG': # equal, but known bad
501	return Result.BUG
502	if qualifier == 'BUG-2':
503	return Result.BUG_2
504
505	if qualifier == 'N-I': # equal, and known UNIMPLEMENTED
506	return Result.NI
507
508	if qualifier == 'OK': # equal, but ok (not ideal)
509	return Result.OK
510	if qualifier == 'OK-2':
511	return Result.OK_2
512	if qualifier == 'OK-3':
513	return Result.OK_3
514	if qualifier == 'OK-4':
515	return Result.OK_4
516
517	return Result.PASS # ideal behavior
518
519
520	class EqualAssertion(object):
521	"""Check that two values are equal."""
522
523	def __init__(self, key, expected, qualifier=None):
524	self.key = key
525	self.expected = expected # expected value
526	self.qualifier = qualifier # whether this was a special case?
527
528	def __repr__(self):
529	return '<EqualAssertion %s == %r>' % (self.key, self.expected)
530
531	def Check(self, shell, record):
532	actual = record[self.key]
533	if actual != self.expected:
534	msg = '''
535	[%s %s]
536	Expected %r
537	Got %r
538	''' % (shell, self.key, self.expected, actual)
539
540	# TODO: Make this better and add a flag for it.
541	if 0:
542	import difflib
543	for line in difflib.unified_diff(self.expected,
544	actual,
545	fromfile='expected',
546	tofile='actual'):
547	print(repr(line))
548
549	return Result.FAIL, msg
550
551	return QualifierToResult(self.qualifier), ''
552
553
554	class SubstringAssertion(object):
555	"""Check that a string like stderr doesn't have a substring."""
556
557	def __init__(self, key, substring):
558	self.key = key
559	self.substring = substring
560
561	def __repr__(self):
562	return '<SubstringAssertion %s == %r>' % (self.key, self.substring)
563
564	def Check(self, shell, record):
565	actual = record[self.key]
566	if self.substring in actual:
567	msg = '[%s %s] Found %r' % (shell, self.key, self.substring)
568	return Result.FAIL, msg
569	return Result.PASS, ''
570
571
572	class Stats(object):
573
574	def __init__(self, num_cases, sh_labels):
575	self.counters = collections.defaultdict(int)
576	c = self.counters
577	c['num_cases'] = num_cases
578	c['oils_num_passed'] = 0
579	c['oils_num_failed'] = 0
580	c['oils_cpp_num_failed'] = 0
581	# Number of osh_ALT results that differed from osh.
582	c['oils_ALT_delta'] = 0
583
584	self.by_shell = {}
585	for sh in sh_labels:
586	self.by_shell[sh] = collections.defaultdict(int)
587	self.nonzero_results = collections.defaultdict(int)
588
589	self.tsv_rows = []
590
591	def Inc(self, counter_name):
592	self.counters[counter_name] += 1
593
594	def Get(self, counter_name):
595	return self.counters[counter_name]
596
597	def Set(self, counter_name, val):
598	self.counters[counter_name] = val
599
600	def ReportCell(self, case_num, cell_result, sh_label):
601	self.tsv_rows.append(
602	(str(case_num), sh_label, TEXT_CELLS[cell_result]))
603
604	self.by_shell[sh_label][cell_result] += 1
605	self.nonzero_results[cell_result] += 1
606
607	c = self.counters
608	if cell_result == Result.TIMEOUT:
609	c['num_timeout'] += 1
610	elif cell_result == Result.FAIL:
611	# Special logic: don't count osh_ALT because its failures will be
612	# counted in the delta.
613	if sh_label not in OTHER_OILS:
614	c['num_failed'] += 1
615
616	if sh_label in OSH_CPYTHON + YSH_CPYTHON:
617	c['oils_num_failed'] += 1
618
619	if sh_label in ('osh-cpp', 'ysh-cpp'):
620	c['oils_cpp_num_failed'] += 1
621	elif cell_result in (Result.BUG, Result.BUG_2):
622	c['num_bug'] += 1
623	elif cell_result == Result.NI:
624	c['num_ni'] += 1
625	elif cell_result in (Result.OK, Result.OK_2, Result.OK_3, Result.OK_4):
626	c['num_ok'] += 1
627	elif cell_result == Result.PASS:
628	c['num_passed'] += 1
629	if sh_label in OSH_CPYTHON + YSH_CPYTHON:
630	c['oils_num_passed'] += 1
631	else:
632	raise AssertionError()
633
634	def WriteTsv(self, f):
635	f.write('case\tshell\tresult\n')
636	for row in self.tsv_rows:
637	f.write('\t'.join(row))
638	f.write('\n')
639
640
641	PIPE = subprocess.PIPE
642
643
644	def _TimedOut(status):
645	# timeout 1s sleep 5 ==> status 124
646	# timeout -s KILL 1s sleep 5 ==> status 137
647	# the latter is more robust
648	#
649	# SIGKILL == -9
650	#return status in (124, 137)
651	return status == -9
652
653
654	def _PrepareCaseTempDir(case_tmp_dir, legacy_tmp_dir=False):
655	# Clean up after the previous run. (The previous run doesn't clean
656	# up after itself, because it's useful to manually inspect the
657	# state.)
658	try:
659	shutil.rmtree(case_tmp_dir)
660	except OSError as e:
661	#log('Error cleaning up %r: %s', case_tmp_dir, e)
662	pass
663
664	try:
665	os.makedirs(case_tmp_dir)
666	except OSError as e:
667	if e.errno != errno.EEXIST:
668	raise
669
670	# Some tests assume _tmp exists
671	if legacy_tmp_dir:
672	try:
673	os.mkdir(os.path.join(case_tmp_dir, '_tmp'))
674	except OSError as e:
675	if e.errno != errno.EEXIST:
676	raise
677
678
679	class ActualOutput(object):
680
681	def __init__(self, f):
682	self.f = f
683
684	def _Write(self, msg, *args):
685	if not self.f:
686	return
687
688	if args:
689	msg = msg % args
690
691	self.f.write(msg)
692	self.f.write('\n')
693
694	def WriteActualRow(self, record):
695	"""
696	"""
697	# Check for different stdout:
698	unique = {}
699	for a in record['actual']:
700	# Don't count stdout for features not implemented
701	if a['cell_result'] == Result.NI:
702	continue
703	# look for existing shells
704	if a['sh_label'] == 'osh':
705	continue
706
707	stdout = a['stdout']
708	if stdout not in unique:
709	unique[stdout] = []
710	unique[stdout].append(a['sh_label'])
711
712	# 4 is more interesting than 3
713	if len(unique) >= 4:
714	self._Write('---')
715	self._Write('TEST CASE %d' % record['case_num'])
716	self._Write('DESC %s' % record['desc'])
717	self._Write('LINE %d' % record['line_num'])
718	self._Write('NUM UNIQUE %d' % len(unique))
719	self._Write('')
720	i = 0
721	for stdout, shells in unique.iteritems():
722	self._Write('UNIQUE STDOUT %d', (i + 1))
723	self._Write('SHELLS: %s', ' '.join(shells))
724	shells = unique[stdout]
725
726	self._Write('')
727	self._Write(stdout)
728	self._Write('')
729	i += 1
730
731	#json.dump(record, sys.stderr, indent=2)
732
733
734	def ResetSignals():
735	"""
736	Added for spec/builtin-trap.sh
737
738	Python 3 doesn't have this issue, it has restore_signals=True
739	https://docs.python.org/3/library/subprocess.html
740
741	But Python 2 does
742	"""
743	signal.signal(signal.SIGPIPE, signal.SIG_DFL)
744	signal.signal(signal.SIGXFSZ, signal.SIG_DFL)
745
746
747	def RunCases(cases,
748	case_predicate,
749	shells,
750	env,
751	out,
752	actual_out,
753	opts,
754	legacy_tmp_dir=False):
755	"""
756	Run a list of test 'cases' for all 'shells' and write output to 'out'.
757	"""
758	if opts.trace:
759	for _, sh in shells:
760	log('\tshell: %s', sh)
761	print('\twhich $SH: ', end='', file=sys.stderr)
762	subprocess.call(['which', sh])
763
764	#pprint.pprint(cases)
765
766	if (isinstance(case_predicate, spec_lib.RangePredicate) and
767	(case_predicate.begin > (len(cases) - 1))):
768	raise RuntimeError(
769	"valid case indexes are from 0 to %s. given range: %s-%s" %
770	((len(cases) - 1), case_predicate.begin, case_predicate.end))
771
772	sh_labels = [sh_label for sh_label, _ in shells]
773
774	out.WriteHeader(sh_labels)
775	stats = Stats(len(cases), sh_labels)
776
777	# Make an environment for each shell. $SH is the path to the shell, so we
778	# can test flags, etc.
779	sh_env = []
780	for _, sh_path in shells:
781	e = dict(env)
782	e[opts.sh_env_var_name] = sh_path
783	sh_env.append(e)
784
785	# Determine which one (if any) is osh-cpython, for comparison against other
786	# shells.
787	osh_cpython_index = -1
788	for i, (sh_label, _) in enumerate(shells):
789	if sh_label in OSH_CPYTHON:
790	osh_cpython_index = i
791	break
792
793	timeout_dir = os.path.abspath('_tmp/spec/timeouts')
794	try:
795	shutil.rmtree(timeout_dir)
796	os.mkdir(timeout_dir)
797	except OSError:
798	pass
799
800	# Now run each case, and print a table.
801	for i, case in enumerate(cases):
802	case['case_num'] = i # for reporting
803
804	line_num = case['line_num']
805	desc = case['desc']
806	code = case['code']
807
808	if opts.trace:
809	log('case %d: %s', i, desc)
810
811	if not case_predicate(i, case):
812	stats.Inc('num_skipped')
813	continue
814
815	if opts.do_print:
816	print('#### %s' % case['desc'])
817	print(case['code'])
818	print()
819	continue
820
821	stats.Inc('num_cases_run')
822
823	result_row = []
824
825	case_actual = dict(case)
826	case_actual['actual'] = []
827
828	for shell_index, (sh_label, sh_path) in enumerate(shells):
829	timeout_file = os.path.join(timeout_dir, '%02d-%s' % (i, sh_label))
830	if opts.timeout:
831	if opts.timeout_bin:
832	# This is what smoosh itself uses. See smoosh/tests/shell_tests.sh
833	# QUIRK: interval can only be a whole number
834	argv = [
835	opts.timeout_bin,
836	'-t',
837	opts.timeout,
838	# Somehow I'm not able to get this timeout file working? I think
839	# it has a bug when using stdin. It waits for the background
840	# process too.
841
842	#'-i', '1',
843	#'-l', timeout_file
844	]
845	else:
846	# This kills hanging tests properly, but somehow they fail with code
847	# -9?
848	argv = ['timeout', '-s', 'KILL', opts.timeout + 's']
849
850	# s suffix for seconds
851	#argv = ['timeout', opts.timeout + 's']
852	else:
853	argv = []
854	argv.append(sh_path)
855
856	# dash doesn't support -o posix
857	if opts.posix and sh_label != 'dash':
858	argv.extend(['-o', 'posix'])
859
860	if opts.trace:
861	log('\targv: %s', ' '.join(argv))
862
863	case_env = sh_env[shell_index]
864
865	# Unique dir for every test case and shell
866	tmp_base = os.path.normpath(opts.tmp_env) # no . or ..
867	case_tmp_dir = os.path.join(tmp_base, '%02d-%s' % (i, sh_label))
868
869	_PrepareCaseTempDir(case_tmp_dir, legacy_tmp_dir=legacy_tmp_dir)
870
871	case_env['TMP'] = case_tmp_dir
872
873	if opts.pyann_out_dir:
874	case_env = dict(case_env)
875	case_env['PYANN_OUT'] = os.path.join(opts.pyann_out_dir,
876	'%d.json' % i)
877
878	try:
879	p = subprocess.Popen(argv,
880	env=case_env,
881	cwd=case_tmp_dir,
882	stdin=PIPE,
883	stdout=PIPE,
884	stderr=PIPE,
885	preexec_fn=ResetSignals)
886	except OSError as e:
887	print('Error running %r: %s' % (sh_path, e), file=sys.stderr)
888	sys.exit(1)
889
890	p.stdin.write(code)
891
892	actual = {'sh_label': sh_label}
893	actual['stdout'], actual['stderr'] = p.communicate()
894	actual['status'] = p.wait()
895
896	case_actual['actual'].append(actual)
897
898	if opts.timeout_bin and os.path.exists(timeout_file):
899	cell_result = Result.TIMEOUT
900	elif not opts.timeout_bin and _TimedOut(actual['status']):
901	cell_result = Result.TIMEOUT
902	else:
903	messages = []
904	cell_result = Result.PASS
905
906	# TODO: Warn about no assertions? Well it will always test the error
907	# code.
908	assertions = CreateAssertions(case, sh_label)
909	for a in assertions:
910	result, msg = a.Check(sh_label, actual)
911	# The minimum one wins.
912	# If any failed, then the result is FAIL.
913	# If any are OK, but none are FAIL, the result is OK.
914	cell_result = min(cell_result, result)
915	if msg:
916	messages.append(msg)
917
918	if cell_result != Result.PASS or opts.details:
919	d = (i, sh_label, actual['stdout'], actual['stderr'],
920	messages)
921	out.AddDetails(d)
922
923	result_row.append(cell_result)
924
925	actual['cell_result'] = cell_result
926
927	stats.ReportCell(i, cell_result, sh_label)
928
929	if sh_label in OTHER_OSH:
930	# This is only an error if we tried to run ANY OSH.
931	if osh_cpython_index == -1:
932	raise RuntimeError(
933	"Couldn't determine index of osh-cpython")
934
935	other_result = result_row[shell_index]
936	cpython_result = result_row[osh_cpython_index]
937	if other_result != cpython_result:
938	stats.Inc('oils_ALT_delta')
939
940	out.WriteRow(i, line_num, result_row, desc)
941	actual_out.WriteActualRow(case_actual)
942
943	return stats
944
945
946	# ANSI color constants
947	_RESET = '\033[0;0m'
948	_BOLD = '\033[1m'
949
950	_RED = '\033[31m'
951	_GREEN = '\033[32m'
952	_YELLOW = '\033[33m'
953	_BLUE = '\033[34m'
954	_PURPLE = '\033[35m'
955	_CYAN = '\033[36m'
956	#_WHITE = '\033[37m'
957
958	TEXT_CELLS = {
959	Result.TIMEOUT: 'TIME',
960	Result.FAIL: 'FAIL',
961	Result.BUG: 'BUG',
962	Result.BUG_2: 'BUG-2',
963	Result.NI: 'N-I',
964	Result.OK: 'ok',
965	Result.OK_2: 'ok-2',
966	Result.OK_3: 'ok-3',
967	Result.OK_4: 'ok-4',
968	Result.PASS: 'pass',
969	}
970
971	ANSI_COLORS = {
972	Result.TIMEOUT: _PURPLE,
973	Result.FAIL: _RED,
974	Result.BUG: _YELLOW,
975	Result.BUG_2: _BLUE,
976	Result.NI: _YELLOW,
977	Result.OK: _YELLOW,
978	Result.OK_2: _BLUE,
979	Result.OK_3: _CYAN,
980	Result.OK_4: _PURPLE,
981	Result.PASS: _GREEN,
982	}
983
984
985	def _AnsiCells():
986	lookup = {}
987	for i in xrange(Result.length):
988	lookup[i] = ''.join([ANSI_COLORS[i], _BOLD, TEXT_CELLS[i], _RESET])
989	return lookup
990
991
992	ANSI_CELLS = _AnsiCells()
993
994	HTML_CELLS = {
995	Result.TIMEOUT: '<td class="timeout">TIME',
996	Result.FAIL: '<td class="fail">FAIL',
997	Result.BUG: '<td class="bug">BUG',
998	Result.BUG_2: '<td class="bug-2">BUG-2',
999	Result.NI: '<td class="n-i">N-I',
1000	Result.OK: '<td class="ok">ok',
1001	Result.OK_2: '<td class="ok-2">ok-2',
1002	Result.OK_3: '<td class="ok-3">ok-3',
1003	Result.OK_4: '<td class="ok-4">ok-4',
1004	Result.PASS: '<td class="pass">pass',
1005	}
1006
1007
1008	def _ValidUtf8String(s):
1009	"""Return an arbitrary string as a readable utf-8 string.
1010
1011	We output utf-8 to either HTML or the console. If we get invalid
1012	utf-8 as stdout/stderr (which is very possible), then show the ASCII
1013	repr().
1014	"""
1015	try:
1016	s.decode('utf-8')
1017	return s # it decoded OK
1018	except UnicodeDecodeError:
1019	return repr(s) # ASCII representation
1020
1021
1022	class Output(object):
1023
1024	def __init__(self, f, verbose):
1025	self.f = f
1026	self.verbose = verbose
1027	self.details = []
1028
1029	def BeginCases(self, test_file):
1030	pass
1031
1032	def WriteHeader(self, sh_labels):
1033	pass
1034
1035	def WriteRow(self, i, line_num, row, desc):
1036	pass
1037
1038	def EndCases(self, sh_labels, stats):
1039	pass
1040
1041	def AddDetails(self, entry):
1042	self.details.append(entry)
1043
1044	# Helper function
1045	def _WriteDetailsAsText(self, details):
1046	for case_index, shell, stdout, stderr, messages in details:
1047	print('case: %d' % case_index, file=self.f)
1048	for m in messages:
1049	print(m, file=self.f)
1050
1051	# Assume the terminal can show utf-8, but we don't want random binary.
1052	print('%s stdout:' % shell, file=self.f)
1053	print(_ValidUtf8String(stdout), file=self.f)
1054
1055	print('%s stderr:' % shell, file=self.f)
1056	print(_ValidUtf8String(stderr), file=self.f)
1057
1058	print('', file=self.f)
1059
1060
1061	class TeeOutput(object):
1062	"""For multiple outputs in one run, e.g. HTML and TSV.
1063
1064	UNUSED
1065	"""
1066
1067	def __init__(self, outs):
1068	self.outs = outs
1069
1070	def BeginCases(self, test_file):
1071	for out in self.outs:
1072	out.BeginCases(test_file)
1073
1074	def WriteHeader(self, sh_labels):
1075	for out in self.outs:
1076	out.WriteHeader(sh_labels)
1077
1078	def WriteRow(self, i, line_num, row, desc):
1079	for out in self.outs:
1080	out.WriteRow(i, line_num, row, desc)
1081
1082	def EndCases(self, sh_labels, stats):
1083	for out in self.outs:
1084	out.EndCases(sh_labels, stats)
1085
1086	def AddDetails(self, entry):
1087	for out in self.outs:
1088	out.AddDetails(entry)
1089
1090
1091	class TsvOutput(Output):
1092	"""Write a plain-text TSV file.
1093
1094	UNUSED since we are outputting LONG format with --tsv-output.
1095	"""
1096
1097	def WriteHeader(self, sh_labels):
1098	self.f.write('case\tline\t') # case number and line number
1099	for sh_label in sh_labels:
1100	self.f.write(sh_label)
1101	self.f.write('\t')
1102	self.f.write('\n')
1103
1104	def WriteRow(self, i, line_num, row, desc):
1105	self.f.write('%3d\t%3d\t' % (i, line_num))
1106
1107	for result in row:
1108	c = TEXT_CELLS[result]
1109	self.f.write(c)
1110	self.f.write('\t')
1111
1112	# note: 'desc' could use TSV8, but just ignore it for now
1113	#self.f.write(desc)
1114	self.f.write('\n')
1115
1116
1117	class AnsiOutput(Output):
1118
1119	def BeginCases(self, test_file):
1120	self.f.write('%s\n' % test_file)
1121
1122	def WriteHeader(self, sh_labels):
1123	self.f.write(_BOLD)
1124	self.f.write('case\tline\t') # case number and line number
1125	for sh_label in sh_labels:
1126	self.f.write(sh_label)
1127	self.f.write('\t')
1128	self.f.write(_RESET)
1129	self.f.write('\n')
1130
1131	def WriteRow(self, i, line_num, row, desc):
1132	self.f.write('%3d\t%3d\t' % (i, line_num))
1133
1134	for result in row:
1135	c = ANSI_CELLS[result]
1136	self.f.write(c)
1137	self.f.write('\t')
1138
1139	self.f.write(desc)
1140	self.f.write('\n')
1141
1142	if self.verbose:
1143	self._WriteDetailsAsText(self.details)
1144	self.details = []
1145
1146	def _WriteShellSummary(self, sh_labels, stats):
1147	if len(stats.nonzero_results) <= 1: # Skip trivial summaries
1148	return
1149
1150	# Reiterate header
1151	self.f.write(_BOLD)
1152	self.f.write('\t\t')
1153	for sh_label in sh_labels:
1154	self.f.write(sh_label)
1155	self.f.write('\t')
1156	self.f.write(_RESET)
1157	self.f.write('\n')
1158
1159	# Write totals by cell.
1160	for result in sorted(stats.nonzero_results, reverse=True):
1161	self.f.write('\t%s' % ANSI_CELLS[result])
1162	for sh_label in sh_labels:
1163	self.f.write('\t%d' % stats.by_shell[sh_label][result])
1164	self.f.write('\n')
1165
1166	# The bottom row is all the same, but it helps readability.
1167	self.f.write('\ttotal')
1168	for sh_label in sh_labels:
1169	self.f.write('\t%d' % stats.counters['num_cases_run'])
1170	self.f.write('\n')
1171	self.f.write('\n')
1172
1173	def EndCases(self, sh_labels, stats):
1174	print()
1175	self._WriteShellSummary(sh_labels, stats)
1176
1177
1178	class HtmlOutput(Output):
1179
1180	def __init__(self, f, verbose, spec_name, sh_labels, cases):
1181	Output.__init__(self, f, verbose)
1182	self.spec_name = spec_name
1183	self.sh_labels = sh_labels # saved from header
1184	self.cases = cases # for linking to code
1185	self.row_html = [] # buffered
1186
1187	def _SourceLink(self, line_num, desc):
1188	return '<a href="%s.test.html#L%d">%s</a>' % (self.spec_name, line_num,
1189	cgi.escape(desc))
1190
1191	def BeginCases(self, test_file):
1192	css_urls = ['../../../web/base.css', '../../../web/spec-tests.css']
1193	title = '%s: spec test case results' % self.spec_name
1194	html_head.Write(self.f, title, css_urls=css_urls)
1195
1196	self.f.write('''\
1197	<body class="width60">
1198	<p id="home-link">
1199	<a href=".">spec test index</a>
1200	/
1201	<a href="/">oils.pub</a>
1202	</p>
1203	<h1>Results for %s</h1>
1204	<table>
1205	''' % test_file)
1206
1207	def _WriteShellSummary(self, sh_labels, stats):
1208	# NOTE: This table has multiple <thead>, which seems OK.
1209	self.f.write('''
1210	<thead>
1211	<tr class="table-header">
1212	''')
1213
1214	columns = ['status'] + sh_labels + ['']
1215	for c in columns:
1216	self.f.write('<td>%s</td>' % c)
1217
1218	self.f.write('''
1219	</tr>
1220	</thead>
1221	''')
1222
1223	# Write totals by cell.
1224	for result in sorted(stats.nonzero_results, reverse=True):
1225	self.f.write('<tr>')
1226
1227	self.f.write(HTML_CELLS[result])
1228	self.f.write('</td> ')
1229
1230	for sh_label in sh_labels:
1231	self.f.write('<td>%d</td>' % stats.by_shell[sh_label][result])
1232
1233	self.f.write('<td></td>')
1234	self.f.write('</tr>\n')
1235
1236	# The bottom row is all the same, but it helps readability.
1237	self.f.write('<tr>')
1238	self.f.write('<td>total</td>')
1239	for sh_label in sh_labels:
1240	self.f.write('<td>%d</td>' % stats.counters['num_cases_run'])
1241	self.f.write('<td></td>')
1242	self.f.write('</tr>\n')
1243
1244	# Blank row for space.
1245	self.f.write('<tr>')
1246	for i in xrange(len(sh_labels) + 2):
1247	self.f.write('<td style="height: 2em"></td>')
1248	self.f.write('</tr>\n')
1249
1250	def WriteHeader(self, sh_labels):
1251	f = cStringIO.StringIO()
1252
1253	f.write('''
1254	<thead>
1255	<tr class="table-header">
1256	''')
1257
1258	columns = ['case'] + sh_labels
1259	for c in columns:
1260	f.write('<td>%s</td>' % c)
1261	f.write('<td class="case-desc">description</td>')
1262
1263	f.write('''
1264	</tr>
1265	</thead>
1266	''')
1267
1268	self.row_html.append(f.getvalue())
1269
1270	def WriteRow(self, i, line_num, row, desc):
1271	f = cStringIO.StringIO()
1272	f.write('<tr>')
1273	f.write('<td>%3d</td>' % i)
1274
1275	show_details = False
1276
1277	for result in row:
1278	c = HTML_CELLS[result]
1279	if result not in (Result.PASS, Result.TIMEOUT): # nothing to show
1280	show_details = True
1281
1282	f.write(c)
1283	f.write('</td>')
1284	f.write('\t')
1285
1286	f.write('<td class="case-desc">')
1287	f.write(self._SourceLink(line_num, desc))
1288	f.write('</td>')
1289	f.write('</tr>\n')
1290
1291	# Show row with details link.
1292	if show_details:
1293	f.write('<tr>')
1294	f.write('<td class="details-row"></td>') # for the number
1295
1296	for col_index, result in enumerate(row):
1297	f.write('<td class="details-row">')
1298	if result != Result.PASS:
1299	sh_label = self.sh_labels[col_index]
1300	f.write('<a href="#details-%s-%s">details</a>' %
1301	(i, sh_label))
1302	f.write('</td>')
1303
1304	f.write('<td class="details-row"></td>') # for the description
1305	f.write('</tr>\n')
1306
1307	self.row_html.append(f.getvalue()) # buffer it
1308
1309	def _WriteStats(self, stats):
1310	self.f.write('%(num_passed)d passed, %(num_ok)d OK, '
1311	'%(num_ni)d not implemented, %(num_bug)d BUG, '
1312	'%(num_failed)d failed, %(num_timeout)d timeouts, '
1313	'%(num_skipped)d cases skipped\n' % stats.counters)
1314
1315	def EndCases(self, sh_labels, stats):
1316	self._WriteShellSummary(sh_labels, stats)
1317
1318	# Write all the buffered rows
1319	for h in self.row_html:
1320	self.f.write(h)
1321
1322	self.f.write('</table>\n')
1323	self.f.write('<pre>')
1324	self._WriteStats(stats)
1325	if stats.Get('oils_num_failed'):
1326	self.f.write('%(oils_num_failed)d failed under osh\n' %
1327	stats.counters)
1328	self.f.write('</pre>')
1329
1330	if self.details:
1331	self._WriteDetails()
1332
1333	self.f.write('</body></html>')
1334
1335	def _WriteDetails(self):
1336	self.f.write("<h2>Details on runs that didn't PASS</h2>")
1337	self.f.write('<table id="details">')
1338
1339	for case_index, sh_label, stdout, stderr, messages in self.details:
1340	self.f.write('<tr>')
1341	self.f.write('<td><a name="details-%s-%s"></a><b>%s</b></td>' %
1342	(case_index, sh_label, sh_label))
1343
1344	self.f.write('<td>')
1345
1346	# Write description and link to the code
1347	case = self.cases[case_index]
1348	line_num = case['line_num']
1349	desc = case['desc']
1350	self.f.write('%d ' % case_index)
1351	self.f.write(self._SourceLink(line_num, desc))
1352	self.f.write('<br/><br/>\n')
1353
1354	for m in messages:
1355	self.f.write('<span class="assertion">%s</span><br/>\n' %
1356	cgi.escape(m))
1357	if messages:
1358	self.f.write('<br/>\n')
1359
1360	def _WriteRaw(s):
1361	self.f.write('<pre>')
1362
1363	# stdout might contain invalid utf-8; make it valid;
1364	valid_utf8 = _ValidUtf8String(s)
1365
1366	self.f.write(cgi.escape(valid_utf8))
1367	self.f.write('</pre>')
1368
1369	self.f.write('<i>stdout:</i> <br/>\n')
1370	_WriteRaw(stdout)
1371
1372	self.f.write('<i>stderr:</i> <br/>\n')
1373	_WriteRaw(stderr)
1374
1375	self.f.write('</td>')
1376	self.f.write('</tr>')
1377
1378	self.f.write('</table>')
1379
1380
1381	def MakeTestEnv(opts):
1382	if not opts.tmp_env:
1383	raise RuntimeError('--tmp-env required')
1384	if not opts.path_env:
1385	raise RuntimeError('--path-env required')
1386	env = {
1387	'PATH': opts.path_env,
1388	#'LANG': opts.lang_env,
1389	}
1390	for p in opts.env_pair:
1391	name, value = p.split('=', 1)
1392	env[name] = value
1393
1394	return env
1395
1396
1397	def _DefaultSuite(spec_name):
1398	if spec_name.startswith('ysh-'):
1399	suite = 'ysh'
1400	elif spec_name.startswith('hay'): # hay.test.sh is ysh
1401	suite = 'ysh'
1402
1403	elif spec_name.startswith('tea-'):
1404	suite = 'tea'
1405	else:
1406	suite = 'osh'
1407
1408	return suite
1409
1410
1411	def ParseTestList(test_files):
1412	for test_file in test_files:
1413	with open(test_file) as f:
1414	tokens = Tokenizer(f)
1415	try:
1416	file_metadata, cases = ParseTestFile(test_file, tokens)
1417	except RuntimeError as e:
1418	log('ERROR in %r', test_file)
1419	raise
1420	except ParseError as e:
1421	log('PARSE ERROR in %r', test_file)
1422	raise
1423
1424	tmp = os.path.basename(test_file)
1425	spec_name = tmp.split('.')[0] # foo.test.sh -> foo
1426
1427	suite = file_metadata.get('suite') or _DefaultSuite(spec_name)
1428
1429	tmp = file_metadata.get('tags')
1430	tags = tmp.split() if tmp else []
1431
1432	# Don't need compare_shells, etc. to decide what to run
1433
1434	row = {
1435	'spec_name': spec_name,
1436	'suite': suite,
1437	'tags': tags,
1438	'cases': cases
1439	}
1440	#print(row)
1441	yield row
1442
1443
1444	def main(argv):
1445	# First check if bash is polluting the environment. Tests rely on the
1446	# environment.
1447	v = os.getenv('RANDOM')
1448	if v is not None:
1449	raise AssertionError('got $RANDOM = %s' % v)
1450	v = os.getenv('PPID')
1451	if v is not None:
1452	raise AssertionError('got $PPID = %s' % v)
1453
1454	p = optparse.OptionParser('%s [options] TEST_FILE shell...' % sys.argv[0])
1455	spec_lib.DefineCommon(p)
1456	spec_lib.DefineShSpec(p)
1457	opts, argv = p.parse_args(argv)
1458
1459	# --print-tagged to figure out what to run
1460	if opts.print_tagged:
1461	to_find = opts.print_tagged
1462	for row in ParseTestList(argv[1:]):
1463	if to_find in row['tags']:
1464	print(row['spec_name'])
1465	return 0
1466
1467	# --print-table to figure out what to run
1468	if opts.print_table:
1469	for row in ParseTestList(argv[1:]):
1470	print('%(suite)s\t%(spec_name)s' % row)
1471	#print(row)
1472	return 0
1473
1474	if opts.test_case_json:
1475	for row in ParseTestList(argv[1:]):
1476	#print('%(suite)s\t%(spec_name)s' % row)
1477	print(json.dumps(row, indent=2))
1478	return 0
1479
1480	#
1481	# Now deal with a single file
1482	#
1483
1484	try:
1485	test_file = argv[1]
1486	except IndexError:
1487	p.print_usage()
1488	return 1
1489
1490	with open(test_file) as f:
1491	tokens = Tokenizer(f)
1492	file_metadata, cases = ParseTestFile(test_file, tokens)
1493
1494	# List test cases and return
1495	if opts.do_list:
1496	for i, case in enumerate(cases):
1497	if opts.verbose: # print the raw dictionary for debugging
1498	print(pprint.pformat(case))
1499	else:
1500	print('%d\t%s' % (i, case['desc']))
1501	return 0
1502
1503	# for test/spec-cpp.sh
1504	if opts.print_spec_suite:
1505	tmp = os.path.basename(test_file)
1506	spec_name = tmp.split('.')[0] # foo.test.sh -> foo
1507
1508	suite = file_metadata.get('suite') or _DefaultSuite(spec_name)
1509	print(suite)
1510	return 0
1511
1512	if opts.verbose:
1513	for k, v in file_metadata.items():
1514	print('\t%-20s: %s' % (k, v), file=sys.stderr)
1515	print('', file=sys.stderr)
1516
1517	if opts.oils_bin_dir: # to find OSH and YSH
1518	shells = []
1519
1520	if opts.compare_shells:
1521	comp = file_metadata.get('compare_shells')
1522	# Compare 'compare_shells' and Python
1523	shells.extend(comp.split() if comp else [])
1524
1525	# Always run with the Python version
1526	our_shell = file_metadata.get('our_shell', 'osh') # default is OSH
1527	if our_shell != '-':
1528	shells.append(os.path.join(opts.oils_bin_dir, our_shell))
1529
1530	# Legacy OVM/CPython build
1531	if opts.ovm_bin_dir:
1532	shells.append(os.path.join(opts.ovm_bin_dir, our_shell))
1533
1534	# New C++ build
1535	if opts.oils_cpp_bin_dir:
1536	shells.append(os.path.join(opts.oils_cpp_bin_dir, our_shell))
1537
1538	# Overwrite it when --oils-bin-dir is set
1539	# It's no longer a flag
1540	opts.oils_failures_allowed = \
1541	int(file_metadata.get('oils_failures_allowed', 0))
1542
1543	else:
1544	# To compare arbitrary shells
1545	shells = argv[2:]
1546
1547	shell_pairs = spec_lib.MakeShellPairs(shells)
1548
1549	if opts.range:
1550	begin, end = spec_lib.ParseRange(opts.range)
1551	case_predicate = spec_lib.RangePredicate(begin, end)
1552	elif opts.regex:
1553	desc_re = re.compile(opts.regex, re.IGNORECASE)
1554	case_predicate = spec_lib.RegexPredicate(desc_re)
1555	else:
1556	case_predicate = lambda i, case: True
1557
1558	out_f = sys.stderr if opts.do_print else sys.stdout
1559
1560	# Set up output style. Also see asdl/format.py
1561	if opts.format == 'ansi':
1562	table_out = AnsiOutput(out_f, opts.verbose)
1563
1564	elif opts.format == 'html':
1565	spec_name = os.path.basename(test_file)
1566	spec_name = spec_name.split('.')[0]
1567
1568	sh_labels = [label for label, _ in shell_pairs]
1569
1570	table_out = HtmlOutput(out_f, opts.verbose, spec_name, sh_labels,
1571	cases)
1572
1573	else:
1574	raise AssertionError()
1575
1576	if opts.yahtzee_out_file:
1577	actual_f = open(opts.yahtzee_out_file, 'w')
1578	else:
1579	actual_f = None
1580
1581	actual_out = ActualOutput(actual_f)
1582
1583	table_out.BeginCases(os.path.basename(test_file))
1584
1585	env = MakeTestEnv(opts)
1586	stats = RunCases(cases,
1587	case_predicate,
1588	shell_pairs,
1589	env,
1590	table_out,
1591	actual_out,
1592	opts,
1593	legacy_tmp_dir=bool(file_metadata.get('legacy_tmp_dir')))
1594
1595	table_out.EndCases([sh_label for sh_label, _ in shell_pairs], stats)
1596
1597	if opts.tsv_output:
1598	with open(opts.tsv_output, 'w') as f:
1599	stats.WriteTsv(f)
1600
1601	# TODO: Could --stats-{file,template} be a separate awk step on .tsv files?
1602	stats.Set('oils_failures_allowed', opts.oils_failures_allowed)
1603
1604	# If it's not set separately for C++, we default to the allowed number
1605	# above
1606	x = int(
1607	file_metadata.get('oils_cpp_failures_allowed',
1608	opts.oils_failures_allowed))
1609	stats.Set('oils_cpp_failures_allowed', x)
1610
1611	if opts.stats_file:
1612	with open(opts.stats_file, 'w') as f:
1613	f.write(opts.stats_template % stats.counters)
1614	f.write('\n') # bash 'read' requires a newline
1615
1616	# spec/smoke.test.sh -> smoke
1617	test_name = os.path.basename(test_file).split('.')[0]
1618
1619	return _SuccessOrFailure(test_name, stats)
1620
1621
1622	def _SuccessOrFailure(test_name, stats):
1623	allowed = stats.Get('oils_failures_allowed')
1624	allowed_cpp = stats.Get('oils_cpp_failures_allowed')
1625
1626	all_count = stats.Get('num_failed')
1627	oils_count = stats.Get('oils_num_failed')
1628	oils_cpp_count = stats.Get('oils_cpp_num_failed')
1629
1630	errors = []
1631	if oils_count != allowed:
1632	errors.append('Got %d Oils failures, but %d are allowed' %
1633	(oils_count, allowed))
1634	else:
1635	if allowed != 0:
1636	log('%s: note: Got %d allowed Oils failures', test_name, allowed)
1637
1638	# TODO: remove special case for 0
1639	if oils_cpp_count != 0:
1640	if oils_cpp_count != allowed_cpp:
1641	errors.append('Got %d Oils C++ failures, but %d are allowed' %
1642	(oils_cpp_count, allowed_cpp))
1643	else:
1644	if allowed_cpp != 0:
1645	log('%s: note: Got %d allowed Oils C++ failures', test_name,
1646	allowed_cpp)
1647
1648	if all_count != allowed:
1649	errors.append('Got %d total failures, but %d are allowed' %
1650	(all_count, allowed))
1651
1652	if errors:
1653	for msg in errors:
1654	log('%s: FATAL: %s', test_name, msg)
1655	return 1
1656
1657	return 0
1658
1659
1660	if __name__ == '__main__':
1661	try:
1662	sys.exit(main(sys.argv))
1663	except KeyboardInterrupt as e:
1664	print('%s: interrupted with Ctrl-C' % sys.argv[0], file=sys.stderr)
1665	sys.exit(1)
1666	except RuntimeError as e:
1667	print('FATAL: %s' % e, file=sys.stderr)
1668	sys.exit(1)