OILS / test / sh_spec.py View on Github | oils.pub

1544 lines, 966 significant
1#!/usr/bin/env python2
2from __future__ import print_function
3"""
4sh_spec.py -- Test framework to compare shells.
5
6Assertion help:
7 stdout: A single line of expected stdout. Newline is implicit.
8 stdout-json: JSON-encoded string. Use for the empty string (no newline),
9 for unicode chars, etc.
10
11 stderr: Ditto for stderr stream.
12 status: Expected shell return code. If not specified, the case must exit 0.
13
14Results:
15 PASS - we got the ideal, expected value
16 OK - we got a value that was not ideal, but expected
17 For OSH this is behavior that was defined to be different?
18 N-I - Not implemented (e.g. $''). Assertions still checked (in case it
19 starts working)
20 BUG - we verified the value of a known bug
21 FAIL - we got an unexpected value. If the implementation can't be changed,
22 it should be converted to BUG or OK. Otherwise it should be made to
23 PASS.
24
25NOTE: The difference between OK and BUG is a matter of judgement. If the ideal
26behavior is a compile time error (code 2), a runtime error is generally OK.
27
28If ALL shells agree on a broken behavior, they are all marked OK (but our
29implementation will be PASS.) But if the behavior is NOT POSIX compliant, then
30it will be a BUG.
31
32If one shell disagrees with others, that is generally a BUG.
33
34Example test case:
35
36#### hello and fail
37echo hello
38echo world
39exit 1
40## status: 1
41#
42# ignored comment
43#
44## STDOUT:
45hello
46world
47## END
48
49"""
50
51import collections
52import cgi
53import cStringIO
54import errno
55import json
56import optparse
57import os
58import pprint
59import re
60import shutil
61import subprocess
62import sys
63
64from test import spec_lib
65from doctools import html_head
66
67log = spec_lib.log
68
69# Magic strings for other variants of OSH.
70
71# NOTE: osh_ALT is usually _bin/osh -- the release binary.
72# It would be better to rename these osh-cpython and osh-ovm. Have the concept
73# of a suffix?
74
75OSH_CPYTHON = ('osh', 'osh-dbg')
76OTHER_OSH = ('osh_ALT', )
77
78YSH_CPYTHON = ('ysh', 'ysh-dbg')
79OTHER_YSH = ('oil_ALT', )
80
81# For now, only count the Oils CPython failures. TODO: the spec-cpp job should
82# assert the osh-cpp and ysh-cpp deltas.
83OTHER_OILS = OTHER_OSH + OTHER_YSH + ('osh-cpp', 'ysh-cpp')
84
85
86class ParseError(Exception):
87 pass
88
89
90# EXAMPLES:
91## stdout: foo
92## stdout-json: ""
93#
94# In other words, it could be (name, value) or (qualifier, name, value)
95
96KEY_VALUE_RE = re.compile(
97 r'''
98 [#][#] \s+
99 # optional prefix with qualifier and shells
100 (?: (OK(?:-\d)? | BUG(?:-\d)? | N-I) \s+ ([\w+/]+) \s+ )?
101 ([\w\-]+) # key
102 :
103 \s* (.*) # value
104''', re.VERBOSE)
105
106END_MULTILINE_RE = re.compile(r'''
107 [#][#] \s+ END
108''', re.VERBOSE)
109
110# Line types
111TEST_CASE_BEGIN = 0 # Starts with ####
112KEY_VALUE = 1 # Metadata
113KEY_VALUE_MULTILINE = 2 # STDOUT STDERR
114END_MULTILINE = 3 # STDOUT STDERR
115PLAIN_LINE = 4 # Uncommented
116EOF = 5
117
118LEX_OUTER = 0 # Ignore blank lines, e.g. for separating cases
119LEX_RAW = 1 # Blank lines are significant
120
121
122class Tokenizer(object):
123 """Modal lexer!"""
124
125 def __init__(self, f):
126 self.f = f
127
128 self.cursor = None
129 self.line_num = 0
130
131 self.next()
132
133 def _ClassifyLine(self, line, lex_mode):
134 if not line: # empty
135 return self.line_num, EOF, ''
136
137 if lex_mode == LEX_OUTER and not line.strip():
138 return None
139
140 if line.startswith('####'):
141 desc = line[4:].strip()
142 return self.line_num, TEST_CASE_BEGIN, desc
143
144 m = KEY_VALUE_RE.match(line)
145 if m:
146 qualifier, shells, name, value = m.groups()
147 # HACK: Expected data should have the newline.
148 if name in ('stdout', 'stderr'):
149 value += '\n'
150
151 if name in ('STDOUT', 'STDERR'):
152 token_type = KEY_VALUE_MULTILINE
153 else:
154 token_type = KEY_VALUE
155 return self.line_num, token_type, (qualifier, shells, name, value)
156
157 m = END_MULTILINE_RE.match(line)
158 if m:
159 return self.line_num, END_MULTILINE, None
160
161 # If it starts with ##, it should be metadata. This finds some typos.
162 if line.lstrip().startswith('##'):
163 raise RuntimeError('Invalid ## line %r' % line)
164
165 if line.lstrip().startswith('#'): # Ignore comments
166 return None # try again
167
168 # Non-empty line that doesn't start with '#'
169 # NOTE: We need the original line to test the whitespace sensitive <<-.
170 # And we need rstrip because we add newlines back below.
171 return self.line_num, PLAIN_LINE, line
172
173 def next(self, lex_mode=LEX_OUTER):
174 """Raises StopIteration when exhausted."""
175 while True:
176 line = self.f.readline()
177 self.line_num += 1
178
179 tok = self._ClassifyLine(line, lex_mode)
180 if tok is not None:
181 break
182
183 self.cursor = tok
184 return self.cursor
185
186 def peek(self):
187 return self.cursor
188
189
190def AddMetadataToCase(case, qualifier, shells, name, value, line_num):
191 shells = shells.split('/') # bash/dash/mksh
192 for shell in shells:
193 if shell not in case:
194 case[shell] = {}
195
196 # Check a duplicate specification
197 name_without_type = re.sub(r'-(json|repr)$', '', name)
198 if (name_without_type in case[shell] or
199 name_without_type + '-json' in case[shell] or
200 name_without_type + '-repr' in case[shell]):
201 raise ParseError('Line %d: duplicate spec %r for %r' %
202 (line_num, name, shell))
203
204 # Check inconsistent qualifier
205 if 'qualifier' in case[shell] and qualifier != case[shell]['qualifier']:
206 raise ParseError(
207 'Line %d: inconsistent qualifier %r is specified for %r, '
208 'but %r was previously specified. ' %
209 (line_num, qualifier, shell, case[shell]['qualifier']))
210
211 case[shell][name] = value
212 case[shell]['qualifier'] = qualifier
213
214
215# Format of a test script.
216#
217# -- Code is either literal lines, or a commented out code: value.
218# code = PLAIN_LINE*
219# | '## code:' VALUE
220#
221# -- Key value pairs can be single- or multi-line
222# key_value = '##' KEY ':' VALUE
223# | KEY_VALUE_MULTILINE PLAIN_LINE* END_MULTILINE
224#
225# -- Description, then key-value pairs surrounding code.
226# test_case = '####' DESC
227# key_value*
228# code
229# key_value*
230#
231# -- Should be a blank line after each test case. Leading comments and code
232# -- are OK.
233#
234# test_file =
235# key_value* -- file level metadata
236# (test_case '\n')*
237
238
239def ParseKeyValue(tokens, case):
240 """Parse commented-out metadata in a test case.
241
242 The metadata must be contiguous.
243
244 Args:
245 tokens: Tokenizer
246 case: dictionary to add to
247 """
248 while True:
249 line_num, kind, item = tokens.peek()
250
251 if kind == KEY_VALUE_MULTILINE:
252 qualifier, shells, name, empty_value = item
253 if empty_value:
254 raise ParseError(
255 'Line %d: got value %r for %r, but the value should be on the '
256 'following lines' % (line_num, empty_value, name))
257
258 value_lines = []
259 while True:
260 tokens.next(lex_mode=LEX_RAW) # empty lines aren't skipped
261 _, kind2, item2 = tokens.peek()
262 if kind2 != PLAIN_LINE:
263 break
264 value_lines.append(item2)
265
266 value = ''.join(value_lines)
267
268 name = name.lower() # STDOUT -> stdout
269 if qualifier:
270 AddMetadataToCase(case, qualifier, shells, name, value,
271 line_num)
272 else:
273 case[name] = value
274
275 # END token is optional.
276 if kind2 == END_MULTILINE:
277 tokens.next()
278
279 elif kind == KEY_VALUE:
280 qualifier, shells, name, value = item
281
282 if qualifier:
283 AddMetadataToCase(case, qualifier, shells, name, value,
284 line_num)
285 else:
286 case[name] = value
287
288 tokens.next()
289
290 else: # Unknown token type
291 break
292
293
294def ParseCodeLines(tokens, case):
295 """Parse uncommented code in a test case."""
296 _, kind, item = tokens.peek()
297 if kind != PLAIN_LINE:
298 raise ParseError('Expected a line of code (got %r, %r)' % (kind, item))
299 code_lines = []
300 while True:
301 _, kind, item = tokens.peek()
302 if kind != PLAIN_LINE:
303 case['code'] = ''.join(code_lines)
304 return
305 code_lines.append(item)
306 tokens.next(lex_mode=LEX_RAW)
307
308
309def ParseTestCase(tokens):
310 """Parse a single test case and return it.
311
312 If at EOF, return None.
313 """
314 line_num, kind, item = tokens.peek()
315 if kind == EOF:
316 return None
317
318 if kind != TEST_CASE_BEGIN:
319 raise RuntimeError("line %d: Expected TEST_CASE_BEGIN, got %r" %
320 (line_num, [kind, item]))
321
322 tokens.next()
323
324 case = {'desc': item, 'line_num': line_num}
325
326 ParseKeyValue(tokens, case)
327
328 # For broken code
329 if 'code' in case: # Got it through a key value pair
330 return case
331
332 ParseCodeLines(tokens, case)
333 ParseKeyValue(tokens, case)
334
335 return case
336
337
338_META_FIELDS = [
339 'our_shell',
340 'compare_shells',
341 'suite',
342 'tags',
343 'oils_failures_allowed',
344 'oils_cpp_failures_allowed',
345]
346
347
348def ParseTestFile(test_file, tokens):
349 """
350 test_file: Only for error message
351 """
352 file_metadata = {}
353 test_cases = []
354
355 try:
356 # Skip over the header. Setup code can go here, although would we have to
357 # execute it on every case?
358 while True:
359 line_num, kind, item = tokens.peek()
360 if kind != KEY_VALUE:
361 break
362
363 qualifier, shells, name, value = item
364 if qualifier is not None:
365 raise RuntimeError('Invalid qualifier in spec file metadata')
366 if shells is not None:
367 raise RuntimeError('Invalid shells in spec file metadata')
368
369 file_metadata[name] = value
370
371 tokens.next()
372
373 while True: # Loop over cases
374 test_case = ParseTestCase(tokens)
375 if test_case is None:
376 break
377 test_cases.append(test_case)
378
379 except StopIteration:
380 raise RuntimeError('Unexpected EOF parsing test cases')
381
382 for name in file_metadata:
383 if name not in _META_FIELDS:
384 raise RuntimeError('Invalid file metadata %r in %r' %
385 (name, test_file))
386
387 return file_metadata, test_cases
388
389
390def CreateStringAssertion(d, key, assertions, qualifier=False):
391 found = False
392
393 exp = d.get(key)
394 if exp is not None:
395 a = EqualAssertion(key, exp, qualifier=qualifier)
396 assertions.append(a)
397 found = True
398
399 exp_json = d.get(key + '-json')
400 if exp_json is not None:
401 exp = json.loads(exp_json, encoding='utf-8')
402 a = EqualAssertion(key, exp, qualifier=qualifier)
403 assertions.append(a)
404 found = True
405
406 # For testing invalid unicode
407 exp_repr = d.get(key + '-repr')
408 if exp_repr is not None:
409 exp = eval(exp_repr)
410 a = EqualAssertion(key, exp, qualifier=qualifier)
411 assertions.append(a)
412 found = True
413
414 return found
415
416
417def CreateIntAssertion(d, key, assertions, qualifier=False):
418 exp = d.get(key) # expected
419 if exp is not None:
420 # For now, turn it into int
421 a = EqualAssertion(key, int(exp), qualifier=qualifier)
422 assertions.append(a)
423 return True
424 return False
425
426
427def CreateAssertions(case, sh_label):
428 """Given a raw test case and a shell label, create EqualAssertion instances
429 to run."""
430 assertions = []
431
432 # Whether we found assertions
433 stdout = False
434 stderr = False
435 status = False
436
437 # So the assertion are exactly the same for osh and osh_ALT
438
439 if sh_label.startswith('osh'):
440 case_sh = 'osh'
441 elif sh_label.startswith('bash'):
442 case_sh = 'bash'
443 else:
444 case_sh = sh_label
445
446 if case_sh in case:
447 q = case[case_sh]['qualifier']
448 if CreateStringAssertion(case[case_sh],
449 'stdout',
450 assertions,
451 qualifier=q):
452 stdout = True
453 if CreateStringAssertion(case[case_sh],
454 'stderr',
455 assertions,
456 qualifier=q):
457 stderr = True
458 if CreateIntAssertion(case[case_sh], 'status', assertions,
459 qualifier=q):
460 status = True
461
462 if not stdout:
463 CreateStringAssertion(case, 'stdout', assertions)
464 if not stderr:
465 CreateStringAssertion(case, 'stderr', assertions)
466 if not status:
467 if 'status' in case:
468 CreateIntAssertion(case, 'status', assertions)
469 else:
470 # If the user didn't specify a 'status' assertion, assert that the exit
471 # code is 0.
472 a = EqualAssertion('status', 0)
473 assertions.append(a)
474
475 no_traceback = SubstringAssertion('stderr', 'Traceback (most recent')
476 assertions.append(no_traceback)
477
478 #print 'SHELL', shell
479 #pprint.pprint(case)
480 #print(assertions)
481 return assertions
482
483
484class Result(object):
485 """Result of an stdout/stderr/status assertion or of a (case, shell) cell.
486
487 Order is important: the result of a cell is the minimum of the results of
488 each assertion.
489 """
490 TIMEOUT = 0 # ONLY a cell result, not an assertion result
491 FAIL = 1
492 BUG = 2
493 BUG_2 = 3
494 NI = 4
495 OK = 5
496 OK_2 = 6
497 OK_3 = 7
498 OK_4 = 8
499 PASS = 9
500
501 length = 10 # for loops
502
503
504def QualifierToResult(qualifier):
505 # type: (str) -> Result
506 if qualifier == 'BUG': # equal, but known bad
507 return Result.BUG
508 if qualifier == 'BUG-2':
509 return Result.BUG_2
510
511 if qualifier == 'N-I': # equal, and known UNIMPLEMENTED
512 return Result.NI
513
514 if qualifier == 'OK': # equal, but ok (not ideal)
515 return Result.OK
516 if qualifier == 'OK-2':
517 return Result.OK_2
518 if qualifier == 'OK-3':
519 return Result.OK_3
520 if qualifier == 'OK-4':
521 return Result.OK_4
522
523 return Result.PASS # ideal behavior
524
525
526class EqualAssertion(object):
527 """Check that two values are equal."""
528
529 def __init__(self, key, expected, qualifier=None):
530 self.key = key
531 self.expected = expected # expected value
532 self.qualifier = qualifier # whether this was a special case?
533
534 def __repr__(self):
535 return '<EqualAssertion %s == %r>' % (self.key, self.expected)
536
537 def Check(self, shell, record):
538 actual = record[self.key]
539 if actual != self.expected:
540 if len(str(self.expected)) < 40:
541 msg = '[%s %s] Expected %r, got %r' % (shell, self.key,
542 self.expected, actual)
543 else:
544 msg = '''
545[%s %s]
546Expected %r
547Got %r
548''' % (shell, self.key, self.expected, actual)
549
550 # TODO: Make this better and add a flag for it.
551 if 0:
552 import difflib
553 for line in difflib.unified_diff(self.expected,
554 actual,
555 fromfile='expected',
556 tofile='actual'):
557 print(repr(line))
558
559 return Result.FAIL, msg
560
561 return QualifierToResult(self.qualifier), ''
562
563
564class SubstringAssertion(object):
565 """Check that a string like stderr doesn't have a substring."""
566
567 def __init__(self, key, substring):
568 self.key = key
569 self.substring = substring
570
571 def __repr__(self):
572 return '<SubstringAssertion %s == %r>' % (self.key, self.substring)
573
574 def Check(self, shell, record):
575 actual = record[self.key]
576 if self.substring in actual:
577 msg = '[%s %s] Found %r' % (shell, self.key, self.substring)
578 return Result.FAIL, msg
579 return Result.PASS, ''
580
581
582class Stats(object):
583
584 def __init__(self, num_cases, sh_labels):
585 self.counters = collections.defaultdict(int)
586 c = self.counters
587 c['num_cases'] = num_cases
588 c['oils_num_passed'] = 0
589 c['oils_num_failed'] = 0
590 c['oils_cpp_num_failed'] = 0
591 # Number of osh_ALT results that differed from osh.
592 c['oils_ALT_delta'] = 0
593
594 self.by_shell = {}
595 for sh in sh_labels:
596 self.by_shell[sh] = collections.defaultdict(int)
597 self.nonzero_results = collections.defaultdict(int)
598
599 self.tsv_rows = []
600
601 def Inc(self, counter_name):
602 self.counters[counter_name] += 1
603
604 def Get(self, counter_name):
605 return self.counters[counter_name]
606
607 def Set(self, counter_name, val):
608 self.counters[counter_name] = val
609
610 def ReportCell(self, case_num, cell_result, sh_label):
611 self.tsv_rows.append(
612 (str(case_num), sh_label, TEXT_CELLS[cell_result]))
613
614 self.by_shell[sh_label][cell_result] += 1
615 self.nonzero_results[cell_result] += 1
616
617 c = self.counters
618 if cell_result == Result.TIMEOUT:
619 c['num_timeout'] += 1
620 elif cell_result == Result.FAIL:
621 # Special logic: don't count osh_ALT because its failures will be
622 # counted in the delta.
623 if sh_label not in OTHER_OILS:
624 c['num_failed'] += 1
625
626 if sh_label in OSH_CPYTHON + YSH_CPYTHON:
627 c['oils_num_failed'] += 1
628
629 if sh_label in ('osh-cpp', 'ysh-cpp'):
630 c['oils_cpp_num_failed'] += 1
631 elif cell_result in (Result.BUG, Result.BUG_2):
632 c['num_bug'] += 1
633 elif cell_result == Result.NI:
634 c['num_ni'] += 1
635 elif cell_result in (Result.OK, Result.OK_2, Result.OK_3, Result.OK_4):
636 c['num_ok'] += 1
637 elif cell_result == Result.PASS:
638 c['num_passed'] += 1
639 if sh_label in OSH_CPYTHON + YSH_CPYTHON:
640 c['oils_num_passed'] += 1
641 else:
642 raise AssertionError()
643
644 def WriteTsv(self, f):
645 f.write('case\tshell\tresult\n')
646 for row in self.tsv_rows:
647 f.write('\t'.join(row))
648 f.write('\n')
649
650
651PIPE = subprocess.PIPE
652
653
654def RunCases(cases, case_predicate, shells, env, out, opts):
655 """Run a list of test 'cases' for all 'shells' and write output to
656 'out'."""
657 if opts.trace:
658 for _, sh in shells:
659 log('\tshell: %s', sh)
660 print('\twhich $SH: ', end='', file=sys.stderr)
661 subprocess.call(['which', sh])
662
663 #pprint.pprint(cases)
664
665 if (isinstance(case_predicate, spec_lib.RangePredicate) and
666 (case_predicate.begin > (len(cases) - 1))):
667 raise RuntimeError(
668 "valid case indexes are from 0 to %s. given range: %s-%s" %
669 ((len(cases) - 1), case_predicate.begin, case_predicate.end))
670
671 sh_labels = [sh_label for sh_label, _ in shells]
672
673 out.WriteHeader(sh_labels)
674 stats = Stats(len(cases), sh_labels)
675
676 # Make an environment for each shell. $SH is the path to the shell, so we
677 # can test flags, etc.
678 sh_env = []
679 for _, sh_path in shells:
680 e = dict(env)
681 e[opts.sh_env_var_name] = sh_path
682 sh_env.append(e)
683
684 # Determine which one (if any) is osh-cpython, for comparison against other
685 # shells.
686 osh_cpython_index = -1
687 for i, (sh_label, _) in enumerate(shells):
688 if sh_label in OSH_CPYTHON:
689 osh_cpython_index = i
690 break
691
692 timeout_dir = os.path.abspath('_tmp/spec/timeouts')
693 try:
694 shutil.rmtree(timeout_dir)
695 os.mkdir(timeout_dir)
696 except OSError:
697 pass
698
699 # Now run each case, and print a table.
700 for i, case in enumerate(cases):
701 line_num = case['line_num']
702 desc = case['desc']
703 code = case['code']
704
705 if opts.trace:
706 log('case %d: %s', i, desc)
707
708 if not case_predicate(i, case):
709 stats.Inc('num_skipped')
710 continue
711
712 if opts.do_print:
713 print('#### %s' % case['desc'])
714 print(case['code'])
715 print()
716 continue
717
718 stats.Inc('num_cases_run')
719
720 result_row = []
721
722 for shell_index, (sh_label, sh_path) in enumerate(shells):
723 timeout_file = os.path.join(timeout_dir, '%02d-%s' % (i, sh_label))
724 if opts.timeout:
725 if opts.timeout_bin:
726 # This is what smoosh itself uses. See smoosh/tests/shell_tests.sh
727 # QUIRK: interval can only be a whole number
728 argv = [
729 opts.timeout_bin,
730 '-t',
731 opts.timeout,
732 # Somehow I'm not able to get this timeout file working? I think
733 # it has a bug when using stdin. It waits for the background
734 # process too.
735
736 #'-i', '1',
737 #'-l', timeout_file
738 ]
739 else:
740 # This kills hanging tests properly, but somehow they fail with code
741 # -9?
742 #argv = ['timeout', '-s', 'KILL', opts.timeout]
743
744 # s suffix for seconds
745 argv = ['timeout', opts.timeout + 's']
746 else:
747 argv = []
748 argv.append(sh_path)
749
750 # dash doesn't support -o posix
751 if opts.posix and sh_label != 'dash':
752 argv.extend(['-o', 'posix'])
753
754 if opts.trace:
755 log('\targv: %s', ' '.join(argv))
756
757 case_env = sh_env[shell_index]
758
759 # Unique dir for every test case and shell
760 tmp_base = os.path.normpath(opts.tmp_env) # no . or ..
761 case_tmp_dir = os.path.join(tmp_base, '%02d-%s' % (i, sh_label))
762
763 try:
764 os.makedirs(case_tmp_dir)
765 except OSError as e:
766 if e.errno != errno.EEXIST:
767 raise
768
769 # Some tests assume _tmp exists
770 # TODO: get rid of this in the common case, to save inodes! I guess have
771 # an opt-in setting per FILE, like make_underscore_tmp: true.
772 try:
773 os.mkdir(os.path.join(case_tmp_dir, '_tmp'))
774 except OSError as e:
775 if e.errno != errno.EEXIST:
776 raise
777
778 case_env['TMP'] = case_tmp_dir
779
780 if opts.pyann_out_dir:
781 case_env = dict(case_env)
782 case_env['PYANN_OUT'] = os.path.join(opts.pyann_out_dir,
783 '%d.json' % i)
784
785 try:
786 p = subprocess.Popen(argv,
787 env=case_env,
788 cwd=case_tmp_dir,
789 stdin=PIPE,
790 stdout=PIPE,
791 stderr=PIPE)
792 except OSError as e:
793 print('Error running %r: %s' % (sh_path, e), file=sys.stderr)
794 sys.exit(1)
795
796 p.stdin.write(code)
797
798 actual = {}
799 actual['stdout'], actual['stderr'] = p.communicate()
800
801 actual['status'] = p.wait()
802
803 if opts.timeout_bin and os.path.exists(timeout_file):
804 cell_result = Result.TIMEOUT
805 elif not opts.timeout_bin and actual['status'] == 124:
806 cell_result = Result.TIMEOUT
807 else:
808 messages = []
809 cell_result = Result.PASS
810
811 # TODO: Warn about no assertions? Well it will always test the error
812 # code.
813 assertions = CreateAssertions(case, sh_label)
814 for a in assertions:
815 result, msg = a.Check(sh_label, actual)
816 # The minimum one wins.
817 # If any failed, then the result is FAIL.
818 # If any are OK, but none are FAIL, the result is OK.
819 cell_result = min(cell_result, result)
820 if msg:
821 messages.append(msg)
822
823 if cell_result != Result.PASS or opts.details:
824 d = (i, sh_label, actual['stdout'], actual['stderr'],
825 messages)
826 out.AddDetails(d)
827
828 result_row.append(cell_result)
829
830 stats.ReportCell(i, cell_result, sh_label)
831
832 if sh_label in OTHER_OSH:
833 # This is only an error if we tried to run ANY OSH.
834 if osh_cpython_index == -1:
835 raise RuntimeError(
836 "Couldn't determine index of osh-cpython")
837
838 other_result = result_row[shell_index]
839 cpython_result = result_row[osh_cpython_index]
840 if other_result != cpython_result:
841 stats.Inc('oils_ALT_delta')
842
843 out.WriteRow(i, line_num, result_row, desc)
844
845 return stats
846
847
848# ANSI color constants
849_RESET = '\033[0;0m'
850_BOLD = '\033[1m'
851
852_RED = '\033[31m'
853_GREEN = '\033[32m'
854_YELLOW = '\033[33m'
855_BLUE = '\033[34m'
856_PURPLE = '\033[35m'
857_CYAN = '\033[36m'
858#_WHITE = '\033[37m'
859
860TEXT_CELLS = {
861 Result.TIMEOUT: 'TIME',
862 Result.FAIL: 'FAIL',
863 Result.BUG: 'BUG',
864 Result.BUG_2: 'BUG-2',
865 Result.NI: 'N-I',
866 Result.OK: 'ok',
867 Result.OK_2: 'ok-2',
868 Result.OK_3: 'ok-3',
869 Result.OK_4: 'ok-4',
870 Result.PASS: 'pass',
871}
872
873ANSI_COLORS = {
874 Result.TIMEOUT: _PURPLE,
875 Result.FAIL: _RED,
876 Result.BUG: _YELLOW,
877 Result.BUG_2: _BLUE,
878 Result.NI: _YELLOW,
879 Result.OK: _YELLOW,
880 Result.OK_2: _BLUE,
881 Result.OK_3: _CYAN,
882 Result.OK_4: _PURPLE,
883 Result.PASS: _GREEN,
884}
885
886
887def _AnsiCells():
888 lookup = {}
889 for i in xrange(Result.length):
890 lookup[i] = ''.join([ANSI_COLORS[i], _BOLD, TEXT_CELLS[i], _RESET])
891 return lookup
892
893
894ANSI_CELLS = _AnsiCells()
895
896HTML_CELLS = {
897 Result.TIMEOUT: '<td class="timeout">TIME',
898 Result.FAIL: '<td class="fail">FAIL',
899 Result.BUG: '<td class="bug">BUG',
900 Result.BUG_2: '<td class="bug-2">BUG-2',
901 Result.NI: '<td class="n-i">N-I',
902 Result.OK: '<td class="ok">ok',
903 Result.OK_2: '<td class="ok-2">ok-2',
904 Result.OK_3: '<td class="ok-3">ok-3',
905 Result.OK_4: '<td class="ok-4">ok-4',
906 Result.PASS: '<td class="pass">pass',
907}
908
909
910def _ValidUtf8String(s):
911 """Return an arbitrary string as a readable utf-8 string.
912
913 We output utf-8 to either HTML or the console. If we get invalid
914 utf-8 as stdout/stderr (which is very possible), then show the ASCII
915 repr().
916 """
917 try:
918 s.decode('utf-8')
919 return s # it decoded OK
920 except UnicodeDecodeError:
921 return repr(s) # ASCII representation
922
923
924class Output(object):
925
926 def __init__(self, f, verbose):
927 self.f = f
928 self.verbose = verbose
929 self.details = []
930
931 def BeginCases(self, test_file):
932 pass
933
934 def WriteHeader(self, sh_labels):
935 pass
936
937 def WriteRow(self, i, line_num, row, desc):
938 pass
939
940 def EndCases(self, sh_labels, stats):
941 pass
942
943 def AddDetails(self, entry):
944 self.details.append(entry)
945
946 # Helper function
947 def _WriteDetailsAsText(self, details):
948 for case_index, shell, stdout, stderr, messages in details:
949 print('case: %d' % case_index, file=self.f)
950 for m in messages:
951 print(m, file=self.f)
952
953 # Assume the terminal can show utf-8, but we don't want random binary.
954 print('%s stdout:' % shell, file=self.f)
955 print(_ValidUtf8String(stdout), file=self.f)
956
957 print('%s stderr:' % shell, file=self.f)
958 print(_ValidUtf8String(stderr), file=self.f)
959
960 print('', file=self.f)
961
962
963class TeeOutput(object):
964 """For multiple outputs in one run, e.g. HTML and TSV.
965
966 UNUSED
967 """
968
969 def __init__(self, outs):
970 self.outs = outs
971
972 def BeginCases(self, test_file):
973 for out in self.outs:
974 out.BeginCases(test_file)
975
976 def WriteHeader(self, sh_labels):
977 for out in self.outs:
978 out.WriteHeader(sh_labels)
979
980 def WriteRow(self, i, line_num, row, desc):
981 for out in self.outs:
982 out.WriteRow(i, line_num, row, desc)
983
984 def EndCases(self, sh_labels, stats):
985 for out in self.outs:
986 out.EndCases(sh_labels, stats)
987
988 def AddDetails(self, entry):
989 for out in self.outs:
990 out.AddDetails(entry)
991
992
993class TsvOutput(Output):
994 """Write a plain-text TSV file.
995
996 UNUSED since we are outputting LONG format with --tsv-output.
997 """
998
999 def WriteHeader(self, sh_labels):
1000 self.f.write('case\tline\t') # case number and line number
1001 for sh_label in sh_labels:
1002 self.f.write(sh_label)
1003 self.f.write('\t')
1004 self.f.write('\n')
1005
1006 def WriteRow(self, i, line_num, row, desc):
1007 self.f.write('%3d\t%3d\t' % (i, line_num))
1008
1009 for result in row:
1010 c = TEXT_CELLS[result]
1011 self.f.write(c)
1012 self.f.write('\t')
1013
1014 # note: 'desc' could use TSV8, but just ignore it for now
1015 #self.f.write(desc)
1016 self.f.write('\n')
1017
1018
1019class AnsiOutput(Output):
1020
1021 def BeginCases(self, test_file):
1022 self.f.write('%s\n' % test_file)
1023
1024 def WriteHeader(self, sh_labels):
1025 self.f.write(_BOLD)
1026 self.f.write('case\tline\t') # case number and line number
1027 for sh_label in sh_labels:
1028 self.f.write(sh_label)
1029 self.f.write('\t')
1030 self.f.write(_RESET)
1031 self.f.write('\n')
1032
1033 def WriteRow(self, i, line_num, row, desc):
1034 self.f.write('%3d\t%3d\t' % (i, line_num))
1035
1036 for result in row:
1037 c = ANSI_CELLS[result]
1038 self.f.write(c)
1039 self.f.write('\t')
1040
1041 self.f.write(desc)
1042 self.f.write('\n')
1043
1044 if self.verbose:
1045 self._WriteDetailsAsText(self.details)
1046 self.details = []
1047
1048 def _WriteShellSummary(self, sh_labels, stats):
1049 if len(stats.nonzero_results) <= 1: # Skip trivial summaries
1050 return
1051
1052 # Reiterate header
1053 self.f.write(_BOLD)
1054 self.f.write('\t\t')
1055 for sh_label in sh_labels:
1056 self.f.write(sh_label)
1057 self.f.write('\t')
1058 self.f.write(_RESET)
1059 self.f.write('\n')
1060
1061 # Write totals by cell.
1062 for result in sorted(stats.nonzero_results, reverse=True):
1063 self.f.write('\t%s' % ANSI_CELLS[result])
1064 for sh_label in sh_labels:
1065 self.f.write('\t%d' % stats.by_shell[sh_label][result])
1066 self.f.write('\n')
1067
1068 # The bottom row is all the same, but it helps readability.
1069 self.f.write('\ttotal')
1070 for sh_label in sh_labels:
1071 self.f.write('\t%d' % stats.counters['num_cases_run'])
1072 self.f.write('\n')
1073 self.f.write('\n')
1074
1075 def EndCases(self, sh_labels, stats):
1076 print()
1077 self._WriteShellSummary(sh_labels, stats)
1078
1079
1080class HtmlOutput(Output):
1081
1082 def __init__(self, f, verbose, spec_name, sh_labels, cases):
1083 Output.__init__(self, f, verbose)
1084 self.spec_name = spec_name
1085 self.sh_labels = sh_labels # saved from header
1086 self.cases = cases # for linking to code
1087 self.row_html = [] # buffered
1088
1089 def _SourceLink(self, line_num, desc):
1090 return '<a href="%s.test.html#L%d">%s</a>' % (self.spec_name, line_num,
1091 cgi.escape(desc))
1092
1093 def BeginCases(self, test_file):
1094 css_urls = ['../../../web/base.css', '../../../web/spec-tests.css']
1095 title = '%s: spec test case results' % self.spec_name
1096 html_head.Write(self.f, title, css_urls=css_urls)
1097
1098 self.f.write('''\
1099 <body class="width60">
1100 <p id="home-link">
1101 <a href=".">spec test index</a>
1102 /
1103 <a href="/">oils.pub</a>
1104 </p>
1105 <h1>Results for %s</h1>
1106 <table>
1107 ''' % test_file)
1108
1109 def _WriteShellSummary(self, sh_labels, stats):
1110 # NOTE: This table has multiple <thead>, which seems OK.
1111 self.f.write('''
1112<thead>
1113 <tr class="table-header">
1114 ''')
1115
1116 columns = ['status'] + sh_labels + ['']
1117 for c in columns:
1118 self.f.write('<td>%s</td>' % c)
1119
1120 self.f.write('''
1121 </tr>
1122</thead>
1123''')
1124
1125 # Write totals by cell.
1126 for result in sorted(stats.nonzero_results, reverse=True):
1127 self.f.write('<tr>')
1128
1129 self.f.write(HTML_CELLS[result])
1130 self.f.write('</td> ')
1131
1132 for sh_label in sh_labels:
1133 self.f.write('<td>%d</td>' % stats.by_shell[sh_label][result])
1134
1135 self.f.write('<td></td>')
1136 self.f.write('</tr>\n')
1137
1138 # The bottom row is all the same, but it helps readability.
1139 self.f.write('<tr>')
1140 self.f.write('<td>total</td>')
1141 for sh_label in sh_labels:
1142 self.f.write('<td>%d</td>' % stats.counters['num_cases_run'])
1143 self.f.write('<td></td>')
1144 self.f.write('</tr>\n')
1145
1146 # Blank row for space.
1147 self.f.write('<tr>')
1148 for i in xrange(len(sh_labels) + 2):
1149 self.f.write('<td style="height: 2em"></td>')
1150 self.f.write('</tr>\n')
1151
1152 def WriteHeader(self, sh_labels):
1153 f = cStringIO.StringIO()
1154
1155 f.write('''
1156<thead>
1157 <tr class="table-header">
1158 ''')
1159
1160 columns = ['case'] + sh_labels
1161 for c in columns:
1162 f.write('<td>%s</td>' % c)
1163 f.write('<td class="case-desc">description</td>')
1164
1165 f.write('''
1166 </tr>
1167</thead>
1168''')
1169
1170 self.row_html.append(f.getvalue())
1171
1172 def WriteRow(self, i, line_num, row, desc):
1173 f = cStringIO.StringIO()
1174 f.write('<tr>')
1175 f.write('<td>%3d</td>' % i)
1176
1177 show_details = False
1178
1179 for result in row:
1180 c = HTML_CELLS[result]
1181 if result not in (Result.PASS, Result.TIMEOUT): # nothing to show
1182 show_details = True
1183
1184 f.write(c)
1185 f.write('</td>')
1186 f.write('\t')
1187
1188 f.write('<td class="case-desc">')
1189 f.write(self._SourceLink(line_num, desc))
1190 f.write('</td>')
1191 f.write('</tr>\n')
1192
1193 # Show row with details link.
1194 if show_details:
1195 f.write('<tr>')
1196 f.write('<td class="details-row"></td>') # for the number
1197
1198 for col_index, result in enumerate(row):
1199 f.write('<td class="details-row">')
1200 if result != Result.PASS:
1201 sh_label = self.sh_labels[col_index]
1202 f.write('<a href="#details-%s-%s">details</a>' %
1203 (i, sh_label))
1204 f.write('</td>')
1205
1206 f.write('<td class="details-row"></td>') # for the description
1207 f.write('</tr>\n')
1208
1209 self.row_html.append(f.getvalue()) # buffer it
1210
1211 def _WriteStats(self, stats):
1212 self.f.write('%(num_passed)d passed, %(num_ok)d OK, '
1213 '%(num_ni)d not implemented, %(num_bug)d BUG, '
1214 '%(num_failed)d failed, %(num_timeout)d timeouts, '
1215 '%(num_skipped)d cases skipped\n' % stats.counters)
1216
1217 def EndCases(self, sh_labels, stats):
1218 self._WriteShellSummary(sh_labels, stats)
1219
1220 # Write all the buffered rows
1221 for h in self.row_html:
1222 self.f.write(h)
1223
1224 self.f.write('</table>\n')
1225 self.f.write('<pre>')
1226 self._WriteStats(stats)
1227 if stats.Get('oils_num_failed'):
1228 self.f.write('%(oils_num_failed)d failed under osh\n' %
1229 stats.counters)
1230 self.f.write('</pre>')
1231
1232 if self.details:
1233 self._WriteDetails()
1234
1235 self.f.write('</body></html>')
1236
1237 def _WriteDetails(self):
1238 self.f.write("<h2>Details on runs that didn't PASS</h2>")
1239 self.f.write('<table id="details">')
1240
1241 for case_index, sh_label, stdout, stderr, messages in self.details:
1242 self.f.write('<tr>')
1243 self.f.write('<td><a name="details-%s-%s"></a><b>%s</b></td>' %
1244 (case_index, sh_label, sh_label))
1245
1246 self.f.write('<td>')
1247
1248 # Write description and link to the code
1249 case = self.cases[case_index]
1250 line_num = case['line_num']
1251 desc = case['desc']
1252 self.f.write('%d ' % case_index)
1253 self.f.write(self._SourceLink(line_num, desc))
1254 self.f.write('<br/><br/>\n')
1255
1256 for m in messages:
1257 self.f.write('<span class="assertion">%s</span><br/>\n' %
1258 cgi.escape(m))
1259 if messages:
1260 self.f.write('<br/>\n')
1261
1262 def _WriteRaw(s):
1263 self.f.write('<pre>')
1264
1265 # stdout might contain invalid utf-8; make it valid;
1266 valid_utf8 = _ValidUtf8String(s)
1267
1268 self.f.write(cgi.escape(valid_utf8))
1269 self.f.write('</pre>')
1270
1271 self.f.write('<i>stdout:</i> <br/>\n')
1272 _WriteRaw(stdout)
1273
1274 self.f.write('<i>stderr:</i> <br/>\n')
1275 _WriteRaw(stderr)
1276
1277 self.f.write('</td>')
1278 self.f.write('</tr>')
1279
1280 self.f.write('</table>')
1281
1282
1283def MakeTestEnv(opts):
1284 if not opts.tmp_env:
1285 raise RuntimeError('--tmp-env required')
1286 if not opts.path_env:
1287 raise RuntimeError('--path-env required')
1288 env = {
1289 'PATH': opts.path_env,
1290 #'LANG': opts.lang_env,
1291 }
1292 for p in opts.env_pair:
1293 name, value = p.split('=', 1)
1294 env[name] = value
1295
1296 return env
1297
1298
1299def _DefaultSuite(spec_name):
1300 if spec_name.startswith('ysh-'):
1301 suite = 'ysh'
1302 elif spec_name.startswith('hay'): # hay.test.sh is ysh
1303 suite = 'ysh'
1304
1305 elif spec_name.startswith('tea-'):
1306 suite = 'tea'
1307 else:
1308 suite = 'osh'
1309
1310 return suite
1311
1312
1313def ParseTestList(test_files):
1314 for test_file in test_files:
1315 with open(test_file) as f:
1316 tokens = Tokenizer(f)
1317 try:
1318 file_metadata, cases = ParseTestFile(test_file, tokens)
1319 except RuntimeError as e:
1320 log('ERROR in %r', test_file)
1321 raise
1322 except ParseError as e:
1323 log('PARSE ERROR in %r', test_file)
1324 raise
1325
1326 tmp = os.path.basename(test_file)
1327 spec_name = tmp.split('.')[0] # foo.test.sh -> foo
1328
1329 suite = file_metadata.get('suite') or _DefaultSuite(spec_name)
1330
1331 tmp = file_metadata.get('tags')
1332 tags = tmp.split() if tmp else []
1333
1334 # Don't need compare_shells, etc. to decide what to run
1335
1336 row = {'spec_name': spec_name, 'suite': suite, 'tags': tags}
1337 #print(row)
1338 yield row
1339
1340
1341def main(argv):
1342 # First check if bash is polluting the environment. Tests rely on the
1343 # environment.
1344 v = os.getenv('RANDOM')
1345 if v is not None:
1346 raise AssertionError('got $RANDOM = %s' % v)
1347 v = os.getenv('PPID')
1348 if v is not None:
1349 raise AssertionError('got $PPID = %s' % v)
1350
1351 p = optparse.OptionParser('%s [options] TEST_FILE shell...' % sys.argv[0])
1352 spec_lib.DefineCommon(p)
1353 spec_lib.DefineShSpec(p)
1354 opts, argv = p.parse_args(argv)
1355
1356 # --print-tagged to figure out what to run
1357 if opts.print_tagged:
1358 to_find = opts.print_tagged
1359 for row in ParseTestList(argv[1:]):
1360 if to_find in row['tags']:
1361 print(row['spec_name'])
1362 return 0
1363
1364 # --print-table to figure out what to run
1365 if opts.print_table:
1366 for row in ParseTestList(argv[1:]):
1367 print('%(suite)s\t%(spec_name)s' % row)
1368 #print(row)
1369 return 0
1370
1371 #
1372 # Now deal with a single file
1373 #
1374
1375 try:
1376 test_file = argv[1]
1377 except IndexError:
1378 p.print_usage()
1379 return 1
1380
1381 with open(test_file) as f:
1382 tokens = Tokenizer(f)
1383 file_metadata, cases = ParseTestFile(test_file, tokens)
1384
1385 # List test cases and return
1386 if opts.do_list:
1387 for i, case in enumerate(cases):
1388 if opts.verbose: # print the raw dictionary for debugging
1389 print(pprint.pformat(case))
1390 else:
1391 print('%d\t%s' % (i, case['desc']))
1392 return 0
1393
1394 # for test/spec-cpp.sh
1395 if opts.print_spec_suite:
1396 tmp = os.path.basename(test_file)
1397 spec_name = tmp.split('.')[0] # foo.test.sh -> foo
1398
1399 suite = file_metadata.get('suite') or _DefaultSuite(spec_name)
1400 print(suite)
1401 return 0
1402
1403 if opts.verbose:
1404 for k, v in file_metadata.items():
1405 print('\t%-20s: %s' % (k, v), file=sys.stderr)
1406 print('', file=sys.stderr)
1407
1408 if opts.oils_bin_dir: # to find OSH and YSH
1409 shells = []
1410
1411 if opts.compare_shells:
1412 comp = file_metadata.get('compare_shells')
1413 # Compare 'compare_shells' and Python
1414 shells.extend(comp.split() if comp else [])
1415
1416 # Always run with the Python version
1417 our_shell = file_metadata.get('our_shell', 'osh') # default is OSH
1418 if our_shell != '-':
1419 shells.append(os.path.join(opts.oils_bin_dir, our_shell))
1420
1421 # Legacy OVM/CPython build
1422 if opts.ovm_bin_dir:
1423 shells.append(os.path.join(opts.ovm_bin_dir, our_shell))
1424
1425 # New C++ build
1426 if opts.oils_cpp_bin_dir:
1427 shells.append(os.path.join(opts.oils_cpp_bin_dir, our_shell))
1428
1429 # Overwrite it when --oils-bin-dir is set
1430 # It's no longer a flag
1431 opts.oils_failures_allowed = \
1432 int(file_metadata.get('oils_failures_allowed', 0))
1433
1434 else:
1435 # To compare arbitrary shells
1436 shells = argv[2:]
1437
1438 shell_pairs = spec_lib.MakeShellPairs(shells)
1439
1440 if opts.range:
1441 begin, end = spec_lib.ParseRange(opts.range)
1442 case_predicate = spec_lib.RangePredicate(begin, end)
1443 elif opts.regex:
1444 desc_re = re.compile(opts.regex, re.IGNORECASE)
1445 case_predicate = spec_lib.RegexPredicate(desc_re)
1446 else:
1447 case_predicate = lambda i, case: True
1448
1449 out_f = sys.stderr if opts.do_print else sys.stdout
1450
1451 # Set up output style. Also see asdl/format.py
1452 if opts.format == 'ansi':
1453 out = AnsiOutput(out_f, opts.verbose)
1454
1455 elif opts.format == 'html':
1456 spec_name = os.path.basename(test_file)
1457 spec_name = spec_name.split('.')[0]
1458
1459 sh_labels = [label for label, _ in shell_pairs]
1460
1461 out = HtmlOutput(out_f, opts.verbose, spec_name, sh_labels, cases)
1462
1463 else:
1464 raise AssertionError()
1465
1466 out.BeginCases(os.path.basename(test_file))
1467
1468 env = MakeTestEnv(opts)
1469 stats = RunCases(cases, case_predicate, shell_pairs, env, out, opts)
1470
1471 out.EndCases([sh_label for sh_label, _ in shell_pairs], stats)
1472
1473 if opts.tsv_output:
1474 with open(opts.tsv_output, 'w') as f:
1475 stats.WriteTsv(f)
1476
1477 # TODO: Could --stats-{file,template} be a separate awk step on .tsv files?
1478 stats.Set('oils_failures_allowed', opts.oils_failures_allowed)
1479
1480 # If it's not set separately for C++, we default to the allowed number
1481 # above
1482 x = int(
1483 file_metadata.get('oils_cpp_failures_allowed',
1484 opts.oils_failures_allowed))
1485 stats.Set('oils_cpp_failures_allowed', x)
1486
1487 if opts.stats_file:
1488 with open(opts.stats_file, 'w') as f:
1489 f.write(opts.stats_template % stats.counters)
1490 f.write('\n') # bash 'read' requires a newline
1491
1492 # spec/smoke.test.sh -> smoke
1493 test_name = os.path.basename(test_file).split('.')[0]
1494
1495 return _SuccessOrFailure(test_name, stats)
1496
1497
1498def _SuccessOrFailure(test_name, stats):
1499 allowed = stats.Get('oils_failures_allowed')
1500 allowed_cpp = stats.Get('oils_cpp_failures_allowed')
1501
1502 all_count = stats.Get('num_failed')
1503 oils_count = stats.Get('oils_num_failed')
1504 oils_cpp_count = stats.Get('oils_cpp_num_failed')
1505
1506 errors = []
1507 if oils_count != allowed:
1508 errors.append('Got %d Oils failures, but %d are allowed' %
1509 (oils_count, allowed))
1510 else:
1511 if allowed != 0:
1512 log('%s: note: Got %d allowed Oils failures', test_name, allowed)
1513
1514 # TODO: remove special case for 0
1515 if oils_cpp_count != 0:
1516 if oils_cpp_count != allowed_cpp:
1517 errors.append('Got %d Oils C++ failures, but %d are allowed' %
1518 (oils_cpp_count, allowed_cpp))
1519 else:
1520 if allowed_cpp != 0:
1521 log('%s: note: Got %d allowed Oils C++ failures', test_name,
1522 allowed_cpp)
1523
1524 if all_count != allowed:
1525 errors.append('Got %d total failures, but %d are allowed' %
1526 (all_count, allowed))
1527
1528 if errors:
1529 for msg in errors:
1530 log('%s: FATAL: %s', test_name, msg)
1531 return 1
1532
1533 return 0
1534
1535
1536if __name__ == '__main__':
1537 try:
1538 sys.exit(main(sys.argv))
1539 except KeyboardInterrupt as e:
1540 print('%s: interrupted with Ctrl-C' % sys.argv[0], file=sys.stderr)
1541 sys.exit(1)
1542 except RuntimeError as e:
1543 print('FATAL: %s' % e, file=sys.stderr)
1544 sys.exit(1)