OILS / test / sh_spec.py View on Github | oilshell.org

1509 lines, 934 significant
1#!/usr/bin/env python2
2from __future__ import print_function
3"""
4sh_spec.py -- Test framework to compare shells.
5
6Assertion help:
7 stdout: A single line of expected stdout. Newline is implicit.
8 stdout-json: JSON-encoded string. Use for the empty string (no newline),
9 for unicode chars, etc.
10
11 stderr: Ditto for stderr stream.
12 status: Expected shell return code. If not specified, the case must exit 0.
13
14Results:
15 PASS - we got the ideal, expected value
16 OK - we got a value that was not ideal, but expected
17 For OSH this is behavior that was defined to be different?
18 N-I - Not implemented (e.g. $''). Assertions still checked (in case it
19 starts working)
20 BUG - we verified the value of a known bug
21 FAIL - we got an unexpected value. If the implementation can't be changed,
22 it should be converted to BUG or OK. Otherwise it should be made to
23 PASS.
24
25NOTE: The difference between OK and BUG is a matter of judgement. If the ideal
26behavior is a compile time error (code 2), a runtime error is generally OK.
27
28If ALL shells agree on a broken behavior, they are all marked OK (but our
29implementation will be PASS.) But if the behavior is NOT POSIX compliant, then
30it will be a BUG.
31
32If one shell disagrees with others, that is generally a BUG.
33
34Example test case:
35
36#### hello and fail
37echo hello
38echo world
39exit 1
40## status: 1
41#
42# ignored comment
43#
44## STDOUT:
45hello
46world
47## END
48
49"""
50
51import collections
52import cgi
53import cStringIO
54import errno
55import json
56import optparse
57import os
58import pprint
59import re
60import shutil
61import subprocess
62import sys
63
64from test import spec_lib
65from doctools import html_head
66
67log = spec_lib.log
68
69# Magic strings for other variants of OSH.
70
71# NOTE: osh_ALT is usually _bin/osh -- the release binary.
72# It would be better to rename these osh-cpython and osh-ovm. Have the concept
73# of a suffix?
74
75OSH_CPYTHON = ('osh', 'osh-dbg')
76OTHER_OSH = ('osh_ALT', )
77
78YSH_CPYTHON = ('ysh', 'ysh-dbg')
79OTHER_YSH = ('oil_ALT', )
80
81# For now, only count the Oils CPython failures. TODO: the spec-cpp job should
82# assert the osh-cpp and ysh-cpp deltas.
83OTHER_OILS = OTHER_OSH + OTHER_YSH + ('osh-cpp', 'ysh-cpp')
84
85
86class ParseError(Exception):
87 pass
88
89
90# EXAMPLES:
91## stdout: foo
92## stdout-json: ""
93#
94# In other words, it could be (name, value) or (qualifier, name, value)
95
96KEY_VALUE_RE = re.compile(
97 r'''
98 [#][#] \s+
99 # optional prefix with qualifier and shells
100 (?: (OK|BUG|N-I) \s+ ([\w+/]+) \s+ )?
101 ([\w\-]+) # key
102 :
103 \s* (.*) # value
104''', re.VERBOSE)
105
106END_MULTILINE_RE = re.compile(r'''
107 [#][#] \s+ END
108''', re.VERBOSE)
109
110# Line types
111TEST_CASE_BEGIN = 0 # Starts with ####
112KEY_VALUE = 1 # Metadata
113KEY_VALUE_MULTILINE = 2 # STDOUT STDERR
114END_MULTILINE = 3 # STDOUT STDERR
115PLAIN_LINE = 4 # Uncommented
116EOF = 5
117
118LEX_OUTER = 0 # Ignore blank lines, e.g. for separating cases
119LEX_RAW = 1 # Blank lines are significant
120
121
122class Tokenizer(object):
123 """Modal lexer!"""
124
125 def __init__(self, f):
126 self.f = f
127
128 self.cursor = None
129 self.line_num = 0
130
131 self.next()
132
133 def _ClassifyLine(self, line, lex_mode):
134 if not line: # empty
135 return self.line_num, EOF, ''
136
137 if lex_mode == LEX_OUTER and not line.strip():
138 return None
139
140 if line.startswith('####'):
141 desc = line[4:].strip()
142 return self.line_num, TEST_CASE_BEGIN, desc
143
144 m = KEY_VALUE_RE.match(line)
145 if m:
146 qualifier, shells, name, value = m.groups()
147 # HACK: Expected data should have the newline.
148 if name in ('stdout', 'stderr'):
149 value += '\n'
150
151 if name in ('STDOUT', 'STDERR'):
152 token_type = KEY_VALUE_MULTILINE
153 else:
154 token_type = KEY_VALUE
155 return self.line_num, token_type, (qualifier, shells, name, value)
156
157 m = END_MULTILINE_RE.match(line)
158 if m:
159 return self.line_num, END_MULTILINE, None
160
161 # If it starts with ##, it should be metadata. This finds some typos.
162 if line.lstrip().startswith('##'):
163 raise RuntimeError('Invalid ## line %r' % line)
164
165 if line.lstrip().startswith('#'): # Ignore comments
166 return None # try again
167
168 # Non-empty line that doesn't start with '#'
169 # NOTE: We need the original line to test the whitespace sensitive <<-.
170 # And we need rstrip because we add newlines back below.
171 return self.line_num, PLAIN_LINE, line
172
173 def next(self, lex_mode=LEX_OUTER):
174 """Raises StopIteration when exhausted."""
175 while True:
176 line = self.f.readline()
177 self.line_num += 1
178
179 tok = self._ClassifyLine(line, lex_mode)
180 if tok is not None:
181 break
182
183 self.cursor = tok
184 return self.cursor
185
186 def peek(self):
187 return self.cursor
188
189
190def AddMetadataToCase(case, qualifier, shells, name, value, line_num):
191 shells = shells.split('/') # bash/dash/mksh
192 for shell in shells:
193 if shell not in case:
194 case[shell] = {}
195
196 # Check a duplicate specification
197 name_without_type = re.sub(r'-(json|repr)$', '', name)
198 if (name_without_type in case[shell] or
199 name_without_type + '-json' in case[shell] or
200 name_without_type + '-repr' in case[shell]):
201 raise ParseError('Line %d: duplicate spec %r for %r' %
202 (line_num, name, shell))
203
204 # Check inconsistent qualifier
205 if 'qualifier' in case[shell] and qualifier != case[shell]['qualifier']:
206 raise ParseError(
207 'Line %d: inconsistent qualifier %r is specified for %r, '
208 'but %r was previously specified. '
209 % (line_num, qualifier, shell, case[shell]['qualifier']))
210
211 case[shell][name] = value
212 case[shell]['qualifier'] = qualifier
213
214
215# Format of a test script.
216#
217# -- Code is either literal lines, or a commented out code: value.
218# code = PLAIN_LINE*
219# | '## code:' VALUE
220#
221# -- Key value pairs can be single- or multi-line
222# key_value = '##' KEY ':' VALUE
223# | KEY_VALUE_MULTILINE PLAIN_LINE* END_MULTILINE
224#
225# -- Description, then key-value pairs surrounding code.
226# test_case = '####' DESC
227# key_value*
228# code
229# key_value*
230#
231# -- Should be a blank line after each test case. Leading comments and code
232# -- are OK.
233#
234# test_file =
235# key_value* -- file level metadata
236# (test_case '\n')*
237
238
239def ParseKeyValue(tokens, case):
240 """Parse commented-out metadata in a test case.
241
242 The metadata must be contiguous.
243
244 Args:
245 tokens: Tokenizer
246 case: dictionary to add to
247 """
248 while True:
249 line_num, kind, item = tokens.peek()
250
251 if kind == KEY_VALUE_MULTILINE:
252 qualifier, shells, name, empty_value = item
253 if empty_value:
254 raise ParseError(
255 'Line %d: got value %r for %r, but the value should be on the '
256 'following lines' % (line_num, empty_value, name))
257
258 value_lines = []
259 while True:
260 tokens.next(lex_mode=LEX_RAW) # empty lines aren't skipped
261 _, kind2, item2 = tokens.peek()
262 if kind2 != PLAIN_LINE:
263 break
264 value_lines.append(item2)
265
266 value = ''.join(value_lines)
267
268 name = name.lower() # STDOUT -> stdout
269 if qualifier:
270 AddMetadataToCase(case, qualifier, shells, name, value,
271 line_num)
272 else:
273 case[name] = value
274
275 # END token is optional.
276 if kind2 == END_MULTILINE:
277 tokens.next()
278
279 elif kind == KEY_VALUE:
280 qualifier, shells, name, value = item
281
282 if qualifier:
283 AddMetadataToCase(case, qualifier, shells, name, value,
284 line_num)
285 else:
286 case[name] = value
287
288 tokens.next()
289
290 else: # Unknown token type
291 break
292
293
294def ParseCodeLines(tokens, case):
295 """Parse uncommented code in a test case."""
296 _, kind, item = tokens.peek()
297 if kind != PLAIN_LINE:
298 raise ParseError('Expected a line of code (got %r, %r)' % (kind, item))
299 code_lines = []
300 while True:
301 _, kind, item = tokens.peek()
302 if kind != PLAIN_LINE:
303 case['code'] = ''.join(code_lines)
304 return
305 code_lines.append(item)
306 tokens.next(lex_mode=LEX_RAW)
307
308
309def ParseTestCase(tokens):
310 """Parse a single test case and return it.
311
312 If at EOF, return None.
313 """
314 line_num, kind, item = tokens.peek()
315 if kind == EOF:
316 return None
317
318 if kind != TEST_CASE_BEGIN:
319 raise RuntimeError("line %d: Expected TEST_CASE_BEGIN, got %r" %
320 (line_num, [kind, item]))
321
322 tokens.next()
323
324 case = {'desc': item, 'line_num': line_num}
325
326 ParseKeyValue(tokens, case)
327
328 # For broken code
329 if 'code' in case: # Got it through a key value pair
330 return case
331
332 ParseCodeLines(tokens, case)
333 ParseKeyValue(tokens, case)
334
335 return case
336
337
338_META_FIELDS = [
339 'our_shell',
340 'compare_shells',
341 'suite',
342 'tags',
343 'oils_failures_allowed',
344 'oils_cpp_failures_allowed',
345]
346
347
348def ParseTestFile(test_file, tokens):
349 """
350 test_file: Only for error message
351 """
352 file_metadata = {}
353 test_cases = []
354
355 try:
356 # Skip over the header. Setup code can go here, although would we have to
357 # execute it on every case?
358 while True:
359 line_num, kind, item = tokens.peek()
360 if kind != KEY_VALUE:
361 break
362
363 qualifier, shells, name, value = item
364 if qualifier is not None:
365 raise RuntimeError('Invalid qualifier in spec file metadata')
366 if shells is not None:
367 raise RuntimeError('Invalid shells in spec file metadata')
368
369 file_metadata[name] = value
370
371 tokens.next()
372
373 while True: # Loop over cases
374 test_case = ParseTestCase(tokens)
375 if test_case is None:
376 break
377 test_cases.append(test_case)
378
379 except StopIteration:
380 raise RuntimeError('Unexpected EOF parsing test cases')
381
382 for name in file_metadata:
383 if name not in _META_FIELDS:
384 raise RuntimeError('Invalid file metadata %r in %r' %
385 (name, test_file))
386
387 return file_metadata, test_cases
388
389
390def CreateStringAssertion(d, key, assertions, qualifier=False):
391 found = False
392
393 exp = d.get(key)
394 if exp is not None:
395 a = EqualAssertion(key, exp, qualifier=qualifier)
396 assertions.append(a)
397 found = True
398
399 exp_json = d.get(key + '-json')
400 if exp_json is not None:
401 exp = json.loads(exp_json, encoding='utf-8')
402 a = EqualAssertion(key, exp, qualifier=qualifier)
403 assertions.append(a)
404 found = True
405
406 # For testing invalid unicode
407 exp_repr = d.get(key + '-repr')
408 if exp_repr is not None:
409 exp = eval(exp_repr)
410 a = EqualAssertion(key, exp, qualifier=qualifier)
411 assertions.append(a)
412 found = True
413
414 return found
415
416
417def CreateIntAssertion(d, key, assertions, qualifier=False):
418 exp = d.get(key) # expected
419 if exp is not None:
420 # For now, turn it into int
421 a = EqualAssertion(key, int(exp), qualifier=qualifier)
422 assertions.append(a)
423 return True
424 return False
425
426
427def CreateAssertions(case, sh_label):
428 """Given a raw test case and a shell label, create EqualAssertion instances
429 to run."""
430 assertions = []
431
432 # Whether we found assertions
433 stdout = False
434 stderr = False
435 status = False
436
437 # So the assertion are exactly the same for osh and osh_ALT
438
439 if sh_label.startswith('osh'):
440 case_sh = 'osh'
441 elif sh_label.startswith('bash'):
442 case_sh = 'bash'
443 else:
444 case_sh = sh_label
445
446 if case_sh in case:
447 q = case[case_sh]['qualifier']
448 if CreateStringAssertion(case[case_sh],
449 'stdout',
450 assertions,
451 qualifier=q):
452 stdout = True
453 if CreateStringAssertion(case[case_sh],
454 'stderr',
455 assertions,
456 qualifier=q):
457 stderr = True
458 if CreateIntAssertion(case[case_sh], 'status', assertions,
459 qualifier=q):
460 status = True
461
462 if not stdout:
463 CreateStringAssertion(case, 'stdout', assertions)
464 if not stderr:
465 CreateStringAssertion(case, 'stderr', assertions)
466 if not status:
467 if 'status' in case:
468 CreateIntAssertion(case, 'status', assertions)
469 else:
470 # If the user didn't specify a 'status' assertion, assert that the exit
471 # code is 0.
472 a = EqualAssertion('status', 0)
473 assertions.append(a)
474
475 no_traceback = SubstringAssertion('stderr', 'Traceback (most recent')
476 assertions.append(no_traceback)
477
478 #print 'SHELL', shell
479 #pprint.pprint(case)
480 #print(assertions)
481 return assertions
482
483
484class Result(object):
485 """Result of an stdout/stderr/status assertion or of a (case, shell) cell.
486
487 Order is important: the result of a cell is the minimum of the results of
488 each assertion.
489 """
490 TIMEOUT = 0 # ONLY a cell result, not an assertion result
491 FAIL = 1
492 BUG = 2
493 NI = 3
494 OK = 4
495 PASS = 5
496
497 length = 6 # for loops
498
499
500def QualifierToResult(qualifier):
501 # type: (str) -> Result
502 if qualifier == 'BUG': # equal, but known bad
503 return Result.BUG
504 if qualifier == 'N-I': # equal, and known UNIMPLEMENTED
505 return Result.NI
506 if qualifier == 'OK': # equal, but ok (not ideal)
507 return Result.OK
508 return Result.PASS # ideal behavior
509
510
511class EqualAssertion(object):
512 """Check that two values are equal."""
513
514 def __init__(self, key, expected, qualifier=None):
515 self.key = key
516 self.expected = expected # expected value
517 self.qualifier = qualifier # whether this was a special case?
518
519 def __repr__(self):
520 return '<EqualAssertion %s == %r>' % (self.key, self.expected)
521
522 def Check(self, shell, record):
523 actual = record[self.key]
524 if actual != self.expected:
525 if len(str(self.expected)) < 40:
526 msg = '[%s %s] Expected %r, got %r' % (shell, self.key,
527 self.expected, actual)
528 else:
529 msg = '''
530[%s %s]
531Expected %r
532Got %r
533''' % (shell, self.key, self.expected, actual)
534
535 # TODO: Make this better and add a flag for it.
536 if 0:
537 import difflib
538 for line in difflib.unified_diff(self.expected,
539 actual,
540 fromfile='expected',
541 tofile='actual'):
542 print(repr(line))
543
544 return Result.FAIL, msg
545
546 return QualifierToResult(self.qualifier), ''
547
548
549class SubstringAssertion(object):
550 """Check that a string like stderr doesn't have a substring."""
551
552 def __init__(self, key, substring):
553 self.key = key
554 self.substring = substring
555
556 def __repr__(self):
557 return '<SubstringAssertion %s == %r>' % (self.key, self.substring)
558
559 def Check(self, shell, record):
560 actual = record[self.key]
561 if self.substring in actual:
562 msg = '[%s %s] Found %r' % (shell, self.key, self.substring)
563 return Result.FAIL, msg
564 return Result.PASS, ''
565
566
567class Stats(object):
568
569 def __init__(self, num_cases, sh_labels):
570 self.counters = collections.defaultdict(int)
571 c = self.counters
572 c['num_cases'] = num_cases
573 c['oils_num_passed'] = 0
574 c['oils_num_failed'] = 0
575 c['oils_cpp_num_failed'] = 0
576 # Number of osh_ALT results that differed from osh.
577 c['oils_ALT_delta'] = 0
578
579 self.by_shell = {}
580 for sh in sh_labels:
581 self.by_shell[sh] = collections.defaultdict(int)
582 self.nonzero_results = collections.defaultdict(int)
583
584 self.tsv_rows = []
585
586 def Inc(self, counter_name):
587 self.counters[counter_name] += 1
588
589 def Get(self, counter_name):
590 return self.counters[counter_name]
591
592 def Set(self, counter_name, val):
593 self.counters[counter_name] = val
594
595 def ReportCell(self, case_num, cell_result, sh_label):
596 self.tsv_rows.append(
597 (str(case_num), sh_label, TEXT_CELLS[cell_result]))
598
599 self.by_shell[sh_label][cell_result] += 1
600 self.nonzero_results[cell_result] += 1
601
602 c = self.counters
603 if cell_result == Result.TIMEOUT:
604 c['num_timeout'] += 1
605 elif cell_result == Result.FAIL:
606 # Special logic: don't count osh_ALT because its failures will be
607 # counted in the delta.
608 if sh_label not in OTHER_OILS:
609 c['num_failed'] += 1
610
611 if sh_label in OSH_CPYTHON + YSH_CPYTHON:
612 c['oils_num_failed'] += 1
613
614 if sh_label in ('osh-cpp', 'ysh-cpp'):
615 c['oils_cpp_num_failed'] += 1
616 elif cell_result == Result.BUG:
617 c['num_bug'] += 1
618 elif cell_result == Result.NI:
619 c['num_ni'] += 1
620 elif cell_result == Result.OK:
621 c['num_ok'] += 1
622 elif cell_result == Result.PASS:
623 c['num_passed'] += 1
624 if sh_label in OSH_CPYTHON + YSH_CPYTHON:
625 c['oils_num_passed'] += 1
626 else:
627 raise AssertionError()
628
629 def WriteTsv(self, f):
630 f.write('case\tshell\tresult\n')
631 for row in self.tsv_rows:
632 f.write('\t'.join(row))
633 f.write('\n')
634
635
636PIPE = subprocess.PIPE
637
638
639def RunCases(cases, case_predicate, shells, env, out, opts):
640 """Run a list of test 'cases' for all 'shells' and write output to
641 'out'."""
642 if opts.trace:
643 for _, sh in shells:
644 log('\tshell: %s', sh)
645 print('\twhich $SH: ', end='', file=sys.stderr)
646 subprocess.call(['which', sh])
647
648 #pprint.pprint(cases)
649
650 sh_labels = [sh_label for sh_label, _ in shells]
651
652 out.WriteHeader(sh_labels)
653 stats = Stats(len(cases), sh_labels)
654
655 # Make an environment for each shell. $SH is the path to the shell, so we
656 # can test flags, etc.
657 sh_env = []
658 for _, sh_path in shells:
659 e = dict(env)
660 e[opts.sh_env_var_name] = sh_path
661 sh_env.append(e)
662
663 # Determine which one (if any) is osh-cpython, for comparison against other
664 # shells.
665 osh_cpython_index = -1
666 for i, (sh_label, _) in enumerate(shells):
667 if sh_label in OSH_CPYTHON:
668 osh_cpython_index = i
669 break
670
671 timeout_dir = os.path.abspath('_tmp/spec/timeouts')
672 try:
673 shutil.rmtree(timeout_dir)
674 os.mkdir(timeout_dir)
675 except OSError:
676 pass
677
678 # Now run each case, and print a table.
679 for i, case in enumerate(cases):
680 line_num = case['line_num']
681 desc = case['desc']
682 code = case['code']
683
684 if opts.trace:
685 log('case %d: %s', i, desc)
686
687 if not case_predicate(i, case):
688 stats.Inc('num_skipped')
689 continue
690
691 if opts.do_print:
692 print('#### %s' % case['desc'])
693 print(case['code'])
694 print()
695 continue
696
697 stats.Inc('num_cases_run')
698
699 result_row = []
700
701 for shell_index, (sh_label, sh_path) in enumerate(shells):
702 timeout_file = os.path.join(timeout_dir, '%02d-%s' % (i, sh_label))
703 if opts.timeout:
704 if opts.timeout_bin:
705 # This is what smoosh itself uses. See smoosh/tests/shell_tests.sh
706 # QUIRK: interval can only be a whole number
707 argv = [
708 opts.timeout_bin,
709 '-t',
710 opts.timeout,
711 # Somehow I'm not able to get this timeout file working? I think
712 # it has a bug when using stdin. It waits for the background
713 # process too.
714
715 #'-i', '1',
716 #'-l', timeout_file
717 ]
718 else:
719 # This kills hanging tests properly, but somehow they fail with code
720 # -9?
721 #argv = ['timeout', '-s', 'KILL', opts.timeout]
722
723 # s suffix for seconds
724 argv = ['timeout', opts.timeout + 's']
725 else:
726 argv = []
727 argv.append(sh_path)
728
729 # dash doesn't support -o posix
730 if opts.posix and sh_label != 'dash':
731 argv.extend(['-o', 'posix'])
732
733 if opts.trace:
734 log('\targv: %s', ' '.join(argv))
735
736 case_env = sh_env[shell_index]
737
738 # Unique dir for every test case and shell
739 tmp_base = os.path.normpath(opts.tmp_env) # no . or ..
740 case_tmp_dir = os.path.join(tmp_base, '%02d-%s' % (i, sh_label))
741
742 try:
743 os.makedirs(case_tmp_dir)
744 except OSError as e:
745 if e.errno != errno.EEXIST:
746 raise
747
748 # Some tests assume _tmp exists
749 # TODO: get rid of this in the common case, to save inodes! I guess have
750 # an opt-in setting per FILE, like make_underscore_tmp: true.
751 try:
752 os.mkdir(os.path.join(case_tmp_dir, '_tmp'))
753 except OSError as e:
754 if e.errno != errno.EEXIST:
755 raise
756
757 case_env['TMP'] = case_tmp_dir
758
759 if opts.pyann_out_dir:
760 case_env = dict(case_env)
761 case_env['PYANN_OUT'] = os.path.join(opts.pyann_out_dir,
762 '%d.json' % i)
763
764 try:
765 p = subprocess.Popen(argv,
766 env=case_env,
767 cwd=case_tmp_dir,
768 stdin=PIPE,
769 stdout=PIPE,
770 stderr=PIPE)
771 except OSError as e:
772 print('Error running %r: %s' % (sh_path, e), file=sys.stderr)
773 sys.exit(1)
774
775 p.stdin.write(code)
776
777 actual = {}
778 actual['stdout'], actual['stderr'] = p.communicate()
779
780 actual['status'] = p.wait()
781
782 if opts.timeout_bin and os.path.exists(timeout_file):
783 cell_result = Result.TIMEOUT
784 elif not opts.timeout_bin and actual['status'] == 124:
785 cell_result = Result.TIMEOUT
786 else:
787 messages = []
788 cell_result = Result.PASS
789
790 # TODO: Warn about no assertions? Well it will always test the error
791 # code.
792 assertions = CreateAssertions(case, sh_label)
793 for a in assertions:
794 result, msg = a.Check(sh_label, actual)
795 # The minimum one wins.
796 # If any failed, then the result is FAIL.
797 # If any are OK, but none are FAIL, the result is OK.
798 cell_result = min(cell_result, result)
799 if msg:
800 messages.append(msg)
801
802 if cell_result != Result.PASS or opts.details:
803 d = (i, sh_label, actual['stdout'], actual['stderr'],
804 messages)
805 out.AddDetails(d)
806
807 result_row.append(cell_result)
808
809 stats.ReportCell(i, cell_result, sh_label)
810
811 if sh_label in OTHER_OSH:
812 # This is only an error if we tried to run ANY OSH.
813 if osh_cpython_index == -1:
814 raise RuntimeError(
815 "Couldn't determine index of osh-cpython")
816
817 other_result = result_row[shell_index]
818 cpython_result = result_row[osh_cpython_index]
819 if other_result != cpython_result:
820 stats.Inc('oils_ALT_delta')
821
822 out.WriteRow(i, line_num, result_row, desc)
823
824 return stats
825
826
827# ANSI color constants
828_RESET = '\033[0;0m'
829_BOLD = '\033[1m'
830
831_RED = '\033[31m'
832_GREEN = '\033[32m'
833_YELLOW = '\033[33m'
834_PURPLE = '\033[35m'
835
836TEXT_CELLS = {
837 Result.TIMEOUT: 'TIME',
838 Result.FAIL: 'FAIL',
839 Result.BUG: 'BUG',
840 Result.NI: 'N-I',
841 Result.OK: 'ok',
842 Result.PASS: 'pass',
843}
844
845ANSI_COLORS = {
846 Result.TIMEOUT: _PURPLE,
847 Result.FAIL: _RED,
848 Result.BUG: _YELLOW,
849 Result.NI: _YELLOW,
850 Result.OK: _YELLOW,
851 Result.PASS: _GREEN,
852}
853
854
855def _AnsiCells():
856 lookup = {}
857 for i in xrange(Result.length):
858 lookup[i] = ''.join([ANSI_COLORS[i], _BOLD, TEXT_CELLS[i], _RESET])
859 return lookup
860
861
862ANSI_CELLS = _AnsiCells()
863
864HTML_CELLS = {
865 Result.TIMEOUT: '<td class="timeout">TIME',
866 Result.FAIL: '<td class="fail">FAIL',
867 Result.BUG: '<td class="bug">BUG',
868 Result.NI: '<td class="n-i">N-I',
869 Result.OK: '<td class="ok">ok',
870 Result.PASS: '<td class="pass">pass',
871}
872
873
874def _ValidUtf8String(s):
875 """Return an arbitrary string as a readable utf-8 string.
876
877 We output utf-8 to either HTML or the console. If we get invalid
878 utf-8 as stdout/stderr (which is very possible), then show the ASCII
879 repr().
880 """
881 try:
882 s.decode('utf-8')
883 return s # it decoded OK
884 except UnicodeDecodeError:
885 return repr(s) # ASCII representation
886
887
888class Output(object):
889
890 def __init__(self, f, verbose):
891 self.f = f
892 self.verbose = verbose
893 self.details = []
894
895 def BeginCases(self, test_file):
896 pass
897
898 def WriteHeader(self, sh_labels):
899 pass
900
901 def WriteRow(self, i, line_num, row, desc):
902 pass
903
904 def EndCases(self, sh_labels, stats):
905 pass
906
907 def AddDetails(self, entry):
908 self.details.append(entry)
909
910 # Helper function
911 def _WriteDetailsAsText(self, details):
912 for case_index, shell, stdout, stderr, messages in details:
913 print('case: %d' % case_index, file=self.f)
914 for m in messages:
915 print(m, file=self.f)
916
917 # Assume the terminal can show utf-8, but we don't want random binary.
918 print('%s stdout:' % shell, file=self.f)
919 print(_ValidUtf8String(stdout), file=self.f)
920
921 print('%s stderr:' % shell, file=self.f)
922 print(_ValidUtf8String(stderr), file=self.f)
923
924 print('', file=self.f)
925
926
927class TeeOutput(object):
928 """For multiple outputs in one run, e.g. HTML and TSV.
929
930 UNUSED
931 """
932
933 def __init__(self, outs):
934 self.outs = outs
935
936 def BeginCases(self, test_file):
937 for out in self.outs:
938 out.BeginCases(test_file)
939
940 def WriteHeader(self, sh_labels):
941 for out in self.outs:
942 out.WriteHeader(sh_labels)
943
944 def WriteRow(self, i, line_num, row, desc):
945 for out in self.outs:
946 out.WriteRow(i, line_num, row, desc)
947
948 def EndCases(self, sh_labels, stats):
949 for out in self.outs:
950 out.EndCases(sh_labels, stats)
951
952 def AddDetails(self, entry):
953 for out in self.outs:
954 out.AddDetails(entry)
955
956
957class TsvOutput(Output):
958 """Write a plain-text TSV file.
959
960 UNUSED since we are outputting LONG format with --tsv-output.
961 """
962
963 def WriteHeader(self, sh_labels):
964 self.f.write('case\tline\t') # case number and line number
965 for sh_label in sh_labels:
966 self.f.write(sh_label)
967 self.f.write('\t')
968 self.f.write('\n')
969
970 def WriteRow(self, i, line_num, row, desc):
971 self.f.write('%3d\t%3d\t' % (i, line_num))
972
973 for result in row:
974 c = TEXT_CELLS[result]
975 self.f.write(c)
976 self.f.write('\t')
977
978 # note: 'desc' could use TSV8, but just ignore it for now
979 #self.f.write(desc)
980 self.f.write('\n')
981
982
983class AnsiOutput(Output):
984
985 def BeginCases(self, test_file):
986 self.f.write('%s\n' % test_file)
987
988 def WriteHeader(self, sh_labels):
989 self.f.write(_BOLD)
990 self.f.write('case\tline\t') # case number and line number
991 for sh_label in sh_labels:
992 self.f.write(sh_label)
993 self.f.write('\t')
994 self.f.write(_RESET)
995 self.f.write('\n')
996
997 def WriteRow(self, i, line_num, row, desc):
998 self.f.write('%3d\t%3d\t' % (i, line_num))
999
1000 for result in row:
1001 c = ANSI_CELLS[result]
1002 self.f.write(c)
1003 self.f.write('\t')
1004
1005 self.f.write(desc)
1006 self.f.write('\n')
1007
1008 if self.verbose:
1009 self._WriteDetailsAsText(self.details)
1010 self.details = []
1011
1012 def _WriteShellSummary(self, sh_labels, stats):
1013 if len(stats.nonzero_results) <= 1: # Skip trivial summaries
1014 return
1015
1016 # Reiterate header
1017 self.f.write(_BOLD)
1018 self.f.write('\t\t')
1019 for sh_label in sh_labels:
1020 self.f.write(sh_label)
1021 self.f.write('\t')
1022 self.f.write(_RESET)
1023 self.f.write('\n')
1024
1025 # Write totals by cell.
1026 for result in sorted(stats.nonzero_results, reverse=True):
1027 self.f.write('\t%s' % ANSI_CELLS[result])
1028 for sh_label in sh_labels:
1029 self.f.write('\t%d' % stats.by_shell[sh_label][result])
1030 self.f.write('\n')
1031
1032 # The bottom row is all the same, but it helps readability.
1033 self.f.write('\ttotal')
1034 for sh_label in sh_labels:
1035 self.f.write('\t%d' % stats.counters['num_cases_run'])
1036 self.f.write('\n')
1037 self.f.write('\n')
1038
1039 def EndCases(self, sh_labels, stats):
1040 print()
1041 self._WriteShellSummary(sh_labels, stats)
1042
1043
1044class HtmlOutput(Output):
1045
1046 def __init__(self, f, verbose, spec_name, sh_labels, cases):
1047 Output.__init__(self, f, verbose)
1048 self.spec_name = spec_name
1049 self.sh_labels = sh_labels # saved from header
1050 self.cases = cases # for linking to code
1051 self.row_html = [] # buffered
1052
1053 def _SourceLink(self, line_num, desc):
1054 return '<a href="%s.test.html#L%d">%s</a>' % (self.spec_name, line_num,
1055 cgi.escape(desc))
1056
1057 def BeginCases(self, test_file):
1058 css_urls = ['../../../web/base.css', '../../../web/spec-tests.css']
1059 title = '%s: spec test case results' % self.spec_name
1060 html_head.Write(self.f, title, css_urls=css_urls)
1061
1062 self.f.write('''\
1063 <body class="width60">
1064 <p id="home-link">
1065 <a href=".">spec test index</a>
1066 /
1067 <a href="/">oilshell.org</a>
1068 </p>
1069 <h1>Results for %s</h1>
1070 <table>
1071 ''' % test_file)
1072
1073 def _WriteShellSummary(self, sh_labels, stats):
1074 # NOTE: This table has multiple <thead>, which seems OK.
1075 self.f.write('''
1076<thead>
1077 <tr class="table-header">
1078 ''')
1079
1080 columns = ['status'] + sh_labels + ['']
1081 for c in columns:
1082 self.f.write('<td>%s</td>' % c)
1083
1084 self.f.write('''
1085 </tr>
1086</thead>
1087''')
1088
1089 # Write totals by cell.
1090 for result in sorted(stats.nonzero_results, reverse=True):
1091 self.f.write('<tr>')
1092
1093 self.f.write(HTML_CELLS[result])
1094 self.f.write('</td> ')
1095
1096 for sh_label in sh_labels:
1097 self.f.write('<td>%d</td>' % stats.by_shell[sh_label][result])
1098
1099 self.f.write('<td></td>')
1100 self.f.write('</tr>\n')
1101
1102 # The bottom row is all the same, but it helps readability.
1103 self.f.write('<tr>')
1104 self.f.write('<td>total</td>')
1105 for sh_label in sh_labels:
1106 self.f.write('<td>%d</td>' % stats.counters['num_cases_run'])
1107 self.f.write('<td></td>')
1108 self.f.write('</tr>\n')
1109
1110 # Blank row for space.
1111 self.f.write('<tr>')
1112 for i in xrange(len(sh_labels) + 2):
1113 self.f.write('<td style="height: 2em"></td>')
1114 self.f.write('</tr>\n')
1115
1116 def WriteHeader(self, sh_labels):
1117 f = cStringIO.StringIO()
1118
1119 f.write('''
1120<thead>
1121 <tr class="table-header">
1122 ''')
1123
1124 columns = ['case'] + sh_labels
1125 for c in columns:
1126 f.write('<td>%s</td>' % c)
1127 f.write('<td class="case-desc">description</td>')
1128
1129 f.write('''
1130 </tr>
1131</thead>
1132''')
1133
1134 self.row_html.append(f.getvalue())
1135
1136 def WriteRow(self, i, line_num, row, desc):
1137 f = cStringIO.StringIO()
1138 f.write('<tr>')
1139 f.write('<td>%3d</td>' % i)
1140
1141 show_details = False
1142
1143 for result in row:
1144 c = HTML_CELLS[result]
1145 if result not in (Result.PASS, Result.TIMEOUT): # nothing to show
1146 show_details = True
1147
1148 f.write(c)
1149 f.write('</td>')
1150 f.write('\t')
1151
1152 f.write('<td class="case-desc">')
1153 f.write(self._SourceLink(line_num, desc))
1154 f.write('</td>')
1155 f.write('</tr>\n')
1156
1157 # Show row with details link.
1158 if show_details:
1159 f.write('<tr>')
1160 f.write('<td class="details-row"></td>') # for the number
1161
1162 for col_index, result in enumerate(row):
1163 f.write('<td class="details-row">')
1164 if result != Result.PASS:
1165 sh_label = self.sh_labels[col_index]
1166 f.write('<a href="#details-%s-%s">details</a>' %
1167 (i, sh_label))
1168 f.write('</td>')
1169
1170 f.write('<td class="details-row"></td>') # for the description
1171 f.write('</tr>\n')
1172
1173 self.row_html.append(f.getvalue()) # buffer it
1174
1175 def _WriteStats(self, stats):
1176 self.f.write('%(num_passed)d passed, %(num_ok)d OK, '
1177 '%(num_ni)d not implemented, %(num_bug)d BUG, '
1178 '%(num_failed)d failed, %(num_timeout)d timeouts, '
1179 '%(num_skipped)d cases skipped\n' % stats.counters)
1180
1181 def EndCases(self, sh_labels, stats):
1182 self._WriteShellSummary(sh_labels, stats)
1183
1184 # Write all the buffered rows
1185 for h in self.row_html:
1186 self.f.write(h)
1187
1188 self.f.write('</table>\n')
1189 self.f.write('<pre>')
1190 self._WriteStats(stats)
1191 if stats.Get('oils_num_failed'):
1192 self.f.write('%(oils_num_failed)d failed under osh\n' %
1193 stats.counters)
1194 self.f.write('</pre>')
1195
1196 if self.details:
1197 self._WriteDetails()
1198
1199 self.f.write('</body></html>')
1200
1201 def _WriteDetails(self):
1202 self.f.write("<h2>Details on runs that didn't PASS</h2>")
1203 self.f.write('<table id="details">')
1204
1205 for case_index, sh_label, stdout, stderr, messages in self.details:
1206 self.f.write('<tr>')
1207 self.f.write('<td><a name="details-%s-%s"></a><b>%s</b></td>' %
1208 (case_index, sh_label, sh_label))
1209
1210 self.f.write('<td>')
1211
1212 # Write description and link to the code
1213 case = self.cases[case_index]
1214 line_num = case['line_num']
1215 desc = case['desc']
1216 self.f.write('%d ' % case_index)
1217 self.f.write(self._SourceLink(line_num, desc))
1218 self.f.write('<br/><br/>\n')
1219
1220 for m in messages:
1221 self.f.write('<span class="assertion">%s</span><br/>\n' %
1222 cgi.escape(m))
1223 if messages:
1224 self.f.write('<br/>\n')
1225
1226 def _WriteRaw(s):
1227 self.f.write('<pre>')
1228
1229 # stdout might contain invalid utf-8; make it valid;
1230 valid_utf8 = _ValidUtf8String(s)
1231
1232 self.f.write(cgi.escape(valid_utf8))
1233 self.f.write('</pre>')
1234
1235 self.f.write('<i>stdout:</i> <br/>\n')
1236 _WriteRaw(stdout)
1237
1238 self.f.write('<i>stderr:</i> <br/>\n')
1239 _WriteRaw(stderr)
1240
1241 self.f.write('</td>')
1242 self.f.write('</tr>')
1243
1244 self.f.write('</table>')
1245
1246
1247def MakeTestEnv(opts):
1248 if not opts.tmp_env:
1249 raise RuntimeError('--tmp-env required')
1250 if not opts.path_env:
1251 raise RuntimeError('--path-env required')
1252 env = {
1253 'PATH': opts.path_env,
1254 #'LANG': opts.lang_env,
1255 }
1256 for p in opts.env_pair:
1257 name, value = p.split('=', 1)
1258 env[name] = value
1259
1260 return env
1261
1262
1263def _DefaultSuite(spec_name):
1264 if spec_name.startswith('ysh-'):
1265 suite = 'ysh'
1266 elif spec_name.startswith('hay'): # hay.test.sh is ysh
1267 suite = 'ysh'
1268
1269 elif spec_name.startswith('tea-'):
1270 suite = 'tea'
1271 else:
1272 suite = 'osh'
1273
1274 return suite
1275
1276
1277def ParseTestList(test_files):
1278 for test_file in test_files:
1279 with open(test_file) as f:
1280 tokens = Tokenizer(f)
1281 try:
1282 file_metadata, cases = ParseTestFile(test_file, tokens)
1283 except RuntimeError as e:
1284 log('ERROR in %r', test_file)
1285 raise
1286 except ParseError as e:
1287 log('PARSE ERROR in %r', test_file)
1288 raise
1289
1290 tmp = os.path.basename(test_file)
1291 spec_name = tmp.split('.')[0] # foo.test.sh -> foo
1292
1293 suite = file_metadata.get('suite') or _DefaultSuite(spec_name)
1294
1295 tmp = file_metadata.get('tags')
1296 tags = tmp.split() if tmp else []
1297
1298 # Don't need compare_shells, etc. to decide what to run
1299
1300 row = {'spec_name': spec_name, 'suite': suite, 'tags': tags}
1301 #print(row)
1302 yield row
1303
1304
1305def main(argv):
1306 # First check if bash is polluting the environment. Tests rely on the
1307 # environment.
1308 v = os.getenv('RANDOM')
1309 if v is not None:
1310 raise AssertionError('got $RANDOM = %s' % v)
1311 v = os.getenv('PPID')
1312 if v is not None:
1313 raise AssertionError('got $PPID = %s' % v)
1314
1315 p = optparse.OptionParser('%s [options] TEST_FILE shell...' % sys.argv[0])
1316 spec_lib.DefineCommon(p)
1317 spec_lib.DefineShSpec(p)
1318 opts, argv = p.parse_args(argv)
1319
1320 # --print-tagged to figure out what to run
1321 if opts.print_tagged:
1322 to_find = opts.print_tagged
1323 for row in ParseTestList(argv[1:]):
1324 if to_find in row['tags']:
1325 print(row['spec_name'])
1326 return 0
1327
1328 # --print-table to figure out what to run
1329 if opts.print_table:
1330 for row in ParseTestList(argv[1:]):
1331 print('%(suite)s\t%(spec_name)s' % row)
1332 #print(row)
1333 return 0
1334
1335 #
1336 # Now deal with a single file
1337 #
1338
1339 try:
1340 test_file = argv[1]
1341 except IndexError:
1342 p.print_usage()
1343 return 1
1344
1345 with open(test_file) as f:
1346 tokens = Tokenizer(f)
1347 file_metadata, cases = ParseTestFile(test_file, tokens)
1348
1349 # List test cases and return
1350 if opts.do_list:
1351 for i, case in enumerate(cases):
1352 if opts.verbose: # print the raw dictionary for debugging
1353 print(pprint.pformat(case))
1354 else:
1355 print('%d\t%s' % (i, case['desc']))
1356 return 0
1357
1358 # for test/spec-cpp.sh
1359 if opts.print_spec_suite:
1360 tmp = os.path.basename(test_file)
1361 spec_name = tmp.split('.')[0] # foo.test.sh -> foo
1362
1363 suite = file_metadata.get('suite') or _DefaultSuite(spec_name)
1364 print(suite)
1365 return 0
1366
1367 if opts.verbose:
1368 for k, v in file_metadata.items():
1369 print('\t%-20s: %s' % (k, v), file=sys.stderr)
1370 print('', file=sys.stderr)
1371
1372 if opts.oils_bin_dir:
1373
1374 shells = []
1375
1376 if opts.compare_shells:
1377 comp = file_metadata.get('compare_shells')
1378 # Compare 'compare_shells' and Python
1379 shells.extend(comp.split() if comp else [])
1380
1381 # Always run with the Python version
1382 our_shell = file_metadata.get('our_shell', 'osh') # default is OSH
1383 if our_shell != '-':
1384 shells.append(os.path.join(opts.oils_bin_dir, our_shell))
1385
1386 # Legacy OVM/CPython build
1387 if opts.ovm_bin_dir:
1388 shells.append(os.path.join(opts.ovm_bin_dir, our_shell))
1389
1390 # New C++ build
1391 if opts.oils_cpp_bin_dir:
1392 shells.append(os.path.join(opts.oils_cpp_bin_dir, our_shell))
1393
1394 # Overwrite it when --oils-bin-dir is set
1395 # It's no longer a flag
1396 opts.oils_failures_allowed = \
1397 int(file_metadata.get('oils_failures_allowed', 0))
1398
1399 else:
1400 # TODO: remove this mode?
1401 shells = argv[2:]
1402
1403 shell_pairs = spec_lib.MakeShellPairs(shells)
1404
1405 if opts.range:
1406 begin, end = spec_lib.ParseRange(opts.range)
1407 case_predicate = spec_lib.RangePredicate(begin, end)
1408 elif opts.regex:
1409 desc_re = re.compile(opts.regex, re.IGNORECASE)
1410 case_predicate = spec_lib.RegexPredicate(desc_re)
1411 else:
1412 case_predicate = lambda i, case: True
1413
1414 out_f = sys.stderr if opts.do_print else sys.stdout
1415
1416 # Set up output style. Also see asdl/format.py
1417 if opts.format == 'ansi':
1418 out = AnsiOutput(out_f, opts.verbose)
1419
1420 elif opts.format == 'html':
1421 spec_name = os.path.basename(test_file)
1422 spec_name = spec_name.split('.')[0]
1423
1424 sh_labels = [label for label, _ in shell_pairs]
1425
1426 out = HtmlOutput(out_f, opts.verbose, spec_name, sh_labels, cases)
1427
1428 else:
1429 raise AssertionError()
1430
1431 out.BeginCases(os.path.basename(test_file))
1432
1433 env = MakeTestEnv(opts)
1434 stats = RunCases(cases, case_predicate, shell_pairs, env, out, opts)
1435
1436 out.EndCases([sh_label for sh_label, _ in shell_pairs], stats)
1437
1438 if opts.tsv_output:
1439 with open(opts.tsv_output, 'w') as f:
1440 stats.WriteTsv(f)
1441
1442 # TODO: Could --stats-{file,template} be a separate awk step on .tsv files?
1443 stats.Set('oils_failures_allowed', opts.oils_failures_allowed)
1444
1445 # If it's not set separately for C++, we default to the allowed number
1446 # above
1447 x = int(
1448 file_metadata.get('oils_cpp_failures_allowed',
1449 opts.oils_failures_allowed))
1450 stats.Set('oils_cpp_failures_allowed', x)
1451
1452 if opts.stats_file:
1453 with open(opts.stats_file, 'w') as f:
1454 f.write(opts.stats_template % stats.counters)
1455 f.write('\n') # bash 'read' requires a newline
1456
1457 # spec/smoke.test.sh -> smoke
1458 test_name = os.path.basename(test_file).split('.')[0]
1459
1460 return _SuccessOrFailure(test_name, stats)
1461
1462
1463def _SuccessOrFailure(test_name, stats):
1464 allowed = stats.Get('oils_failures_allowed')
1465 allowed_cpp = stats.Get('oils_cpp_failures_allowed')
1466
1467 all_count = stats.Get('num_failed')
1468 oils_count = stats.Get('oils_num_failed')
1469 oils_cpp_count = stats.Get('oils_cpp_num_failed')
1470
1471 errors = []
1472 if oils_count != allowed:
1473 errors.append('Got %d Oils failures, but %d are allowed' %
1474 (oils_count, allowed))
1475 else:
1476 if allowed != 0:
1477 log('%s: note: Got %d allowed Oils failures', test_name, allowed)
1478
1479 # TODO: remove special case for 0
1480 if oils_cpp_count != 0:
1481 if oils_cpp_count != allowed_cpp:
1482 errors.append('Got %d Oils C++ failures, but %d are allowed' %
1483 (oils_cpp_count, allowed_cpp))
1484 else:
1485 if allowed_cpp != 0:
1486 log('%s: note: Got %d allowed Oils C++ failures', test_name,
1487 allowed_cpp)
1488
1489 if all_count != allowed:
1490 errors.append('Got %d total failures, but %d are allowed' %
1491 (all_count, allowed))
1492
1493 if errors:
1494 for msg in errors:
1495 log('%s: FATAL: %s', test_name, msg)
1496 return 1
1497
1498 return 0
1499
1500
1501if __name__ == '__main__':
1502 try:
1503 sys.exit(main(sys.argv))
1504 except KeyboardInterrupt as e:
1505 print('%s: interrupted with Ctrl-C' % sys.argv[0], file=sys.stderr)
1506 sys.exit(1)
1507 except RuntimeError as e:
1508 print('FATAL: %s' % e, file=sys.stderr)
1509 sys.exit(1)