OILS / test / spec-runner.sh View on Github | oils.pub

415 lines, 150 significant
1#!/usr/bin/env bash
2#
3# Run tests against multiple shells with the sh_spec framework.
4#
5# Usage:
6# test/spec-runner.sh <function name>
7
8set -o nounset
9set -o pipefail
10set -o errexit
11shopt -s strict:all 2>/dev/null || true # dogfood for OSH
12
13REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
14
15source build/dev-shell.sh
16source test/common.sh
17source test/spec-common.sh
18source test/tsv-lib.sh # $TAB
19
20NUM_SPEC_TASKS=${NUM_SPEC_TASKS:-400}
21
22# Option to use our xargs implementation.
23#xargs() {
24# echo "Using ~/git/oilshell/xargs.py/xargs.py"
25# ~/git/oilshell/xargs.py/xargs.py "$@"
26#}
27
28#
29# Test Runner
30#
31
32write-suite-manifests() {
33 #test/sh_spec.py --print-table spec/*.test.sh
34 { test/sh_spec.py --print-table spec/*.test.sh | while read suite name; do
35 case $suite in
36 osh) echo $name >& $osh ;;
37 ysh) echo $name >& $ysh ;;
38 disabled) ;; # ignore
39 *) die "Invalid suite $suite" ;;
40 esac
41 done
42 } {osh}>_tmp/spec/SUITE-osh.txt \
43 {ysh}>_tmp/spec/SUITE-ysh.txt \
44 {needs_terminal}>_tmp/spec/SUITE-needs-terminal.txt
45
46 # These are kind of pseudo-suites, not the main 3
47 test/sh_spec.py --print-tagged interactive \
48 spec/*.test.sh > _tmp/spec/SUITE-interactive.txt
49
50 test/sh_spec.py --print-tagged dev-minimal \
51 spec/*.test.sh > _tmp/spec/SUITE-osh-minimal.txt
52}
53
54_print-task-file() {
55 cat <<'EOF'
56#!/usr/bin/env bash
57#
58# This file is GENERATED -- DO NOT EDIT.
59#
60# Update it with:
61# test/spec-runner.sh gen-task-file
62#
63# Usage:
64# test/spec.sh <function name>
65
66: ${LIB_OSH=stdlib/osh}
67source $LIB_OSH/bash-strict.sh
68source $LIB_OSH/task-five.sh
69
70source build/dev-shell.sh
71EOF
72
73 while read spec_name; do
74 echo "
75$spec_name() {
76 test/spec-py.sh run-file $spec_name \"\$@\"
77}"
78 done
79
80 echo
81 echo 'task-five "$@"'
82}
83
84gen-task-file() {
85 test/sh_spec.py --print-table spec/*.test.sh | while read suite name; do
86 echo $name
87 done | _print-task-file > test/spec.sh
88}
89
90diff-manifest() {
91 ### temporary test
92
93 write-suite-manifests
94 #return
95
96 # crazy sorting, affects glob
97 # doesn't work
98 #LANG=C
99 #LC_COLLATE=C
100 #LC_ALL=C
101 #export LANG LC_COLLATE LC_ALL
102
103 for suite in osh ysh interactive osh-minimal; do
104 echo
105 echo [$suite]
106 echo
107
108 diff -u -r <(sort spec2/SUITE-$suite.txt) <(sort _tmp/spec/SUITE-$suite.txt) #|| true
109 done
110}
111
112dispatch-one() {
113 # Determines what binaries to compare against: compare-py | compare-cpp | release-alpine
114 local compare_mode=${1:-compare-py}
115 # Which subdir of _tmp/spec: osh-py ysh-py osh-cpp ysh-cpp smoosh
116 local spec_subdir=${2:-osh-py}
117 local spec_name=$3
118 shift 3 # rest are more flags
119
120 log "__ $spec_name"
121
122 local -a prefix
123 case $compare_mode in
124
125 #compare-py) prefix=(test/spec.sh) ;;
126 compare-py) prefix=(test/spec-py.sh run-file) ;;
127
128 compare-cpp) prefix=(test/spec-cpp.sh run-file) ;;
129
130 # For interactive comparison
131 osh-only) prefix=(test/spec-util.sh run-file-with-osh) ;;
132 bash-only) prefix=(test/spec-util.sh run-file-with-bash) ;;
133
134 release-alpine) prefix=(test/spec-alpine.sh run-file) ;;
135
136 *) die "Invalid compare mode $compare_mode" ;;
137 esac
138
139 local base_dir=_tmp/spec/$spec_subdir
140
141 # TODO: Could --stats-{file,template} be a separate awk step on .tsv files?
142 run-task-with-status \
143 $base_dir/${spec_name}.task.txt \
144 "${prefix[@]}" $spec_name \
145 --format html \
146 --stats-file $base_dir/${spec_name}.stats.txt \
147 --stats-template \
148 '%(num_cases)d %(oils_num_passed)d %(oils_num_failed)d %(oils_failures_allowed)d %(oils_ALT_delta)d' \
149 "$@" \
150 > $base_dir/${spec_name}.html
151}
152
153
154_html-summary() {
155 ### Print an HTML summary to stdout and return whether all tests succeeded
156
157 local sh_label=$1 # osh or ysh
158 local base_dir=$2 # e.g. _tmp/spec/ysh-cpp
159 local totals=$3 # path to print HTML to
160 local manifest=$4
161
162 html-head --title "Spec Test Summary" \
163 ../../../web/base.css ../../../web/spec-tests.css
164
165 cat <<EOF
166 <body class="width50">
167
168<p id="home-link">
169 <!-- The release index is two dirs up -->
170 <a href="../..">Up</a> |
171 <a href="/">oils.pub</a>
172</p>
173
174<h1>Spec Test Results Summary</h1>
175
176<table>
177 <thead>
178 <tr>
179 <td>name</td>
180 <td># cases</td> <td>$sh_label # passed</td> <td>$sh_label # failed</td>
181 <td>$sh_label failures allowed</td>
182 <td>$sh_label ALT delta</td>
183 <td>Elapsed Seconds</td>
184 </tr>
185 </thead>
186 <!-- TOTALS -->
187EOF
188
189 # Awk notes:
190 # - "getline" is kind of like bash "read", but it doesn't allow you do
191 # specify variable names. You have to destructure it yourself.
192 # - Lack of string interpolation is very annoying
193
194 head -n $NUM_SPEC_TASKS $manifest | sort | awk -v totals=$totals -v base_dir=$base_dir '
195 # Awk problem: getline errors are ignored by default!
196 function error(path) {
197 print "Error reading line from file: " path > "/dev/stderr"
198 exit(1)
199 }
200
201 {
202 spec_name = $0
203
204 # Read from the task files
205 path = ( base_dir "/" spec_name ".task.txt" )
206 n = getline < path
207 if (n != 1) {
208 error(path)
209 }
210 status = $1
211 wall_secs = $2
212
213 path = ( base_dir "/" spec_name ".stats.txt" )
214 n = getline < path
215 if (n != 1) {
216 error(path)
217 }
218 num_cases = $1
219 oils_num_passed = $2
220 oils_num_failed = $3
221 oils_failures_allowed = $4
222 oils_ALT_delta = $5
223
224 sum_status += status
225 sum_wall_secs += wall_secs
226 sum_num_cases += num_cases
227 sum_oils_num_passed += oils_num_passed
228 sum_oils_num_failed += oils_num_failed
229 sum_oils_failures_allowed += oils_failures_allowed
230 sum_oils_ALT_delta += oils_ALT_delta
231 num_rows += 1
232
233 # For the console
234 if (status == 0) {
235 num_passed += 1
236 } else {
237 num_failed += 1
238 print spec_name " failed with status " status > "/dev/stderr"
239 }
240
241 if (status != 0) {
242 css_class = "failed"
243 } else if (oils_num_failed != 0) {
244 css_class = "osh-allow-fail"
245 } else if (oils_num_passed != 0) {
246 css_class = "osh-pass"
247 } else {
248 css_class = ""
249 }
250 print "<tr class=" css_class ">"
251 print "<td><a href=" spec_name ".html>" spec_name "</a></td>"
252 print "<td>" num_cases "</td>"
253 print "<td>" oils_num_passed "</td>"
254 print "<td>" oils_num_failed "</td>"
255 print "<td>" oils_failures_allowed "</td>"
256 print "<td>" oils_ALT_delta "</td>"
257 printf("<td>%.2f</td>\n", wall_secs);
258 print "</tr>"
259 }
260
261 END {
262 print "<tr class=totals>" >totals
263 print "<td>TOTAL (" num_rows " rows) </td>" >totals
264 print "<td>" sum_num_cases "</td>" >totals
265 print "<td>" sum_oils_num_passed "</td>" >totals
266 print "<td>" sum_oils_num_failed "</td>" >totals
267 print "<td>" sum_oils_failures_allowed "</td>" >totals
268 print "<td>" sum_oils_ALT_delta "</td>" >totals
269 printf("<td>%.2f</td>\n", sum_wall_secs) > totals
270 print "</tr>" >totals
271
272 print "<tfoot>"
273 print "<!-- TOTALS -->"
274 print "</tfoot>"
275
276 # For the console
277 print "" > "/dev/stderr"
278 if (num_failed == 0) {
279 print "*** All " num_passed " tests PASSED" > "/dev/stderr"
280 } else {
281 print "*** " num_failed " tests FAILED" > "/dev/stderr"
282 exit(1) # failure
283 }
284 }
285 '
286 all_passed=$?
287
288 cat <<EOF
289 </table>
290
291 <h3>Version Information</h3>
292 <pre>
293EOF
294
295 # TODO: can pass shells here, e.g. for test/spec-cpp.sh
296 test/spec-version.sh ${suite}-version-text
297
298 cat <<EOF
299 </pre>
300 </body>
301</html>
302EOF
303
304 return $all_passed
305}
306
307html-summary() {
308 local suite=$1
309 local base_dir=$2
310
311 local manifest="_tmp/spec/SUITE-$suite.txt"
312
313 local totals=$base_dir/totals-$suite.html
314 local tmp=$base_dir/tmp-$suite.html
315
316 local out=$base_dir/index.html
317
318 # TODO: Do we also need $base_dir/{osh,oil}-details-for-toil.json
319 # osh failures, and all failures
320 # When deploying, if they exist, them copy them outside?
321 # I guess toil_web.py can use the zipfile module?
322 # To get _tmp/spec/...
323 # it can read JSON like:
324 # { "task_tsv": "_tmp/toil/INDEX.tsv",
325 # "details_json": [ ... ],
326 # }
327
328 set +o errexit
329 _html-summary $suite $base_dir $totals $manifest > $tmp
330 all_passed=$?
331 set -o errexit
332
333 # Total rows are displayed at both the top and bottom.
334 awk -v totals="$(cat $totals)" '
335 /<!-- TOTALS -->/ {
336 print totals
337 next
338 }
339 { print }
340 ' < $tmp > $out
341
342 echo
343 echo "Results: file://$PWD/$out"
344
345 return $all_passed
346}
347
348_all-parallel() {
349 local suite=${1:-osh}
350 local compare_mode=${2:-compare-py}
351 local spec_subdir=${3:-survey}
352
353 # The rest are more flags
354 shift 3
355
356 local manifest="_tmp/spec/SUITE-$suite.txt"
357 local output_base_dir="_tmp/spec/$spec_subdir"
358 mkdir -p $output_base_dir
359
360 write-suite-manifests
361
362 # The exit codes are recorded in files for html-summary to aggregate.
363 set +o errexit
364 head -n $NUM_SPEC_TASKS $manifest \
365 | xargs -I {} -P $MAX_PROCS -- \
366 $0 dispatch-one $compare_mode $spec_subdir {} "$@"
367 set -o errexit
368
369 all-tests-to-html $manifest $output_base_dir
370
371 # note: the HTML links to ../../web/, which is in the repo.
372 html-summary $suite $output_base_dir # returns whether all passed
373}
374
375all-parallel() {
376 ### Run spec tests in parallel.
377
378 # Note that this function doesn't fail because 'run-file' saves the status
379 # to a file.
380
381 time $0 _all-parallel "$@"
382}
383
384src-tree-py() {
385 PYTHONPATH='.:vendor/' doctools/src_tree.py "$@"
386}
387
388all-tests-to-html() {
389 local manifest=$1
390 local output_base_dir=$2
391 # ignore attrs output
392 head -n $NUM_SPEC_TASKS $manifest \
393 | xargs --verbose -- $0 src-tree-py spec-files $output_base_dir >/dev/null
394
395 #| xargs -n 1 -P $MAX_PROCS -- $0 test-to-html $output_base_dir
396 log "done: all-tests-to-html"
397}
398
399shell-sanity-check() {
400 echo "PWD = $PWD"
401 echo "PATH = $PATH"
402
403 for sh in "$@"; do
404 # note: shells are in $PATH, but not $OSH_LIST
405 if ! $sh -c 'echo -n "hello from $0: "; command -v $0 || true'; then
406 echo "ERROR: $sh failed sanity check"
407 return 1
408 fi
409 done
410}
411
412filename=$(basename $0)
413if test "$filename" = 'spec-runner.sh'; then
414 "$@"
415fi