OILS / test / stateful.sh View on Github | oilshell.org

239 lines, 104 significant
1#!/usr/bin/env bash
2#
3# Wrapper for test cases in spec/stateful
4#
5# Usage:
6# test/stateful.sh <function name>
7#
8# Examples:
9# test/stateful.sh signals -r 0-1 # run a range of tests
10# test/stateful.sh signals --list # list tests
11# test/stateful.sh job-control --num-retries 0
12#
13# test/stateful.sh signals-quick # not all shells
14#
15# test/stateful.sh soil-run
16#
17# TODO: Should have QUICKLY=1 variants
18
19set -o nounset
20set -o pipefail
21set -o errexit
22
23REPO_ROOT=$(cd $(dirname $0)/.. && pwd) # tsv-lib.sh uses this
24readonly REPO_ROOT
25
26source test/common.sh # log, $OSH
27source test/tsv-lib.sh
28
29source build/dev-shell.sh
30
31readonly BASE_DIR=_tmp/spec/stateful
32
33# Hack for testing the harness
34#readonly FIRST='-r 0'
35readonly FIRST=''
36readonly OSH_CPP=_bin/cxx-asan/osh
37
38readonly -a QUICK_SHELLS=( $OSH bash )
39# readonly -a QUICK_SHELLS=( $OSH $OSH_CPP bash )
40
41#
42# Suites in spec/stateful
43#
44
45signals() {
46 spec/stateful/signals.py $FIRST "$@"
47}
48
49interactive() {
50 spec/stateful/interactive.py $FIRST "$@"
51}
52
53job-control() {
54 spec/stateful/job_control.py $FIRST --oils-failures-allowed 0 "$@"
55}
56
57bind() {
58 spec/stateful/bind.py $FIRST --oils-failures-allowed 2 "$@"
59}
60
61# Run on just 2 shells
62
63signals-quick() { signals "${QUICK_SHELLS[@]}" "$@"; }
64interactive-quick() { interactive "${QUICK_SHELLS[@]}" "$@"; }
65job-control-quick() { job-control "${QUICK_SHELLS[@]}" "$@"; }
66bind-quick() { bind "${QUICK_SHELLS[@]}" "$@"; }
67
68# Run on all shells we can
69
70# They now pass for dash and mksh, with wait -n and PIPESTATUS skipped. zsh
71# doesn't work now, but could if the prompt was changed to $ ?
72signals-all() { signals "${QUICK_SHELLS[@]}" dash mksh "$@"; }
73
74interactive-all() { interactive "${QUICK_SHELLS[@]}" dash mksh "$@"; }
75
76job-control-all() { job-control "${QUICK_SHELLS[@]}" dash "$@"; }
77
78# On non-bash shells, bind is either unsupported or the syntax is too different
79bind-all() { bind "${QUICK_SHELLS[@]}" "$@"; }
80
81
82#
83# More automation
84#
85
86print-tasks() {
87 ### List all tests
88
89 # TODO:
90 # - Print a table with --osh-allowed-failures and shells. It can be filtered
91
92 if test -n "${QUICKLY:-}"; then
93 echo 'interactive'
94 else
95 echo 'bind'
96 echo 'interactive'
97 echo 'job-control'
98 echo 'signals'
99 fi
100}
101
102run-file() {
103 ### Run a spec/stateful file, logging output
104
105 local spec_name=$1
106
107 log "__ $spec_name"
108
109 local base_dir=$BASE_DIR
110
111 local log_filename=$spec_name.log.txt
112 local results_filename=$spec_name.results.txt
113
114 time-tsv -o $base_dir/${spec_name}.task.txt \
115 --field $spec_name --field $log_filename --field $results_filename -- \
116 $0 "$spec_name-all" --results-file $base_dir/$results_filename \
117 >$base_dir/$log_filename 2>&1 || true
118}
119
120html-summary() {
121 ### Summarize all files
122
123 # Note: In retrospect, it would be better if every process writes a "long"
124 # TSV file of results.
125 # And then we concatenate them and write the "wide" summary here.
126
127 html-head --title 'Stateful Tests' \
128 ../../../web/base.css ../../../web/spec-tests.css
129
130 # Similar to test/spec-runner.sh and soil format-wwz-index
131
132 cat <<EOF
133 <body class="width50">
134
135<p id="home-link">
136 <!-- up to .wwz index -->
137 <a href="../..">Up</a> |
138 <a href="/">Home</a>
139</p>
140
141 <h1>Stateful Tests with <a href="//www.oilshell.org/cross-ref.html#pexpect">pexpect</a> </h1>
142
143 <table>
144 <thead>
145 <tr>
146 <td>Test File</td>
147 <td>Elapsed seconds</td>
148 <td>Status</td>
149 </tr>
150 </thead>
151EOF
152
153 local all_passed=0
154
155 shopt -s lastpipe # to mutate all_passed in while
156
157 local results_tmp=$BASE_DIR/results.html
158 echo '' > $results_tmp # Accumulate more here
159
160 print-tasks | while read spec_name; do
161
162 # Note: in test/spec-runner.sh, an awk script creates this table. It reads
163 # *.task.txt and *.stats.txt. I could add --stats-file to harness.py
164 # with pass/fail stats
165 read status elapsed _ log_filename results_filename < $BASE_DIR/${spec_name}.task.txt
166
167 echo '<tr>'
168 echo "<td> <a href="$log_filename">$spec_name</a> </td>"
169
170 printf -v elapsed_str '%.1f' $elapsed
171 echo "<td>$elapsed_str</td>"
172
173 case $status in
174 (0) # exit code 0 is success
175 echo " <td>$status</td>"
176 ;;
177 (*) # everything else is a failure
178 # Add extra text to make red stand out.
179 echo " <td class=\"fail\">status: $status</td>"
180
181 # Mark failure
182 all_passed=1
183 ;;
184 esac
185 echo '</tr>'
186
187 # Append to temp file
188 {
189 echo "<h2>$spec_name</h2>"
190 echo '<pre>'
191 escape-html $BASE_DIR/$results_filename
192 echo '</pre>'
193 } >> $results_tmp
194
195 done
196 echo '</table>'
197
198 cat $results_tmp
199
200 cat <<EOF
201 </table>
202 </body>
203</html>
204EOF
205
206 log "all_passed = $all_passed"
207
208 return $all_passed
209}
210
211soil-run() {
212 ninja $OSH_CPP
213
214 mkdir -p $BASE_DIR
215
216 print-tasks | xargs -n 1 -- $0 run-file
217
218 # Returns whether all passed
219 html-summary > $BASE_DIR/index.html
220}
221
222#
223# Debugging
224#
225
226test-stop() {
227 python3 spec/stateful/harness.py test-stop demo/cpython/fork_signal_state.py
228}
229
230strace-py-fork() {
231 rm -f -v _tmp/py-fork.*
232 strace -ff -o _tmp/py-fork demo/cpython/fork_signal_state.py
233 ls -l _tmp/py-fork.*
234
235 # I see rt_sigaction(SIGSTP, ...) which is good
236 # so yeah this seems perfectly fine -- why is it ignoring SIGTSTP? :-(
237}
238
239"$@"