OILS / test / stateful.sh View on Github | oils.pub

247 lines, 111 significant
1#!/usr/bin/env bash
2#
3# Wrapper for test cases in spec/stateful
4#
5# Usage:
6# test/stateful.sh <function name>
7#
8# Examples:
9# test/stateful.sh signals -r 0-1 # run a range of tests
10# test/stateful.sh signals --list # list tests
11# test/stateful.sh job-control --num-retries 0
12#
13# test/stateful.sh signals-quick # not all shells
14#
15# test/stateful.sh soil-run
16#
17# TODO: Should have QUICKLY=1 variants
18
19set -o nounset
20set -o pipefail
21set -o errexit
22
23REPO_ROOT=$(cd $(dirname $0)/.. && pwd) # tsv-lib.sh uses this
24readonly REPO_ROOT
25
26source test/common.sh # log, $OSH
27source test/tsv-lib.sh
28
29source build/dev-shell.sh
30
31readonly BASE_DIR=_tmp/spec/stateful
32
33# Hack for testing the harness
34#readonly FIRST='-r 0'
35readonly FIRST=''
36readonly OSH_CPP=_bin/cxx-asan/osh
37
38readonly -a QUICK_SHELLS=( $OSH bash )
39readonly -a ALL_SHELLS=( $OSH $OSH_CPP bash )
40
41#
42# Suites in spec/stateful
43#
44
45signals() {
46 spec/stateful/signals.py $FIRST "$@"
47}
48
49interactive() {
50 spec/stateful/interactive.py $FIRST "$@"
51}
52
53job-control() {
54 spec/stateful/job_control.py $FIRST --oils-failures-allowed 0 "$@"
55}
56
57bind() {
58 spec/stateful/bind.py $FIRST --oils-failures-allowed 0 --num-lines 24 --num-columns 80 "$@"
59}
60
61history-expand() {
62 spec/stateful/history_expand.py $FIRST --oils-failures-allowed 1 "$@"
63}
64
65
66# Run on just 2 shells
67
68signals-quick() { signals "${QUICK_SHELLS[@]}" "$@"; }
69interactive-quick() { interactive "${QUICK_SHELLS[@]}" "$@"; }
70job-control-quick() { job-control "${QUICK_SHELLS[@]}" "$@"; }
71bind-quick() { bind "${QUICK_SHELLS[@]}" "$@"; }
72history-expand-quick() { history-expand "${QUICK_SHELLS[@]}" --num-retries 1 "$@"; }
73
74# Run on all shells we can
75
76# They now pass for dash and mksh, with wait -n and PIPESTATUS skipped. zsh
77# doesn't work now, but could if the prompt was changed to $ ?
78signals-all() { signals "${ALL_SHELLS[@]}" dash mksh "$@"; }
79
80interactive-all() { interactive "${ALL_SHELLS[@]}" dash mksh "$@"; }
81
82job-control-all() { job-control "${ALL_SHELLS[@]}" dash "$@"; }
83
84# On non-bash shells, bind is either unsupported or the syntax is too different
85bind-all() { bind "${ALL_SHELLS[@]}" "$@"; }
86
87history-expand-all() { history-expand "${ALL_SHELLS[@]}" "$@"; }
88
89
90#
91# More automation
92#
93
94print-tasks() {
95 ### List all tests
96
97 # TODO:
98 # - Print a table with --osh-allowed-failures and shells. It can be filtered
99
100 if test -n "${QUICKLY:-}"; then
101 echo 'interactive'
102 else
103 echo 'bind'
104 echo 'interactive'
105 echo 'job-control'
106 echo 'signals'
107 echo 'history-expand'
108 fi
109}
110
111run-file() {
112 ### Run a spec/stateful file via the "FOO-all" function logging output
113
114 local spec_name=$1
115
116 log "__ $spec_name"
117
118 local base_dir=$BASE_DIR
119
120 local log_filename=$spec_name.log.txt
121 local results_filename=$spec_name.results.txt
122
123 time-tsv -o $base_dir/${spec_name}.task.txt \
124 --field $spec_name --field $log_filename --field $results_filename -- \
125 $0 "$spec_name-all" --results-file $base_dir/$results_filename \
126 >$base_dir/$log_filename 2>&1 || true
127}
128
129html-summary() {
130 ### Summarize all files
131
132 # Note: In retrospect, it would be better if every process writes a "long"
133 # TSV file of results.
134 # And then we concatenate them and write the "wide" summary here.
135
136 html-head --title 'Stateful Tests' \
137 ../../../web/base.css ../../../web/spec-tests.css
138
139 # Similar to test/spec-runner.sh and soil format-wwz-index
140
141 cat <<EOF
142 <body class="width50">
143
144<p id="home-link">
145 <!-- up to .wwz index -->
146 <a href="../..">Up</a> |
147 <a href="/">Home</a>
148</p>
149
150 <h1>Stateful Tests with <a href="//www.oilshell.org/cross-ref.html#pexpect">pexpect</a> </h1>
151
152 <table>
153 <thead>
154 <tr>
155 <td>Test File</td>
156 <td>Elapsed seconds</td>
157 <td>Status</td>
158 </tr>
159 </thead>
160EOF
161
162 local all_passed=0
163
164 shopt -s lastpipe # to mutate all_passed in while
165
166 local results_tmp=$BASE_DIR/results.html
167 echo '' > $results_tmp # Accumulate more here
168
169 print-tasks | while read spec_name; do
170
171 # Note: in test/spec-runner.sh, an awk script creates this table. It reads
172 # *.task.txt and *.stats.txt. I could add --stats-file to harness.py
173 # with pass/fail stats
174 read status elapsed _ log_filename results_filename < $BASE_DIR/${spec_name}.task.txt
175
176 echo '<tr>'
177 echo "<td> <a href="$log_filename">$spec_name</a> </td>"
178
179 printf -v elapsed_str '%.1f' $elapsed
180 echo "<td>$elapsed_str</td>"
181
182 case $status in
183 (0) # exit code 0 is success
184 echo " <td>$status</td>"
185 ;;
186 (*) # everything else is a failure
187 # Add extra text to make red stand out.
188 echo " <td class=\"fail\">status: $status</td>"
189
190 # Mark failure
191 all_passed=1
192 ;;
193 esac
194 echo '</tr>'
195
196 # Append to temp file
197 {
198 echo "<h2>$spec_name</h2>"
199 echo '<pre>'
200 escape-html $BASE_DIR/$results_filename
201 echo '</pre>'
202 } >> $results_tmp
203
204 done
205 echo '</table>'
206
207 cat $results_tmp
208
209 cat <<EOF
210 </body>
211</html>
212EOF
213
214 log "all_passed = $all_passed"
215
216 return $all_passed
217}
218
219soil-run() {
220 ninja $OSH_CPP
221
222 mkdir -p $BASE_DIR
223
224 print-tasks | xargs -n 1 -- $0 run-file
225
226 # Returns whether all passed
227 html-summary > $BASE_DIR/index.html
228}
229
230#
231# Debugging
232#
233
234test-stop() {
235 python3 spec/stateful/harness.py test-stop demo/cpython/fork_signal_state.py
236}
237
238strace-py-fork() {
239 rm -f -v _tmp/py-fork.*
240 strace -ff -o _tmp/py-fork demo/cpython/fork_signal_state.py
241 ls -l _tmp/py-fork.*
242
243 # I see rt_sigaction(SIGSTP, ...) which is good
244 # so yeah this seems perfectly fine -- why is it ignoring SIGTSTP? :-(
245}
246
247"$@"