OILS / test / spec-cpp.sh View on Github | oils.pub

391 lines, 169 significant
1#!/usr/bin/env bash
2#
3# Test the C++ translation of Oils.
4#
5# Usage:
6# test/spec-cpp.sh <function name>
7#
8# Examples:
9# test/spec-cpp.sh run-file smoke -r 0 -v
10# NUM_SPEC_TASKS=2 test/spec-cpp.sh osh-all
11
12: ${LIB_OSH=stdlib/osh}
13source $LIB_OSH/bash-strict.sh
14source $LIB_OSH/task-five.sh
15
16REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
17
18source build/dev-shell.sh # PYTHONPATH
19source test/common.sh # html-head
20source test/spec-common.sh
21source test/tsv-lib.sh
22source web/table/html.sh
23
24shopt -s failglob # to debug TSV expansion failure below
25
26OSH_PY=$REPO_ROOT/bin/osh
27YSH_PY=$REPO_ROOT/bin/ysh
28
29# Run with ASAN binary by default. Release overrides this
30OSH_CC=${OSH_CC:-$REPO_ROOT/_bin/cxx-asan/osh}
31YSH_CC=${YSH_CC:-$REPO_ROOT/_bin/cxx-asan/ysh}
32
33# So we can pass ASAN. Note that test/spec-common.sh has to pass this to
34# sh_spec.py.
35export OILS_GC_ON_EXIT=1
36
37#
38# For translation
39#
40
41run-file() {
42 local spec_name=$1
43 shift
44
45 local spec_file=spec/$spec_name.test.sh
46
47 local suite
48 suite=$(test/sh_spec.py --print-spec-suite $spec_file)
49
50 local spec_subdir
51 case $suite in
52 osh) spec_subdir='osh-cpp' ;;
53 ysh) spec_subdir='ysh-cpp' ;;
54 disabled) spec_subdir='disabled-cpp' ;;
55 *) die "Invalid suite $suite" ;;
56 esac
57
58 local base_dir=_tmp/spec/$spec_subdir
59 mkdir -v -p $base_dir
60
61 # Compare Python and C++ shells by passing --oils-cpp-bin-dir
62
63 local variant='cxx-asan'
64 # TODO: turning on gcalways will find more bugs
65 #local variant='cxx-asan+gcalways'
66
67 sh-spec $spec_file \
68 --timeout 10 \
69 --oils-bin-dir $PWD/bin \
70 --oils-cpp-bin-dir $REPO_ROOT/_bin/$variant \
71 --tsv-output $base_dir/${spec_name}.result.tsv \
72 "$@"
73}
74
75osh-all() {
76 # Like test/spec.sh {osh,ysh}-all, but it compares against different binaries
77
78 # For debugging hangs
79 #export MAX_PROCS=1
80
81 ninja _bin/cxx-asan/{osh,ysh}
82
83 test/spec-runner.sh shell-sanity-check $OSH_PY $OSH_CC
84
85 local spec_subdir=osh-cpp
86
87 local status
88 set +o errexit
89 # $suite $compare_mode
90 test/spec-runner.sh all-parallel \
91 osh compare-cpp $spec_subdir "$@"
92 status=$?
93 set -o errexit
94
95 # Write comparison even if we failed
96 write-compare-html $spec_subdir
97
98 return $status
99}
100
101ysh-all() {
102 ninja _bin/cxx-asan/{osh,ysh}
103
104 local spec_subdir=ysh-cpp
105
106 local status
107 set +o errexit
108 # $suite $compare_mode
109 test/spec-runner.sh all-parallel \
110 ysh compare-cpp $spec_subdir "$@"
111 status=$?
112 set -o errexit
113
114 write-compare-html $spec_subdir
115
116 return $status
117}
118
119console-row() {
120 ### Print out a histogram of results
121
122 awk '
123FNR == 1 {
124 #print FILENAME > "/dev/stderr"
125}
126FNR != 1 {
127 case_num = $1
128 sh = $2
129 result = $3
130
131 if (sh == "osh") {
132 osh[result] += 1
133 } else if (sh == "osh_cpp") { # bin/osh_cpp
134 oe_py[result] += 1
135 } else if (sh == "osh_ALT") { # _bin/*/osh
136 oe_cpp[result] += 1
137 }
138}
139
140function print_hist(sh, hist) {
141 printf("%s\t", sh)
142
143 k = "pass"
144 printf("%s %4d\t", k, hist[k])
145 k = "FAIL"
146 printf("%s %4d\t", k, hist[k])
147
148 print ""
149
150 # This prints N-I, ok, bug, etc.
151 #for (k in hist) {
152 # printf("%s %s\t", k, hist[k])
153 #}
154
155}
156
157END {
158 print_hist("osh", osh)
159 print_hist("osh_cpp", oe_py)
160 print_hist("osh_ALT", oe_cpp)
161}
162 ' "$@"
163}
164
165console-summary() {
166 ### Report on our progress translating
167
168 local spec_subdir=$1
169
170 # Can't go at the top level because files won't exist!
171 readonly TSV=(_tmp/spec/$spec_subdir/*.result.tsv)
172
173 wc -l "${TSV[@]}"
174
175 for file in "${TSV[@]}"; do
176 echo
177 echo "$file"
178 console-row $file
179 done
180
181 echo
182 echo "TOTAL"
183 console-row "${TSV[@]}"
184}
185
186#
187# HTML
188#
189
190summary-tsv-row() {
191 ### Print one row or the last total row
192
193 local spec_subdir=$1
194 shift
195
196 if test $# -eq 1; then
197 local spec_name=$1
198 local -a tsv_files=( _tmp/spec/$spec_subdir/$spec_name.result.tsv )
199 status=`awk '{printf $1}' "_tmp/spec/$spec_subdir/$spec_name.task.txt"`
200 local -a cpp_failures_allowed=(`awk '{printf $6}' "_tmp/spec/$spec_subdir/$spec_name.stats.txt"`)
201 local -a cpp_failures=(`awk '{printf $7}' "_tmp/spec/$spec_subdir/$spec_name.stats.txt"`)
202 else
203 local spec_name='TOTAL'
204 local -a tsv_files=( "$@" )
205 local -a cpp_failures_allowed=( "$@" )
206 local -a cpp_failures=( "$@" )
207 status=0
208 fi
209
210 awk -v spec_name=$spec_name -v status=$status \
211 -v cpp_failures_allowed=$cpp_failures_allowed -v cpp_failures=$cpp_failures '
212# skip the first row
213FNR != 1 {
214 case_num = $1
215 sh = $2
216 result = $3
217
218 if (sh == "osh" || sh == "ysh") {
219 osh[result] += 1
220 } else if (sh == "osh-cpp" || sh == "ysh-cpp") { # bin/osh
221 osh_native[result] += 1
222 }
223}
224
225END {
226 num_py = osh["pass"]
227 num_cpp = osh_native["pass"]
228 if (spec_name == "TOTAL") {
229 href = ""
230 } else {
231 href = sprintf("%s.html", spec_name)
232 }
233
234 if (num_py == num_cpp) {
235 row_css_class = "cpp-good" # green
236 }
237
238 if (status != 0) {
239 row_css_class = "cpp-failed" # red
240 }
241
242 row = sprintf("%s %s %s %d %d %d %d %d",
243 row_css_class,
244 spec_name, href,
245 num_py,
246 num_cpp,
247 num_py - num_cpp,
248 cpp_failures,
249 cpp_failures_allowed)
250
251 # Turn tabs into spaces - awk mutates the row!
252 gsub(/ /, "\t", row)
253 print row
254}
255' "${tsv_files[@]}"
256}
257
258summary-tsv() {
259 local spec_subdir=$1
260
261 local sh_label
262 local manifest
263
264 case $spec_subdir in
265 osh-cpp)
266 sh_label=osh
267 manifest=_tmp/spec/SUITE-osh.txt
268 ;;
269 ysh-cpp)
270 sh_label=ysh
271 manifest=_tmp/spec/SUITE-ysh.txt
272 ;;
273 *)
274 die "Invalid dir $spec_subdir"
275 ;;
276 esac
277
278 # Can't go at the top level because files might not exist!
279 #echo "ROW_CSS_CLASS,name,name_HREF,${sh_label}_py,${sh_label}_cpp,delta"
280 tsv-row \
281 'ROW_CSS_CLASS' 'name' 'name_HREF' ${sh_label}_py ${sh_label}_cpp 'delta' 'cpp_num_failed' 'cpp_failures_allowed'
282
283 # total row rows goes at the TOP, so it's in <thead> and not sorted.
284 summary-tsv-row $spec_subdir _tmp/spec/$spec_subdir/*.result.tsv
285
286 head -n $NUM_SPEC_TASKS $manifest | sort |
287 while read spec_name; do
288 summary-tsv-row $spec_subdir $spec_name
289 done
290}
291
292html-summary-header() {
293 local prefix=../../..
294
295 spec-html-head $prefix 'Passing Spec Tests in C++'
296 table-sort-begin "width50"
297
298 echo '
299<p id="home-link">
300 <!-- The release index is two dirs up -->
301 <a href="../..">Up</a> |
302 <a href="/">oils.pub</a>
303</p>
304
305<h1>Python vs C++</h1>
306
307<p>Here is the total number of passing tests. TODO: we should also verify
308tests that do not pass.
309</p>
310
311<p>Another view: <a href="index.html">index.html</a>.
312</p>
313'
314}
315
316html-summary-footer() {
317 echo '
318<p>Generated by <code>test/spec-cpp.sh</code>.
319</p>
320
321<p><a href="SUMMARY.tsv">Raw TSV</a>
322</p>
323'
324 table-sort-end 'SUMMARY' # The table name
325}
326
327write-compare-html() {
328 local spec_subdir=$1
329
330 local sh_label
331 case $spec_subdir in
332 osh-cpp)
333 sh_label=osh
334 ;;
335 ysh-cpp)
336 sh_label=ysh
337 ;;
338 *)
339 die "Invalid dir $spec_subdir"
340 ;;
341 esac
342
343 local dir=_tmp/spec/$spec_subdir
344 local out=$dir/compare.html
345
346 summary-tsv $spec_subdir >$dir/SUMMARY.tsv
347
348 # The underscores are stripped when we don't want them to be!
349 # Note: we could also put "pretty_heading" in the schema
350
351 here-schema-tsv >$dir/SUMMARY.schema.tsv <<EOF
352column_name type
353ROW_CSS_CLASS string
354name string
355name_HREF string
356${sh_label}_py integer
357${sh_label}_cpp integer
358cpp_num_failed integer
359cpp_failures_allowed integer
360delta integer
361EOF
362
363 { html-summary-header
364 # total row isn't sorted
365 tsv2html --thead-offset 1 $dir/SUMMARY.tsv
366 html-summary-footer
367 } > $out
368
369 log "Comparison: file://$REPO_ROOT/$out"
370}
371
372#
373# Misc
374#
375
376tsv-demo() {
377 sh-spec spec/arith.test.sh --tsv-output _tmp/arith.tsv dash bash "$@"
378 cat _tmp/arith.tsv
379}
380
381repro() {
382 test/spec.sh alias -r 0 -p > _tmp/a
383 ninja _bin/clang-dbg/osh
384 _bin/clang-dbg/osh _tmp/a
385}
386
387repro-all() {
388 OSH_CC=$REPO_ROOT/_bin/clang-dbg/osh $0 all
389}
390
391task-five "$@"