OILS / test / spec-cpp.sh View on Github | oils.pub

393 lines, 170 significant
1#!/usr/bin/env bash
2#
3# Test the C++ translation of Oils.
4#
5# Usage:
6# test/spec-cpp.sh <function name>
7#
8# Examples:
9# test/spec-cpp.sh run-file smoke -r 0 -v
10# NUM_SPEC_TASKS=2 test/spec-cpp.sh osh-all
11
12: ${LIB_OSH=stdlib/osh}
13source $LIB_OSH/bash-strict.sh
14source $LIB_OSH/task-five.sh
15
16REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
17
18source build/dev-shell.sh # PYTHONPATH
19source test/common.sh # html-head
20source test/spec-common.sh
21source test/tsv-lib.sh
22source web/table/html.sh
23
24shopt -s failglob # to debug TSV expansion failure below
25
26OSH_PY=$REPO_ROOT/bin/osh
27YSH_PY=$REPO_ROOT/bin/ysh
28
29# Run with ASAN binary by default. Release overrides this
30OSH_CC=${OSH_CC:-$REPO_ROOT/_bin/cxx-asan/osh}
31YSH_CC=${YSH_CC:-$REPO_ROOT/_bin/cxx-asan/ysh}
32
33# So we can pass ASAN. Note that test/spec-common.sh has to pass this to
34# sh_spec.py.
35export OILS_GC_ON_EXIT=1
36
37#
38# For translation
39#
40
41run-file() {
42 local spec_name=$1
43 shift
44
45 local spec_file=spec/$spec_name.test.sh
46
47 local suite
48 suite=$(test/sh_spec.py --print-spec-suite $spec_file)
49
50 local spec_subdir
51 case $suite in
52 osh) spec_subdir='osh-cpp' ;;
53 ysh) spec_subdir='ysh-cpp' ;;
54 disabled) spec_subdir='disabled-cpp' ;;
55 *) die "Invalid suite $suite" ;;
56 esac
57
58 local base_dir=_tmp/spec/$spec_subdir
59 mkdir -v -p $base_dir
60
61 # Compare Python and C++ shells by passing --oils-cpp-bin-dir
62
63 local variant='cxx-asan'
64 # TODO: turning on gcalways will find more bugs
65 #local variant='cxx-asan+gcalways'
66
67 sh-spec $spec_file \
68 --timeout 10 \
69 --oils-bin-dir $PWD/bin \
70 --oils-cpp-bin-dir $REPO_ROOT/_bin/$variant \
71 --tsv-output $base_dir/${spec_name}.result.tsv \
72 "$@"
73}
74
75osh-all() {
76 # Like test/spec.sh {osh,ysh}-all, but it compares against different binaries
77
78 # For debugging hangs
79 #export MAX_PROCS=1
80
81 ninja _bin/cxx-asan/{osh,ysh}
82
83 test/spec-runner.sh shell-sanity-check $OSH_PY $OSH_CC
84
85 local spec_subdir=osh-cpp
86
87 local status
88 set +o errexit
89 # $suite $compare_mode
90 test/spec-runner.sh all-parallel \
91 osh compare-cpp $spec_subdir "$@"
92 status=$?
93 set -o errexit
94
95 # Write comparison even if we failed
96 write-compare-html $spec_subdir
97
98 return $status
99}
100
101ysh-all() {
102 ninja _bin/cxx-asan/{osh,ysh}
103
104 local spec_subdir=ysh-cpp
105
106 local status
107 set +o errexit
108 # $suite $compare_mode
109 test/spec-runner.sh all-parallel \
110 ysh compare-cpp $spec_subdir "$@"
111 status=$?
112 set -o errexit
113
114 write-compare-html $spec_subdir
115
116 return $status
117}
118
119console-row() {
120 ### Print out a histogram of results
121
122 awk '
123FNR == 1 {
124 #print FILENAME > "/dev/stderr"
125}
126FNR != 1 {
127 case_num = $1
128 sh = $2
129 result = $3
130
131 if (sh == "osh") {
132 osh[result] += 1
133 } else if (sh == "osh_cpp") { # bin/osh_cpp
134 oe_py[result] += 1
135 } else if (sh == "osh_ALT") { # _bin/*/osh
136 oe_cpp[result] += 1
137 }
138}
139
140function print_hist(sh, hist) {
141 printf("%s\t", sh)
142
143 k = "pass"
144 printf("%s %4d\t", k, hist[k])
145 k = "FAIL"
146 printf("%s %4d\t", k, hist[k])
147
148 print ""
149
150 # This prints N-I, ok, bug, etc.
151 #for (k in hist) {
152 # printf("%s %s\t", k, hist[k])
153 #}
154
155}
156
157END {
158 print_hist("osh", osh)
159 print_hist("osh_cpp", oe_py)
160 print_hist("osh_ALT", oe_cpp)
161}
162 ' "$@"
163}
164
165console-summary() {
166 ### Report on our progress translating
167
168 local spec_subdir=$1
169
170 # Can't go at the top level because files won't exist!
171 readonly TSV=(_tmp/spec/$spec_subdir/*.result.tsv)
172
173 wc -l "${TSV[@]}"
174
175 for file in "${TSV[@]}"; do
176 echo
177 echo "$file"
178 console-row $file
179 done
180
181 echo
182 echo "TOTAL"
183 console-row "${TSV[@]}"
184}
185
186#
187# HTML
188#
189
190summary-tsv-row() {
191 ### Print one row or the last total row
192
193 local spec_subdir=$1
194 shift
195
196 local status cpp_failures_allowed cpp_failures
197
198 if test $# -eq 1; then
199 local spec_name=$1
200 local -a tsv_files=( _tmp/spec/$spec_subdir/$spec_name.result.tsv )
201 status=$( awk '{printf $1}' "_tmp/spec/$spec_subdir/$spec_name.task.txt" )
202 cpp_failures_allowed=$( awk '{printf $6}' "_tmp/spec/$spec_subdir/$spec_name.stats.txt" )
203 cpp_failures=$( awk '{printf $7}' "_tmp/spec/$spec_subdir/$spec_name.stats.txt" )
204 else
205 local spec_name='TOTAL'
206 local -a tsv_files=( "$@" )
207 cpp_failures_allowed="-"
208 cpp_failures="-"
209 status=0
210 fi
211
212 awk -v spec_name=$spec_name -v status=$status \
213 -v cpp_failures_allowed=$cpp_failures_allowed -v cpp_failures=$cpp_failures '
214# skip the first row
215FNR != 1 {
216 case_num = $1
217 sh = $2
218 result = $3
219
220 if (sh == "osh" || sh == "ysh") {
221 osh[result] += 1
222 } else if (sh == "osh-cpp" || sh == "ysh-cpp") { # bin/osh
223 osh_native[result] += 1
224 }
225}
226
227END {
228 num_py = osh["pass"]
229 num_cpp = osh_native["pass"]
230 if (spec_name == "TOTAL") {
231 href = ""
232 } else {
233 href = sprintf("%s.html", spec_name)
234 }
235
236 if (num_py == num_cpp) {
237 row_css_class = "cpp-good" # green
238 }
239
240 if (status != 0) {
241 row_css_class = "cpp-failed" # red
242 }
243
244 row = sprintf("%s %s %s %d %d %d %d %d",
245 row_css_class,
246 spec_name, href,
247 num_py,
248 num_cpp,
249 num_py - num_cpp,
250 cpp_failures,
251 cpp_failures_allowed)
252
253 # Turn tabs into spaces - awk mutates the row!
254 gsub(/ /, "\t", row)
255 print row
256}
257' "${tsv_files[@]}"
258}
259
260summary-tsv() {
261 local spec_subdir=$1
262
263 local sh_label
264 local manifest
265
266 case $spec_subdir in
267 osh-cpp)
268 sh_label=osh
269 manifest=_tmp/spec/SUITE-osh.txt
270 ;;
271 ysh-cpp)
272 sh_label=ysh
273 manifest=_tmp/spec/SUITE-ysh.txt
274 ;;
275 *)
276 die "Invalid dir $spec_subdir"
277 ;;
278 esac
279
280 # Can't go at the top level because files might not exist!
281 #echo "ROW_CSS_CLASS,name,name_HREF,${sh_label}_py,${sh_label}_cpp,delta"
282 tsv-row \
283 'ROW_CSS_CLASS' 'name' 'name_HREF' ${sh_label}_py ${sh_label}_cpp 'delta' 'cpp_num_failed' 'cpp_failures_allowed'
284
285 # total row rows goes at the TOP, so it's in <thead> and not sorted.
286 summary-tsv-row $spec_subdir _tmp/spec/$spec_subdir/*.result.tsv
287
288 head -n $NUM_SPEC_TASKS $manifest | sort |
289 while read spec_name; do
290 summary-tsv-row $spec_subdir $spec_name
291 done
292}
293
294html-summary-header() {
295 local prefix=../../..
296
297 spec-html-head $prefix 'Passing Spec Tests in C++'
298 table-sort-begin "width50"
299
300 echo '
301<p id="home-link">
302 <!-- The release index is two dirs up -->
303 <a href="../..">Up</a> |
304 <a href="/">oils.pub</a>
305</p>
306
307<h1>Python vs C++</h1>
308
309<p>Here is the total number of passing tests. TODO: we should also verify
310tests that do not pass.
311</p>
312
313<p>Another view: <a href="index.html">index.html</a>.
314</p>
315'
316}
317
318html-summary-footer() {
319 echo '
320<p>Generated by <code>test/spec-cpp.sh</code>.
321</p>
322
323<p><a href="SUMMARY.tsv">Raw TSV</a>
324</p>
325'
326 table-sort-end 'SUMMARY' # The table name
327}
328
329write-compare-html() {
330 local spec_subdir=$1
331
332 local sh_label
333 case $spec_subdir in
334 osh-cpp)
335 sh_label=osh
336 ;;
337 ysh-cpp)
338 sh_label=ysh
339 ;;
340 *)
341 die "Invalid dir $spec_subdir"
342 ;;
343 esac
344
345 local dir=_tmp/spec/$spec_subdir
346 local out=$dir/compare.html
347
348 summary-tsv $spec_subdir >$dir/SUMMARY.tsv
349
350 # The underscores are stripped when we don't want them to be!
351 # Note: we could also put "pretty_heading" in the schema
352
353 here-schema-tsv >$dir/SUMMARY.schema.tsv <<EOF
354column_name type
355ROW_CSS_CLASS string
356name string
357name_HREF string
358${sh_label}_py integer
359${sh_label}_cpp integer
360cpp_num_failed integer
361cpp_failures_allowed integer
362delta integer
363EOF
364
365 { html-summary-header
366 # total row isn't sorted
367 tsv2html --thead-offset 1 $dir/SUMMARY.tsv
368 html-summary-footer
369 } > $out
370
371 log "Comparison: file://$REPO_ROOT/$out"
372}
373
374#
375# Misc
376#
377
378tsv-demo() {
379 sh-spec spec/arith.test.sh --tsv-output _tmp/arith.tsv dash bash "$@"
380 cat _tmp/arith.tsv
381}
382
383repro() {
384 test/spec.sh alias -r 0 -p > _tmp/a
385 ninja _bin/clang-dbg/osh
386 _bin/clang-dbg/osh _tmp/a
387}
388
389repro-all() {
390 OSH_CC=$REPO_ROOT/_bin/clang-dbg/osh $0 all
391}
392
393task-five "$@"