OILS / test / spec-cpp.sh View on Github | oils.pub

368 lines, 164 significant
1#!/usr/bin/env bash
2#
3# Test the C++ translation of Oils.
4#
5# Usage:
6# test/spec-cpp.sh <function name>
7#
8# Examples:
9# test/spec-cpp.sh run-file smoke -r 0 -v
10# NUM_SPEC_TASKS=2 test/spec-cpp.sh osh-all
11
12: ${LIB_OSH=stdlib/osh}
13source $LIB_OSH/bash-strict.sh
14source $LIB_OSH/task-five.sh
15
16REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
17
18source build/dev-shell.sh # PYTHONPATH
19source test/common.sh # html-head
20source test/spec-common.sh
21source web/table/html.sh
22
23shopt -s failglob # to debug TSV expansion failure below
24
25OSH_PY=$REPO_ROOT/bin/osh
26YSH_PY=$REPO_ROOT/bin/ysh
27
28# Run with ASAN binary by default. Release overrides this
29OSH_CC=${OSH_CC:-$REPO_ROOT/_bin/cxx-asan/osh}
30YSH_CC=${YSH_CC:-$REPO_ROOT/_bin/cxx-asan/ysh}
31
32# Same variable in test/spec-runner.sh
33NUM_SPEC_TASKS=${NUM_SPEC_TASKS:-400}
34
35# So we can pass ASAN. Note that test/spec-common.sh has to pass this to
36# sh_spec.py.
37export OILS_GC_ON_EXIT=1
38
39#
40# For translation
41#
42
43run-file() {
44 local spec_name=$1
45 shift
46
47 local spec_file=spec/$spec_name.test.sh
48
49 local suite
50 suite=$(test/sh_spec.py --print-spec-suite $spec_file)
51
52 local spec_subdir
53 case $suite in
54 osh) spec_subdir='osh-cpp' ;;
55 ysh) spec_subdir='ysh-cpp' ;;
56 disabled) spec_subdir='disabled-cpp' ;;
57 *) die "Invalid suite $suite" ;;
58 esac
59
60 local base_dir=_tmp/spec/$spec_subdir
61 mkdir -v -p $base_dir
62
63 # Compare Python and C++ shells by passing --oils-cpp-bin-dir
64 sh-spec $spec_file \
65 --timeout 10 \
66 --oils-bin-dir $PWD/bin \
67 --oils-cpp-bin-dir $REPO_ROOT/_bin/cxx-asan \
68 --tsv-output $base_dir/${spec_name}.tsv \
69 "$@"
70}
71
72osh-all() {
73 # Like test/spec.sh {osh,ysh}-all, but it compares against different binaries
74
75 # For debugging hangs
76 #export MAX_PROCS=1
77
78 ninja _bin/cxx-asan/{osh,ysh}
79
80 test/spec-runner.sh shell-sanity-check $OSH_PY $OSH_CC
81
82 local spec_subdir=osh-cpp
83
84 local status
85 set +o errexit
86 # $suite $compare_mode
87 test/spec-runner.sh all-parallel \
88 osh compare-cpp $spec_subdir "$@"
89 status=$?
90 set -o errexit
91
92 # Write comparison even if we failed
93 write-compare-html $spec_subdir
94
95 return $status
96}
97
98ysh-all() {
99 ninja _bin/cxx-asan/{osh,ysh}
100
101 local spec_subdir=ysh-cpp
102
103 # $suite $compare_mode
104 test/spec-runner.sh all-parallel \
105 ysh compare-cpp $spec_subdir "$@"
106
107 write-compare-html $spec_subdir
108}
109
110console-row() {
111 ### Print out a histogram of results
112
113 awk '
114FNR == 1 {
115 #print FILENAME > "/dev/stderr"
116}
117FNR != 1 {
118 case_num = $1
119 sh = $2
120 result = $3
121
122 if (sh == "osh") {
123 osh[result] += 1
124 } else if (sh == "osh_cpp") { # bin/osh_cpp
125 oe_py[result] += 1
126 } else if (sh == "osh_ALT") { # _bin/*/osh
127 oe_cpp[result] += 1
128 }
129}
130
131function print_hist(sh, hist) {
132 printf("%s\t", sh)
133
134 k = "pass"
135 printf("%s %4d\t", k, hist[k])
136 k = "FAIL"
137 printf("%s %4d\t", k, hist[k])
138
139 print ""
140
141 # This prints N-I, ok, bug, etc.
142 #for (k in hist) {
143 # printf("%s %s\t", k, hist[k])
144 #}
145
146}
147
148END {
149 print_hist("osh", osh)
150 print_hist("osh_cpp", oe_py)
151 print_hist("osh_ALT", oe_cpp)
152}
153 ' "$@"
154}
155
156console-summary() {
157 ### Report on our progress translating
158
159 local spec_subdir=$1
160
161 # Can't go at the top level because files won't exist!
162 readonly TSV=(_tmp/spec/$spec_subdir/*.tsv)
163
164 wc -l "${TSV[@]}"
165
166 for file in "${TSV[@]}"; do
167 echo
168 echo "$file"
169 console-row $file
170 done
171
172 echo
173 echo "TOTAL"
174 console-row "${TSV[@]}"
175}
176
177#
178# HTML
179#
180
181summary-csv-row() {
182 ### Print one row or the last total row
183
184 local spec_subdir=$1
185 shift
186
187 if test $# -eq 1; then
188 local spec_name=$1
189 local -a tsv_files=( _tmp/spec/$spec_subdir/$spec_name.tsv )
190 else
191 local spec_name='TOTAL'
192 local -a tsv_files=( "$@" )
193 fi
194
195 awk -v spec_name=$spec_name '
196# skip the first row
197FNR != 1 {
198 case_num = $1
199 sh = $2
200 result = $3
201
202 if (sh == "osh" || sh == "ysh") {
203 osh[result] += 1
204 } else if (sh == "osh-cpp" || sh == "ysh-cpp") { # bin/osh
205 osh_native[result] += 1
206 }
207}
208
209END {
210 num_py = osh["pass"]
211 num_cpp = osh_native["pass"]
212 if (spec_name == "TOTAL") {
213 href = ""
214 } else {
215 href = sprintf("%s.html", spec_name)
216 }
217
218 if (num_py == num_cpp) {
219 row_css_class = "cpp-good" # green
220 }
221
222 printf("%s,%s,%s,%d,%d,%d\n",
223 row_css_class,
224 spec_name, href,
225 num_py,
226 num_cpp,
227 num_py - num_cpp)
228}
229' "${tsv_files[@]}"
230}
231
232summary-csv() {
233 local spec_subdir=$1
234
235 local sh_label
236 local manifest
237
238 case $spec_subdir in
239 osh-cpp)
240 sh_label=osh
241 manifest=_tmp/spec/SUITE-osh.txt
242 ;;
243 ysh-cpp)
244 sh_label=ysh
245 manifest=_tmp/spec/SUITE-ysh.txt
246 ;;
247 *)
248 die "Invalid dir $spec_subdir"
249 ;;
250 esac
251
252 # Can't go at the top level because files might not exist!
253 cat <<EOF
254ROW_CSS_CLASS,name,name_HREF,${sh_label}_py,${sh_label}_cpp,delta
255EOF
256
257 # total row rows goes at the TOP, so it's in <thead> and not sorted.
258 summary-csv-row $spec_subdir _tmp/spec/$spec_subdir/*.tsv
259
260 head -n $NUM_SPEC_TASKS $manifest | sort |
261 while read spec_name; do
262 summary-csv-row $spec_subdir $spec_name
263 done
264}
265
266html-summary-header() {
267 local prefix=../../..
268 html-head --title 'Passing Spec Tests in C++' \
269 $prefix/web/ajax.js \
270 $prefix/web/table/table-sort.js $prefix/web/table/table-sort.css \
271 $prefix/web/base.css \
272 $prefix/web/spec-cpp.css
273
274 table-sort-begin "width50"
275
276 cat <<EOF
277<p id="home-link">
278 <!-- The release index is two dirs up -->
279 <a href="../..">Up</a> |
280 <a href="/">oils.pub</a>
281</p>
282
283<h1>Python vs C++</h1>
284
285<p>These numbers measure the progress of the C++ translation.
286Compare with <a href=".">index.html</a>.
287</p>
288
289EOF
290}
291
292html-summary-footer() {
293 cat <<EOF
294<p>Generated by <code>test/spec-cpp.sh</code>.</p>
295EOF
296 table-sort-end 'summary' # The table name
297}
298
299# TODO: Use here-schema-tsv in test/tsv-lib.sh
300here-schema() {
301 ### Read a legible text format on stdin, and write CSV on stdout
302
303 # This is a little like: https://wiki.xxiivv.com/site/tablatal.html
304 # TODO: generalize this in stdlib/here.sh
305 while read one two; do
306 echo "$one,$two"
307 done
308}
309
310write-compare-html() {
311 local spec_subdir=$1
312
313 local sh_label
314 case $spec_subdir in
315 osh-cpp)
316 sh_label=osh
317 ;;
318 ysh-cpp)
319 sh_label=ysh
320 ;;
321 *)
322 die "Invalid dir $spec_subdir"
323 ;;
324 esac
325
326 local dir=_tmp/spec/$spec_subdir
327 local out=$dir/compare.html
328
329 summary-csv $spec_subdir >$dir/summary.csv
330
331 # The underscores are stripped when we don't want them to be!
332 # Note: we could also put "pretty_heading" in the schema
333
334 here-schema >$dir/summary.schema.csv <<EOF
335column_name type
336ROW_CSS_CLASS string
337name string
338name_HREF string
339${sh_label}_py integer
340${sh_label}_cpp integer
341delta integer
342EOF
343
344 { html-summary-header
345 # total row isn't sorted
346 web/table/csv2html.py --thead-offset 1 $dir/summary.csv
347 html-summary-footer
348 } > $out
349
350 log "Comparison: file://$REPO_ROOT/$out"
351}
352
353tsv-demo() {
354 sh-spec spec/arith.test.sh --tsv-output _tmp/arith.tsv dash bash "$@"
355 cat _tmp/arith.tsv
356}
357
358repro() {
359 test/spec.sh alias -r 0 -p > _tmp/a
360 ninja _bin/clang-dbg/osh
361 _bin/clang-dbg/osh _tmp/a
362}
363
364repro-all() {
365 OSH_CC=$REPO_ROOT/_bin/clang-dbg/osh $0 all
366}
367
368task-five "$@"