OILS / test / spec-cpp.sh View on Github | oilshell.org

365 lines, 158 significant
1#!/usr/bin/env bash
2#
3# Test the C++ translation of Oils.
4#
5# Usage:
6# test/spec-cpp.sh <function name>
7#
8# Examples:
9# test/spec-cpp.sh run-file smoke -r 0 -v
10# NUM_SPEC_TASKS=2 test/spec-cpp.sh osh-all
11
12: ${LIB_OSH=stdlib/osh}
13source $LIB_OSH/bash-strict.sh
14source $LIB_OSH/task-five.sh
15
16REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
17
18source build/dev-shell.sh # PYTHONPATH
19source test/common.sh # html-head
20source test/spec-common.sh
21source web/table/html.sh
22
23shopt -s failglob # to debug TSV expansion failure below
24
25OSH_PY=$REPO_ROOT/bin/osh
26YSH_PY=$REPO_ROOT/bin/ysh
27
28# Run with ASAN binary by default. Release overrides this
29OSH_CC=${OSH_CC:-$REPO_ROOT/_bin/cxx-asan/osh}
30YSH_CC=${YSH_CC:-$REPO_ROOT/_bin/cxx-asan/ysh}
31
32# Same variable in test/spec-runner.sh
33NUM_SPEC_TASKS=${NUM_SPEC_TASKS:-400}
34
35# So we can pass ASAN. Note that test/spec-common.sh has to pass this to
36# sh_spec.py.
37export OILS_GC_ON_EXIT=1
38
39#
40# For translation
41#
42
43run-file() {
44 local spec_name=$1
45 shift
46
47 local spec_file=spec/$spec_name.test.sh
48
49 local suite
50 suite=$(test/sh_spec.py --print-spec-suite $spec_file)
51
52 local spec_subdir
53 case $suite in
54 osh) spec_subdir='osh-cpp' ;;
55 ysh) spec_subdir='ysh-cpp' ;;
56 *) die "Invalid suite $suite" ;;
57 esac
58
59 local base_dir=_tmp/spec/$spec_subdir
60 mkdir -v -p $base_dir
61
62 # Compare Python and C++ shells by passing --oils-cpp-bin-dir
63 sh-spec $spec_file \
64 --timeout 10 \
65 --oils-bin-dir $PWD/bin \
66 --oils-cpp-bin-dir $REPO_ROOT/_bin/cxx-asan \
67 --tsv-output $base_dir/${spec_name}.tsv \
68 "$@"
69}
70
71osh-all() {
72 # Like test/spec.sh {osh,ysh}-all, but it compares against different binaries
73
74 # For debugging hangs
75 #export MAX_PROCS=1
76
77 ninja _bin/cxx-asan/{osh,ysh}
78
79 test/spec-runner.sh shell-sanity-check $OSH_PY $OSH_CC
80
81 local spec_subdir=osh-cpp
82
83 # TODO: test/spec-cpp.sh runfile X fails, where X=let,assign-deferred,assoc-zsh
84 # This is because they are using the old style.
85 # But also spec/nul-bytes passes when it should probably fail -- we get 1
86 # failure instead of 2.
87
88 # $suite $compare_mode
89 test/spec-runner.sh all-parallel \
90 osh compare-cpp $spec_subdir "$@" || true # OK if it fails
91
92 write-compare-html $spec_subdir
93}
94
95ysh-all() {
96 ninja _bin/cxx-asan/{osh,ysh}
97
98 local spec_subdir=ysh-cpp
99
100 # $suite $compare_mode
101 test/spec-runner.sh all-parallel \
102 ysh compare-cpp $spec_subdir "$@"
103
104 write-compare-html $spec_subdir
105}
106
107console-row() {
108 ### Print out a histogram of results
109
110 awk '
111FNR == 1 {
112 #print FILENAME > "/dev/stderr"
113}
114FNR != 1 {
115 case_num = $1
116 sh = $2
117 result = $3
118
119 if (sh == "osh") {
120 osh[result] += 1
121 } else if (sh == "osh_cpp") { # bin/osh_cpp
122 oe_py[result] += 1
123 } else if (sh == "osh_ALT") { # _bin/*/osh
124 oe_cpp[result] += 1
125 }
126}
127
128function print_hist(sh, hist) {
129 printf("%s\t", sh)
130
131 k = "pass"
132 printf("%s %4d\t", k, hist[k])
133 k = "FAIL"
134 printf("%s %4d\t", k, hist[k])
135
136 print ""
137
138 # This prints N-I, ok, bug, etc.
139 #for (k in hist) {
140 # printf("%s %s\t", k, hist[k])
141 #}
142
143}
144
145END {
146 print_hist("osh", osh)
147 print_hist("osh_cpp", oe_py)
148 print_hist("osh_ALT", oe_cpp)
149}
150 ' "$@"
151}
152
153console-summary() {
154 ### Report on our progress translating
155
156 local spec_subdir=$1
157
158 # Can't go at the top level because files won't exist!
159 readonly TSV=(_tmp/spec/$spec_subdir/*.tsv)
160
161 wc -l "${TSV[@]}"
162
163 for file in "${TSV[@]}"; do
164 echo
165 echo "$file"
166 console-row $file
167 done
168
169 echo
170 echo "TOTAL"
171 console-row "${TSV[@]}"
172}
173
174#
175# HTML
176#
177
178summary-csv-row() {
179 ### Print one row or the last total row
180
181 local spec_subdir=$1
182 shift
183
184 if test $# -eq 1; then
185 local spec_name=$1
186 local -a tsv_files=( _tmp/spec/$spec_subdir/$spec_name.tsv )
187 else
188 local spec_name='TOTAL'
189 local -a tsv_files=( "$@" )
190 fi
191
192 awk -v spec_name=$spec_name '
193# skip the first row
194FNR != 1 {
195 case_num = $1
196 sh = $2
197 result = $3
198
199 if (sh == "osh" || sh == "ysh") {
200 osh[result] += 1
201 } else if (sh == "osh-cpp" || sh == "ysh-cpp") { # bin/osh
202 osh_native[result] += 1
203 }
204}
205
206END {
207 num_py = osh["pass"]
208 num_cpp = osh_native["pass"]
209 if (spec_name == "TOTAL") {
210 href = ""
211 } else {
212 href = sprintf("%s.html", spec_name)
213 }
214
215 if (num_py == num_cpp) {
216 row_css_class = "cpp-good" # green
217 }
218
219 printf("%s,%s,%s,%d,%d,%d\n",
220 row_css_class,
221 spec_name, href,
222 num_py,
223 num_cpp,
224 num_py - num_cpp)
225}
226' "${tsv_files[@]}"
227}
228
229summary-csv() {
230 local spec_subdir=$1
231
232 local sh_label
233 local manifest
234
235 case $spec_subdir in
236 osh-cpp)
237 sh_label=osh
238 manifest=_tmp/spec/SUITE-osh.txt
239 ;;
240 ysh-cpp)
241 sh_label=ysh
242 manifest=_tmp/spec/SUITE-ysh.txt
243 ;;
244 *)
245 die "Invalid dir $spec_subdir"
246 ;;
247 esac
248
249 # Can't go at the top level because files might not exist!
250 cat <<EOF
251ROW_CSS_CLASS,name,name_HREF,${sh_label}_py,${sh_label}_cpp,delta
252EOF
253
254 # total row rows goes at the TOP, so it's in <thead> and not sorted.
255 summary-csv-row $spec_subdir _tmp/spec/$spec_subdir/*.tsv
256
257 head -n $NUM_SPEC_TASKS $manifest | sort |
258 while read spec_name; do
259 summary-csv-row $spec_subdir $spec_name
260 done
261}
262
263html-summary-header() {
264 local prefix=../../..
265 html-head --title 'Passing Spec Tests in C++' \
266 $prefix/web/ajax.js \
267 $prefix/web/table/table-sort.js $prefix/web/table/table-sort.css \
268 $prefix/web/base.css \
269 $prefix/web/spec-cpp.css
270
271 table-sort-begin "width50"
272
273 cat <<EOF
274<p id="home-link">
275 <!-- The release index is two dirs up -->
276 <a href="../..">Up</a> |
277 <a href="/">oilshell.org</a>
278</p>
279
280<h1>Python vs C++</h1>
281
282<p>These numbers measure the progress of the C++ translation.
283Compare with <a href=".">index.html</a>.
284</p>
285
286EOF
287}
288
289html-summary-footer() {
290 cat <<EOF
291<p>Generated by <code>test/spec-cpp.sh</code>.</p>
292EOF
293 table-sort-end 'summary' # The table name
294}
295
296# TODO: Use here-schema-tsv in test/tsv-lib.sh
297here-schema() {
298 ### Read a legible text format on stdin, and write CSV on stdout
299
300 # This is a little like: https://wiki.xxiivv.com/site/tablatal.html
301 # TODO: generalize this in stdlib/here.sh
302 while read one two; do
303 echo "$one,$two"
304 done
305}
306
307write-compare-html() {
308 local spec_subdir=$1
309
310 local sh_label
311 case $spec_subdir in
312 osh-cpp)
313 sh_label=osh
314 ;;
315 ysh-cpp)
316 sh_label=ysh
317 ;;
318 *)
319 die "Invalid dir $spec_subdir"
320 ;;
321 esac
322
323 local dir=_tmp/spec/$spec_subdir
324 local out=$dir/compare.html
325
326 summary-csv $spec_subdir >$dir/summary.csv
327
328 # The underscores are stripped when we don't want them to be!
329 # Note: we could also put "pretty_heading" in the schema
330
331 here-schema >$dir/summary.schema.csv <<EOF
332column_name type
333ROW_CSS_CLASS string
334name string
335name_HREF string
336${sh_label}_py integer
337${sh_label}_cpp integer
338delta integer
339EOF
340
341 { html-summary-header
342 # total row isn't sorted
343 web/table/csv2html.py --thead-offset 1 $dir/summary.csv
344 html-summary-footer
345 } > $out
346
347 log "Comparison: file://$REPO_ROOT/$out"
348}
349
350tsv-demo() {
351 sh-spec spec/arith.test.sh --tsv-output _tmp/arith.tsv dash bash "$@"
352 cat _tmp/arith.tsv
353}
354
355repro() {
356 test/spec.sh alias -r 0 -p > _tmp/a
357 ninja _bin/clang-dbg/osh
358 _bin/clang-dbg/osh _tmp/a
359}
360
361repro-all() {
362 OSH_CC=$REPO_ROOT/_bin/clang-dbg/osh $0 all
363}
364
365task-five "$@"