OILS / test / spec-cpp.sh View on Github | oils.pub

373 lines, 165 significant
1#!/usr/bin/env bash
2#
3# Test the C++ translation of Oils.
4#
5# Usage:
6# test/spec-cpp.sh <function name>
7#
8# Examples:
9# test/spec-cpp.sh run-file smoke -r 0 -v
10# NUM_SPEC_TASKS=2 test/spec-cpp.sh osh-all
11
12: ${LIB_OSH=stdlib/osh}
13source $LIB_OSH/bash-strict.sh
14source $LIB_OSH/task-five.sh
15
16REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
17
18source build/dev-shell.sh # PYTHONPATH
19source test/common.sh # html-head
20source test/spec-common.sh
21source web/table/html.sh
22
23shopt -s failglob # to debug TSV expansion failure below
24
25OSH_PY=$REPO_ROOT/bin/osh
26YSH_PY=$REPO_ROOT/bin/ysh
27
28# Run with ASAN binary by default. Release overrides this
29OSH_CC=${OSH_CC:-$REPO_ROOT/_bin/cxx-asan/osh}
30YSH_CC=${YSH_CC:-$REPO_ROOT/_bin/cxx-asan/ysh}
31
32# Same variable in test/spec-runner.sh
33NUM_SPEC_TASKS=${NUM_SPEC_TASKS:-400}
34
35# So we can pass ASAN. Note that test/spec-common.sh has to pass this to
36# sh_spec.py.
37export OILS_GC_ON_EXIT=1
38
39#
40# For translation
41#
42
43run-file() {
44 local spec_name=$1
45 shift
46
47 local spec_file=spec/$spec_name.test.sh
48
49 local suite
50 suite=$(test/sh_spec.py --print-spec-suite $spec_file)
51
52 local spec_subdir
53 case $suite in
54 osh) spec_subdir='osh-cpp' ;;
55 ysh) spec_subdir='ysh-cpp' ;;
56 disabled) spec_subdir='disabled-cpp' ;;
57 *) die "Invalid suite $suite" ;;
58 esac
59
60 local base_dir=_tmp/spec/$spec_subdir
61 mkdir -v -p $base_dir
62
63 # Compare Python and C++ shells by passing --oils-cpp-bin-dir
64
65 local variant='cxx-asan'
66 # TODO: turning on gcalways will find more bugs
67 #local variant='cxx-asan+gcalways'
68
69 sh-spec $spec_file \
70 --timeout 10 \
71 --oils-bin-dir $PWD/bin \
72 --oils-cpp-bin-dir $REPO_ROOT/_bin/$variant \
73 --tsv-output $base_dir/${spec_name}.tsv \
74 "$@"
75}
76
77osh-all() {
78 # Like test/spec.sh {osh,ysh}-all, but it compares against different binaries
79
80 # For debugging hangs
81 #export MAX_PROCS=1
82
83 ninja _bin/cxx-asan/{osh,ysh}
84
85 test/spec-runner.sh shell-sanity-check $OSH_PY $OSH_CC
86
87 local spec_subdir=osh-cpp
88
89 local status
90 set +o errexit
91 # $suite $compare_mode
92 test/spec-runner.sh all-parallel \
93 osh compare-cpp $spec_subdir "$@"
94 status=$?
95 set -o errexit
96
97 # Write comparison even if we failed
98 write-compare-html $spec_subdir
99
100 return $status
101}
102
103ysh-all() {
104 ninja _bin/cxx-asan/{osh,ysh}
105
106 local spec_subdir=ysh-cpp
107
108 # $suite $compare_mode
109 test/spec-runner.sh all-parallel \
110 ysh compare-cpp $spec_subdir "$@"
111
112 write-compare-html $spec_subdir
113}
114
115console-row() {
116 ### Print out a histogram of results
117
118 awk '
119FNR == 1 {
120 #print FILENAME > "/dev/stderr"
121}
122FNR != 1 {
123 case_num = $1
124 sh = $2
125 result = $3
126
127 if (sh == "osh") {
128 osh[result] += 1
129 } else if (sh == "osh_cpp") { # bin/osh_cpp
130 oe_py[result] += 1
131 } else if (sh == "osh_ALT") { # _bin/*/osh
132 oe_cpp[result] += 1
133 }
134}
135
136function print_hist(sh, hist) {
137 printf("%s\t", sh)
138
139 k = "pass"
140 printf("%s %4d\t", k, hist[k])
141 k = "FAIL"
142 printf("%s %4d\t", k, hist[k])
143
144 print ""
145
146 # This prints N-I, ok, bug, etc.
147 #for (k in hist) {
148 # printf("%s %s\t", k, hist[k])
149 #}
150
151}
152
153END {
154 print_hist("osh", osh)
155 print_hist("osh_cpp", oe_py)
156 print_hist("osh_ALT", oe_cpp)
157}
158 ' "$@"
159}
160
161console-summary() {
162 ### Report on our progress translating
163
164 local spec_subdir=$1
165
166 # Can't go at the top level because files won't exist!
167 readonly TSV=(_tmp/spec/$spec_subdir/*.tsv)
168
169 wc -l "${TSV[@]}"
170
171 for file in "${TSV[@]}"; do
172 echo
173 echo "$file"
174 console-row $file
175 done
176
177 echo
178 echo "TOTAL"
179 console-row "${TSV[@]}"
180}
181
182#
183# HTML
184#
185
186summary-csv-row() {
187 ### Print one row or the last total row
188
189 local spec_subdir=$1
190 shift
191
192 if test $# -eq 1; then
193 local spec_name=$1
194 local -a tsv_files=( _tmp/spec/$spec_subdir/$spec_name.tsv )
195 else
196 local spec_name='TOTAL'
197 local -a tsv_files=( "$@" )
198 fi
199
200 awk -v spec_name=$spec_name '
201# skip the first row
202FNR != 1 {
203 case_num = $1
204 sh = $2
205 result = $3
206
207 if (sh == "osh" || sh == "ysh") {
208 osh[result] += 1
209 } else if (sh == "osh-cpp" || sh == "ysh-cpp") { # bin/osh
210 osh_native[result] += 1
211 }
212}
213
214END {
215 num_py = osh["pass"]
216 num_cpp = osh_native["pass"]
217 if (spec_name == "TOTAL") {
218 href = ""
219 } else {
220 href = sprintf("%s.html", spec_name)
221 }
222
223 if (num_py == num_cpp) {
224 row_css_class = "cpp-good" # green
225 }
226
227 printf("%s,%s,%s,%d,%d,%d\n",
228 row_css_class,
229 spec_name, href,
230 num_py,
231 num_cpp,
232 num_py - num_cpp)
233}
234' "${tsv_files[@]}"
235}
236
237summary-csv() {
238 local spec_subdir=$1
239
240 local sh_label
241 local manifest
242
243 case $spec_subdir in
244 osh-cpp)
245 sh_label=osh
246 manifest=_tmp/spec/SUITE-osh.txt
247 ;;
248 ysh-cpp)
249 sh_label=ysh
250 manifest=_tmp/spec/SUITE-ysh.txt
251 ;;
252 *)
253 die "Invalid dir $spec_subdir"
254 ;;
255 esac
256
257 # Can't go at the top level because files might not exist!
258 cat <<EOF
259ROW_CSS_CLASS,name,name_HREF,${sh_label}_py,${sh_label}_cpp,delta
260EOF
261
262 # total row rows goes at the TOP, so it's in <thead> and not sorted.
263 summary-csv-row $spec_subdir _tmp/spec/$spec_subdir/*.tsv
264
265 head -n $NUM_SPEC_TASKS $manifest | sort |
266 while read spec_name; do
267 summary-csv-row $spec_subdir $spec_name
268 done
269}
270
271html-summary-header() {
272 local prefix=../../..
273 html-head --title 'Passing Spec Tests in C++' \
274 $prefix/web/ajax.js \
275 $prefix/web/table/table-sort.js $prefix/web/table/table-sort.css \
276 $prefix/web/base.css \
277 $prefix/web/spec-cpp.css
278
279 table-sort-begin "width50"
280
281 cat <<EOF
282<p id="home-link">
283 <!-- The release index is two dirs up -->
284 <a href="../..">Up</a> |
285 <a href="/">oils.pub</a>
286</p>
287
288<h1>Python vs C++</h1>
289
290<p>These numbers measure the progress of the C++ translation.
291Compare with <a href=".">index.html</a>.
292</p>
293
294EOF
295}
296
297html-summary-footer() {
298 cat <<EOF
299<p>Generated by <code>test/spec-cpp.sh</code>.</p>
300EOF
301 table-sort-end 'summary' # The table name
302}
303
304# TODO: Use here-schema-tsv in test/tsv-lib.sh
305here-schema() {
306 ### Read a legible text format on stdin, and write CSV on stdout
307
308 # This is a little like: https://wiki.xxiivv.com/site/tablatal.html
309 # TODO: generalize this in stdlib/here.sh
310 while read one two; do
311 echo "$one,$two"
312 done
313}
314
315write-compare-html() {
316 local spec_subdir=$1
317
318 local sh_label
319 case $spec_subdir in
320 osh-cpp)
321 sh_label=osh
322 ;;
323 ysh-cpp)
324 sh_label=ysh
325 ;;
326 *)
327 die "Invalid dir $spec_subdir"
328 ;;
329 esac
330
331 local dir=_tmp/spec/$spec_subdir
332 local out=$dir/compare.html
333
334 summary-csv $spec_subdir >$dir/summary.csv
335
336 # The underscores are stripped when we don't want them to be!
337 # Note: we could also put "pretty_heading" in the schema
338
339 here-schema >$dir/summary.schema.csv <<EOF
340column_name type
341ROW_CSS_CLASS string
342name string
343name_HREF string
344${sh_label}_py integer
345${sh_label}_cpp integer
346delta integer
347EOF
348
349 { html-summary-header
350 # total row isn't sorted
351 web/table/csv2html.py --thead-offset 1 $dir/summary.csv
352 html-summary-footer
353 } > $out
354
355 log "Comparison: file://$REPO_ROOT/$out"
356}
357
358tsv-demo() {
359 sh-spec spec/arith.test.sh --tsv-output _tmp/arith.tsv dash bash "$@"
360 cat _tmp/arith.tsv
361}
362
363repro() {
364 test/spec.sh alias -r 0 -p > _tmp/a
365 ninja _bin/clang-dbg/osh
366 _bin/clang-dbg/osh _tmp/a
367}
368
369repro-all() {
370 OSH_CC=$REPO_ROOT/_bin/clang-dbg/osh $0 all
371}
372
373task-five "$@"