OILS / test / spec-cpp.sh View on Github | oilshell.org

362 lines, 160 significant
1#!/usr/bin/env bash
2#
3# Test the C++ translation of Oils.
4#
5# Usage:
6# test/spec-cpp.sh <function name>
7#
8# Examples:
9# test/spec-cpp.sh run-file smoke -r 0 -v
10# NUM_SPEC_TASKS=2 test/spec-cpp.sh osh-all
11
12: ${LIB_OSH=stdlib/osh}
13source $LIB_OSH/bash-strict.sh
14source $LIB_OSH/task-five.sh
15
16REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
17
18source build/dev-shell.sh # PYTHONPATH
19source test/common.sh # html-head
20source test/spec-common.sh
21source web/table/html.sh
22
23shopt -s failglob # to debug TSV expansion failure below
24
25OSH_PY=$REPO_ROOT/bin/osh
26YSH_PY=$REPO_ROOT/bin/ysh
27
28# Run with ASAN binary by default. Release overrides this
29OSH_CC=${OSH_CC:-$REPO_ROOT/_bin/cxx-asan/osh}
30YSH_CC=${YSH_CC:-$REPO_ROOT/_bin/cxx-asan/ysh}
31OSH_SOUFFLE_CC=${OSH_CC:-$REPO_ROOT/_bin/cxx-asan/osh-souffle}
32YSH_SOUFFLE_CC=${YSH_CC:-$REPO_ROOT/_bin/cxx-asan/ysh-souffle}
33
34# Same variable in test/spec-runner.sh
35NUM_SPEC_TASKS=${NUM_SPEC_TASKS:-400}
36
37# So we can pass ASAN. Note that test/spec-common.sh has to pass this to
38# sh_spec.py.
39export OILS_GC_ON_EXIT=1
40
41#
42# For translation
43#
44
45run-file() {
46 local spec_name=$1
47 shift
48
49 local spec_file=spec/$spec_name.test.sh
50
51 local suite
52 suite=$(test/sh_spec.py --print-spec-suite $spec_file)
53
54 local spec_subdir
55 case $suite in
56 osh) spec_subdir='osh-cpp' ;;
57 ysh) spec_subdir='ysh-cpp' ;;
58 *) die "Invalid suite $suite" ;;
59 esac
60
61 local base_dir=_tmp/spec/$spec_subdir
62 mkdir -v -p $base_dir
63
64 # Compare Python and C++ shells by passing --oils-cpp-bin-dir
65 sh-spec $spec_file \
66 --timeout 10 \
67 --oils-bin-dir $PWD/bin \
68 --oils-cpp-bin-dir $REPO_ROOT/_bin/cxx-asan \
69 --tsv-output $base_dir/${spec_name}.tsv \
70 "$@"
71}
72
73osh-all() {
74 # Like test/spec.sh {osh,ysh}-all, but it compares against different binaries
75
76 # For debugging hangs
77 #export MAX_PROCS=1
78
79 ninja _bin/cxx-asan/{osh,ysh,osh-souffle,ysh-souffle}
80
81 test/spec-runner.sh shell-sanity-check $OSH_PY $OSH_CC $OSH_SOUFFLE_CC
82
83 local spec_subdir=osh-cpp
84
85 # $suite $compare_mode
86 test/spec-runner.sh all-parallel \
87 osh compare-cpp $spec_subdir "$@" || true # OK if it fails
88
89 write-compare-html $spec_subdir
90}
91
92ysh-all() {
93 ninja _bin/cxx-asan/{osh,ysh}
94
95 local spec_subdir=ysh-cpp
96
97 # $suite $compare_mode
98 test/spec-runner.sh all-parallel \
99 ysh compare-cpp $spec_subdir "$@" || true # OK if it fails
100
101 write-compare-html $spec_subdir
102}
103
104console-row() {
105 ### Print out a histogram of results
106
107 awk '
108FNR == 1 {
109 #print FILENAME > "/dev/stderr"
110}
111FNR != 1 {
112 case_num = $1
113 sh = $2
114 result = $3
115
116 if (sh == "osh") {
117 osh[result] += 1
118 } else if (sh == "osh_cpp") { # bin/osh_cpp
119 oe_py[result] += 1
120 } else if (sh == "osh_ALT") { # _bin/*/osh
121 oe_cpp[result] += 1
122 }
123}
124
125function print_hist(sh, hist) {
126 printf("%s\t", sh)
127
128 k = "pass"
129 printf("%s %4d\t", k, hist[k])
130 k = "FAIL"
131 printf("%s %4d\t", k, hist[k])
132
133 print ""
134
135 # This prints N-I, ok, bug, etc.
136 #for (k in hist) {
137 # printf("%s %s\t", k, hist[k])
138 #}
139
140}
141
142END {
143 print_hist("osh", osh)
144 print_hist("osh_cpp", oe_py)
145 print_hist("osh_ALT", oe_cpp)
146}
147 ' "$@"
148}
149
150console-summary() {
151 ### Report on our progress translating
152
153 local spec_subdir=$1
154
155 # Can't go at the top level because files won't exist!
156 readonly TSV=(_tmp/spec/$spec_subdir/*.tsv)
157
158 wc -l "${TSV[@]}"
159
160 for file in "${TSV[@]}"; do
161 echo
162 echo "$file"
163 console-row $file
164 done
165
166 echo
167 echo "TOTAL"
168 console-row "${TSV[@]}"
169}
170
171#
172# HTML
173#
174
175summary-csv-row() {
176 ### Print one row or the last total row
177
178 local spec_subdir=$1
179 shift
180
181 if test $# -eq 1; then
182 local spec_name=$1
183 local -a tsv_files=( _tmp/spec/$spec_subdir/$spec_name.tsv )
184 else
185 local spec_name='TOTAL'
186 local -a tsv_files=( "$@" )
187 fi
188
189 awk -v spec_name=$spec_name '
190# skip the first row
191FNR != 1 {
192 case_num = $1
193 sh = $2
194 result = $3
195
196 if (sh == "osh" || sh == "ysh") {
197 osh[result] += 1
198 } else if (sh == "osh-cpp" || sh == "ysh-cpp") { # bin/osh
199 osh_native[result] += 1
200 }
201}
202
203END {
204 num_py = osh["pass"]
205 num_cpp = osh_native["pass"]
206 if (spec_name == "TOTAL") {
207 href = ""
208 } else {
209 href = sprintf("%s.html", spec_name)
210 }
211
212 if (num_py == num_cpp) {
213 row_css_class = "cpp-good" # green
214 }
215
216 printf("%s,%s,%s,%d,%d,%d\n",
217 row_css_class,
218 spec_name, href,
219 num_py,
220 num_cpp,
221 num_py - num_cpp)
222}
223' "${tsv_files[@]}"
224}
225
226summary-csv() {
227 local spec_subdir=$1
228
229 local sh_label
230 local manifest
231
232 case $spec_subdir in
233 osh-cpp)
234 sh_label=osh
235 manifest=_tmp/spec/SUITE-osh.txt
236 ;;
237 ysh-cpp)
238 sh_label=ysh
239 manifest=_tmp/spec/SUITE-ysh.txt
240 ;;
241 *)
242 die "Invalid dir $spec_subdir"
243 ;;
244 esac
245
246 # Can't go at the top level because files might not exist!
247 cat <<EOF
248ROW_CSS_CLASS,name,name_HREF,${sh_label}_py,${sh_label}_cpp,delta
249EOF
250
251 # total row rows goes at the TOP, so it's in <thead> and not sorted.
252 summary-csv-row $spec_subdir _tmp/spec/$spec_subdir/*.tsv
253
254 head -n $NUM_SPEC_TASKS $manifest | sort |
255 while read spec_name; do
256 summary-csv-row $spec_subdir $spec_name
257 done
258}
259
260html-summary-header() {
261 local prefix=../../..
262 html-head --title 'Passing Spec Tests in C++' \
263 $prefix/web/ajax.js \
264 $prefix/web/table/table-sort.js $prefix/web/table/table-sort.css \
265 $prefix/web/base.css \
266 $prefix/web/spec-cpp.css
267
268 table-sort-begin "width50"
269
270 cat <<EOF
271<p id="home-link">
272 <!-- The release index is two dirs up -->
273 <a href="../..">Up</a> |
274 <a href="/">oilshell.org</a>
275</p>
276
277<h1>Python vs C++</h1>
278
279<p>These numbers measure the progress of the C++ translation.
280Compare with <a href=".">index.html</a>.
281</p>
282
283EOF
284}
285
286html-summary-footer() {
287 cat <<EOF
288<p>Generated by <code>test/spec-cpp.sh</code>.</p>
289EOF
290 table-sort-end 'summary' # The table name
291}
292
293# TODO: Use here-schema-tsv in test/tsv-lib.sh
294here-schema() {
295 ### Read a legible text format on stdin, and write CSV on stdout
296
297 # This is a little like: https://wiki.xxiivv.com/site/tablatal.html
298 # TODO: generalize this in stdlib/here.sh
299 while read one two; do
300 echo "$one,$two"
301 done
302}
303
304write-compare-html() {
305 local spec_subdir=$1
306
307 local sh_label
308 case $spec_subdir in
309 osh-cpp)
310 sh_label=osh
311 ;;
312 ysh-cpp)
313 sh_label=ysh
314 ;;
315 *)
316 die "Invalid dir $spec_subdir"
317 ;;
318 esac
319
320 local dir=_tmp/spec/$spec_subdir
321 local out=$dir/compare.html
322
323 summary-csv $spec_subdir >$dir/summary.csv
324
325 # The underscores are stripped when we don't want them to be!
326 # Note: we could also put "pretty_heading" in the schema
327
328 here-schema >$dir/summary.schema.csv <<EOF
329column_name type
330ROW_CSS_CLASS string
331name string
332name_HREF string
333${sh_label}_py integer
334${sh_label}_cpp integer
335delta integer
336EOF
337
338 { html-summary-header
339 # total row isn't sorted
340 web/table/csv2html.py --thead-offset 1 $dir/summary.csv
341 html-summary-footer
342 } > $out
343
344 log "Comparison: file://$REPO_ROOT/$out"
345}
346
347tsv-demo() {
348 sh-spec spec/arith.test.sh --tsv-output _tmp/arith.tsv dash bash "$@"
349 cat _tmp/arith.tsv
350}
351
352repro() {
353 test/spec.sh alias -r 0 -p > _tmp/a
354 ninja _bin/clang-dbg/osh
355 _bin/clang-dbg/osh _tmp/a
356}
357
358repro-all() {
359 OSH_CC=$REPO_ROOT/_bin/clang-dbg/osh $0 all
360}
361
362task-five "$@"