1 | #!/usr/bin/env bash
|
2 | #
|
3 | # Test the C++ translation of Oils.
|
4 | #
|
5 | # Usage:
|
6 | # test/spec-cpp.sh <function name>
|
7 | #
|
8 | # Examples:
|
9 | # test/spec-cpp.sh run-file smoke -r 0 -v
|
10 | # NUM_SPEC_TASKS=2 test/spec-cpp.sh osh-all
|
11 |
|
12 | : ${LIB_OSH=stdlib/osh}
|
13 | source $LIB_OSH/bash-strict.sh
|
14 | source $LIB_OSH/task-five.sh
|
15 |
|
16 | REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
|
17 |
|
18 | source build/dev-shell.sh # PYTHONPATH
|
19 | source test/common.sh # html-head
|
20 | source test/spec-common.sh
|
21 | source web/table/html.sh
|
22 |
|
23 | shopt -s failglob # to debug TSV expansion failure below
|
24 |
|
25 | OSH_PY=$REPO_ROOT/bin/osh
|
26 | YSH_PY=$REPO_ROOT/bin/ysh
|
27 |
|
28 | # Run with ASAN binary by default. Release overrides this
|
29 | OSH_CC=${OSH_CC:-$REPO_ROOT/_bin/cxx-asan/osh}
|
30 | YSH_CC=${YSH_CC:-$REPO_ROOT/_bin/cxx-asan/ysh}
|
31 | OSH_SOUFFLE_CC=${OSH_CC:-$REPO_ROOT/_bin/cxx-asan/osh-souffle}
|
32 | YSH_SOUFFLE_CC=${YSH_CC:-$REPO_ROOT/_bin/cxx-asan/ysh-souffle}
|
33 |
|
34 | # Same variable in test/spec-runner.sh
|
35 | NUM_SPEC_TASKS=${NUM_SPEC_TASKS:-400}
|
36 |
|
37 | # So we can pass ASAN. Note that test/spec-common.sh has to pass this to
|
38 | # sh_spec.py.
|
39 | export OILS_GC_ON_EXIT=1
|
40 |
|
41 | #
|
42 | # For translation
|
43 | #
|
44 |
|
45 | run-file() {
|
46 | local spec_name=$1
|
47 | shift
|
48 |
|
49 | local spec_file=spec/$spec_name.test.sh
|
50 |
|
51 | local suite
|
52 | suite=$(test/sh_spec.py --print-spec-suite $spec_file)
|
53 |
|
54 | local spec_subdir
|
55 | case $suite in
|
56 | osh) spec_subdir='osh-cpp' ;;
|
57 | ysh) spec_subdir='ysh-cpp' ;;
|
58 | *) die "Invalid suite $suite" ;;
|
59 | esac
|
60 |
|
61 | local base_dir=_tmp/spec/$spec_subdir
|
62 | mkdir -v -p $base_dir
|
63 |
|
64 | # Compare Python and C++ shells by passing --oils-cpp-bin-dir
|
65 | sh-spec $spec_file \
|
66 | --timeout 10 \
|
67 | --oils-bin-dir $PWD/bin \
|
68 | --oils-cpp-bin-dir $REPO_ROOT/_bin/cxx-asan \
|
69 | --tsv-output $base_dir/${spec_name}.tsv \
|
70 | "$@"
|
71 | }
|
72 |
|
73 | osh-all() {
|
74 | # Like test/spec.sh {osh,ysh}-all, but it compares against different binaries
|
75 |
|
76 | # For debugging hangs
|
77 | #export MAX_PROCS=1
|
78 |
|
79 | ninja _bin/cxx-asan/{osh,ysh,osh-souffle,ysh-souffle}
|
80 |
|
81 | test/spec-runner.sh shell-sanity-check $OSH_PY $OSH_CC $OSH_SOUFFLE_CC
|
82 |
|
83 | local spec_subdir=osh-cpp
|
84 |
|
85 | # $suite $compare_mode
|
86 | test/spec-runner.sh all-parallel \
|
87 | osh compare-cpp $spec_subdir "$@" || true # OK if it fails
|
88 |
|
89 | write-compare-html $spec_subdir
|
90 | }
|
91 |
|
92 | ysh-all() {
|
93 | ninja _bin/cxx-asan/{osh,ysh}
|
94 |
|
95 | local spec_subdir=ysh-cpp
|
96 |
|
97 | # $suite $compare_mode
|
98 | test/spec-runner.sh all-parallel \
|
99 | ysh compare-cpp $spec_subdir "$@" || true # OK if it fails
|
100 |
|
101 | write-compare-html $spec_subdir
|
102 | }
|
103 |
|
104 | console-row() {
|
105 | ### Print out a histogram of results
|
106 |
|
107 | awk '
|
108 | FNR == 1 {
|
109 | #print FILENAME > "/dev/stderr"
|
110 | }
|
111 | FNR != 1 {
|
112 | case_num = $1
|
113 | sh = $2
|
114 | result = $3
|
115 |
|
116 | if (sh == "osh") {
|
117 | osh[result] += 1
|
118 | } else if (sh == "osh_cpp") { # bin/osh_cpp
|
119 | oe_py[result] += 1
|
120 | } else if (sh == "osh_ALT") { # _bin/*/osh
|
121 | oe_cpp[result] += 1
|
122 | }
|
123 | }
|
124 |
|
125 | function print_hist(sh, hist) {
|
126 | printf("%s\t", sh)
|
127 |
|
128 | k = "pass"
|
129 | printf("%s %4d\t", k, hist[k])
|
130 | k = "FAIL"
|
131 | printf("%s %4d\t", k, hist[k])
|
132 |
|
133 | print ""
|
134 |
|
135 | # This prints N-I, ok, bug, etc.
|
136 | #for (k in hist) {
|
137 | # printf("%s %s\t", k, hist[k])
|
138 | #}
|
139 |
|
140 | }
|
141 |
|
142 | END {
|
143 | print_hist("osh", osh)
|
144 | print_hist("osh_cpp", oe_py)
|
145 | print_hist("osh_ALT", oe_cpp)
|
146 | }
|
147 | ' "$@"
|
148 | }
|
149 |
|
150 | console-summary() {
|
151 | ### Report on our progress translating
|
152 |
|
153 | local spec_subdir=$1
|
154 |
|
155 | # Can't go at the top level because files won't exist!
|
156 | readonly TSV=(_tmp/spec/$spec_subdir/*.tsv)
|
157 |
|
158 | wc -l "${TSV[@]}"
|
159 |
|
160 | for file in "${TSV[@]}"; do
|
161 | echo
|
162 | echo "$file"
|
163 | console-row $file
|
164 | done
|
165 |
|
166 | echo
|
167 | echo "TOTAL"
|
168 | console-row "${TSV[@]}"
|
169 | }
|
170 |
|
171 | #
|
172 | # HTML
|
173 | #
|
174 |
|
175 | summary-csv-row() {
|
176 | ### Print one row or the last total row
|
177 |
|
178 | local spec_subdir=$1
|
179 | shift
|
180 |
|
181 | if test $# -eq 1; then
|
182 | local spec_name=$1
|
183 | local -a tsv_files=( _tmp/spec/$spec_subdir/$spec_name.tsv )
|
184 | else
|
185 | local spec_name='TOTAL'
|
186 | local -a tsv_files=( "$@" )
|
187 | fi
|
188 |
|
189 | awk -v spec_name=$spec_name '
|
190 | # skip the first row
|
191 | FNR != 1 {
|
192 | case_num = $1
|
193 | sh = $2
|
194 | result = $3
|
195 |
|
196 | if (sh == "osh" || sh == "ysh") {
|
197 | osh[result] += 1
|
198 | } else if (sh == "osh-cpp" || sh == "ysh-cpp") { # bin/osh
|
199 | osh_native[result] += 1
|
200 | }
|
201 | }
|
202 |
|
203 | END {
|
204 | num_py = osh["pass"]
|
205 | num_cpp = osh_native["pass"]
|
206 | if (spec_name == "TOTAL") {
|
207 | href = ""
|
208 | } else {
|
209 | href = sprintf("%s.html", spec_name)
|
210 | }
|
211 |
|
212 | if (num_py == num_cpp) {
|
213 | row_css_class = "cpp-good" # green
|
214 | }
|
215 |
|
216 | printf("%s,%s,%s,%d,%d,%d\n",
|
217 | row_css_class,
|
218 | spec_name, href,
|
219 | num_py,
|
220 | num_cpp,
|
221 | num_py - num_cpp)
|
222 | }
|
223 | ' "${tsv_files[@]}"
|
224 | }
|
225 |
|
226 | summary-csv() {
|
227 | local spec_subdir=$1
|
228 |
|
229 | local sh_label
|
230 | local manifest
|
231 |
|
232 | case $spec_subdir in
|
233 | osh-cpp)
|
234 | sh_label=osh
|
235 | manifest=_tmp/spec/SUITE-osh.txt
|
236 | ;;
|
237 | ysh-cpp)
|
238 | sh_label=ysh
|
239 | manifest=_tmp/spec/SUITE-ysh.txt
|
240 | ;;
|
241 | *)
|
242 | die "Invalid dir $spec_subdir"
|
243 | ;;
|
244 | esac
|
245 |
|
246 | # Can't go at the top level because files might not exist!
|
247 | cat <<EOF
|
248 | ROW_CSS_CLASS,name,name_HREF,${sh_label}_py,${sh_label}_cpp,delta
|
249 | EOF
|
250 |
|
251 | # total row rows goes at the TOP, so it's in <thead> and not sorted.
|
252 | summary-csv-row $spec_subdir _tmp/spec/$spec_subdir/*.tsv
|
253 |
|
254 | head -n $NUM_SPEC_TASKS $manifest | sort |
|
255 | while read spec_name; do
|
256 | summary-csv-row $spec_subdir $spec_name
|
257 | done
|
258 | }
|
259 |
|
260 | html-summary-header() {
|
261 | local prefix=../../..
|
262 | html-head --title 'Passing Spec Tests in C++' \
|
263 | $prefix/web/ajax.js \
|
264 | $prefix/web/table/table-sort.js $prefix/web/table/table-sort.css \
|
265 | $prefix/web/base.css \
|
266 | $prefix/web/spec-cpp.css
|
267 |
|
268 | table-sort-begin "width50"
|
269 |
|
270 | cat <<EOF
|
271 | <p id="home-link">
|
272 | <!-- The release index is two dirs up -->
|
273 | <a href="../..">Up</a> |
|
274 | <a href="/">oilshell.org</a>
|
275 | </p>
|
276 |
|
277 | <h1>Python vs C++</h1>
|
278 |
|
279 | <p>These numbers measure the progress of the C++ translation.
|
280 | Compare with <a href=".">index.html</a>.
|
281 | </p>
|
282 |
|
283 | EOF
|
284 | }
|
285 |
|
286 | html-summary-footer() {
|
287 | cat <<EOF
|
288 | <p>Generated by <code>test/spec-cpp.sh</code>.</p>
|
289 | EOF
|
290 | table-sort-end 'summary' # The table name
|
291 | }
|
292 |
|
293 | # TODO: Use here-schema-tsv in test/tsv-lib.sh
|
294 | here-schema() {
|
295 | ### Read a legible text format on stdin, and write CSV on stdout
|
296 |
|
297 | # This is a little like: https://wiki.xxiivv.com/site/tablatal.html
|
298 | # TODO: generalize this in stdlib/here.sh
|
299 | while read one two; do
|
300 | echo "$one,$two"
|
301 | done
|
302 | }
|
303 |
|
304 | write-compare-html() {
|
305 | local spec_subdir=$1
|
306 |
|
307 | local sh_label
|
308 | case $spec_subdir in
|
309 | osh-cpp)
|
310 | sh_label=osh
|
311 | ;;
|
312 | ysh-cpp)
|
313 | sh_label=ysh
|
314 | ;;
|
315 | *)
|
316 | die "Invalid dir $spec_subdir"
|
317 | ;;
|
318 | esac
|
319 |
|
320 | local dir=_tmp/spec/$spec_subdir
|
321 | local out=$dir/compare.html
|
322 |
|
323 | summary-csv $spec_subdir >$dir/summary.csv
|
324 |
|
325 | # The underscores are stripped when we don't want them to be!
|
326 | # Note: we could also put "pretty_heading" in the schema
|
327 |
|
328 | here-schema >$dir/summary.schema.csv <<EOF
|
329 | column_name type
|
330 | ROW_CSS_CLASS string
|
331 | name string
|
332 | name_HREF string
|
333 | ${sh_label}_py integer
|
334 | ${sh_label}_cpp integer
|
335 | delta integer
|
336 | EOF
|
337 |
|
338 | { html-summary-header
|
339 | # total row isn't sorted
|
340 | web/table/csv2html.py --thead-offset 1 $dir/summary.csv
|
341 | html-summary-footer
|
342 | } > $out
|
343 |
|
344 | log "Comparison: file://$REPO_ROOT/$out"
|
345 | }
|
346 |
|
347 | tsv-demo() {
|
348 | sh-spec spec/arith.test.sh --tsv-output _tmp/arith.tsv dash bash "$@"
|
349 | cat _tmp/arith.tsv
|
350 | }
|
351 |
|
352 | repro() {
|
353 | test/spec.sh alias -r 0 -p > _tmp/a
|
354 | ninja _bin/clang-dbg/osh
|
355 | _bin/clang-dbg/osh _tmp/a
|
356 | }
|
357 |
|
358 | repro-all() {
|
359 | OSH_CC=$REPO_ROOT/_bin/clang-dbg/osh $0 all
|
360 | }
|
361 |
|
362 | task-five "$@"
|