OILS / test / spec-compat-html.sh View on Github | oils.pub

418 lines, 101 significant
1#!/usr/bin/env bash
2#
3# Create reports, published at https://pages.oils.pub/
4#
5# Usage:
6# test/spec-compat-html.sh <function name>
7#
8# Examples:
9# $0 deploy 2025-11-02 # Deploys result of test/spec-compat.sh osh-all to ../pages/
10#
11# TODO:
12# - improve pages.oils.pub/ index.html
13# - Add epoch or Build timestamp on page - that's at the top
14# - Improve
15# - summary/percentages in TOP.html?
16# - More shells: might as well include ash, dash, ysh
17# - Make a container with `buildah`, so others can collaborate?
18# - Refactor
19# - maybe clean up test/spec-runner.sh arguments
20
21: ${LIB_OSH=stdlib/osh}
22source $LIB_OSH/bash-strict.sh
23source $LIB_OSH/task-five.sh
24
25REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
26
27source benchmarks/common.sh # cmark function
28source test/common.sh
29source test/spec-common.sh
30source test/tsv-lib.sh # tsv-row
31source web/table/html.sh # table-sort-begin
32
33# Matches SHELLS in test/spec-compat.sh
34readonly -a SH_LABELS=(bash dash ash zsh mksh ksh toysh sush brush osh)
35
36summary-tsv-row() {
37 ### Print one row or the last total row
38
39 local report=$1
40 local spec_subdir=$2
41 shift 2
42
43 if test $# -eq 1; then
44 local spec_name=$1
45 local -a tsv_files=( _tmp/spec/$spec_subdir/$spec_name.result.tsv )
46 else
47 local spec_name='TOTAL'
48 local -a tsv_files=( "$@" )
49 fi
50
51 awk -v report=$report -v spec_name=$spec_name '
52# skip the first row
53FNR != 1 {
54 case_num = $1
55 sh = $2
56 result = $3
57
58 if (sh == "bash") {
59 bash[result] += 1
60 } else if (sh == "dash") {
61 dash[result] += 1
62 } else if (sh == "ash") {
63 ash[result] += 1
64 } else if (sh == "zsh") {
65 zsh[result] += 1
66 } else if (sh == "mksh") {
67 mksh[result] += 1
68 } else if (sh == "ksh") {
69 ksh[result] += 1
70 } else if (sh == "toysh") {
71 toysh[result] += 1
72 } else if (sh == "sush") {
73 sush[result] += 1
74 } else if (sh == "brush") {
75 brush[result] += 1
76 } else if (sh == "osh") {
77 osh[result] += 1
78 }
79}
80
81END {
82 if (spec_name == "TOTAL") {
83 href = ""
84 } else {
85 href = sprintf("%s.html", spec_name)
86 }
87
88 if (report == "PASSING") {
89 bash_total = ("pass" in bash) ? bash["pass"] : 0
90 dash_total = ("pass" in dash) ? dash["pass"] : 0
91 ash_total = ("pass" in ash) ? ash["pass"] : 0
92 zsh_total = ("pass" in zsh) ? zsh["pass"] : 0
93 mksh_total = ("pass" in mksh) ? mksh["pass"] : 0
94 ksh_total = ("pass" in ksh) ? ksh["pass"] : 0
95 toysh_total = ("pass" in toysh) ? toysh["pass"] : 0
96 sush_total = ("pass" in sush) ? sush["pass"] : 0
97 brush_total = ("pass" in brush) ? brush["pass"] : 0
98 osh_total = ("pass" in osh) ? osh["pass"] : 0
99
100 } else if (report == "DELTA-osh") {
101 bash_total = bash["pass"] - osh["pass"]
102 dash_total = dash["pass"] - osh["pass"]
103 ash_total = ash["pass"] - osh["pass"]
104 zsh_total = zsh["pass"] - osh["pass"]
105 mksh_total = mksh["pass"] - osh["pass"]
106 ksh_total = ksh["pass"] - osh["pass"]
107 toysh_total = toysh["pass"] - osh["pass"]
108 sush_total = sush["pass"] - osh["pass"]
109 brush_total = brush["pass"] - osh["pass"]
110 osh_total = osh["pass"] - osh["pass"]
111
112 } else if (report == "DELTA-bash") {
113 bash_total = bash["pass"] - bash["pass"]
114 dash_total = dash["pass"] - bash["pass"]
115 ash_total = ash["pass"] - bash["pass"]
116 zsh_total = zsh["pass"] - bash["pass"]
117 mksh_total = mksh["pass"] - bash["pass"]
118 ksh_total = ksh["pass"] - bash["pass"]
119 toysh_total = toysh["pass"] - bash["pass"]
120 sush_total = sush["pass"] - bash["pass"]
121 brush_total = brush["pass"] - bash["pass"]
122 osh_total = osh["pass"] - bash["pass"]
123 }
124
125 # TODO: change this color
126 row_css_class = "cpp-good" # green
127
128 row = sprintf("%s %s %s %d %d %d %d %d %d %d %d %d %d",
129 row_css_class,
130 spec_name, href,
131 bash_total,
132 dash_total,
133 ash_total,
134 zsh_total,
135 mksh_total,
136 ksh_total,
137 toysh_total,
138 sush_total,
139 brush_total,
140 osh_total)
141
142 # Turn tabs into spaces - awk mutates the row!
143 gsub(/ /, "\t", row)
144 print row
145}
146' "${tsv_files[@]}"
147}
148
149summary-tsv() {
150 local report=$1
151 local spec_subdir=$2
152
153 local manifest=_tmp/spec/SUITE-compat.txt
154
155 # Can't go at the top level because files might not exist!
156 tsv-row \
157 'ROW_CSS_CLASS' 'name' 'name_HREF' "${SH_LABELS[@]}"
158
159 # total row rows goes at the TOP, so it's in <thead> and not sorted.
160 summary-tsv-row $report $spec_subdir _tmp/spec/$spec_subdir/*.result.tsv
161
162 head -n $NUM_SPEC_TASKS $manifest | sort |
163 while read spec_name; do
164 summary-tsv-row $report $spec_subdir $spec_name
165 done
166}
167
168html-summary-header() {
169 local report=$1
170
171 local prefix=../../..
172 spec-html-head $prefix "$report - Shell Compatibility "
173
174 table-sort-begin "width50"
175
176 cat <<EOF
177<p id="home-link">
178 <a href="/">Root</a> |
179 <a href="https://oils.pub/">oils.pub</a>
180</p>
181
182<h1>$report - Shell Compatibility</h1>
183
184<p>Back to <a href="TOP.html">TOP.html</a>
185</p>
186EOF
187}
188
189html-summary-footer() {
190 local report=$1
191
192 echo "
193<p>Generated by <code>test/spec-compat.sh</code>.
194</p>
195
196<p><a href="$report.tsv">Raw TSV</a>
197</p>
198"
199 table-sort-end "$report" # The table name
200}
201
202write-summary-html() {
203 local report=$1
204 local spec_subdir=$2
205
206 local dir=_tmp/spec/$spec_subdir
207 local out=$dir/$report.html
208
209 summary-tsv $report $spec_subdir >$dir/$report.tsv
210
211 # The underscores are stripped when we don't want them to be!
212 # Note: we could also put "pretty_heading" in the schema
213
214 here-schema-tsv >$dir/$report.schema.tsv <<EOF
215column_name type
216ROW_CSS_CLASS string
217name string
218name_HREF string
219bash integer
220dash integer
221ash integer
222zsh integer
223mksh integer
224ksh integer
225toysh integer
226sush integer
227brush integer
228osh integer
229EOF
230
231 { html-summary-header "$report"
232 # total row isn't sorted
233 tsv2html --thead-offset 1 $dir/$report.tsv
234 html-summary-footer "$report"
235 } > $out
236
237 log "Comparison: file://$REPO_ROOT/$out"
238}
239
240top-html() {
241 local base_url='../../../web'
242 html-head --title 'Shell Compatibility Reports' \
243 "$base_url/base.css"
244
245 echo '
246 <body class="width35">
247 <style>
248 code { color: green; }
249 </style>
250
251 <p id="home-link">
252 <a href="/">Root</a> |
253 <a href="https://oils.pub/">oils.pub</a>
254 </p>
255 '
256
257 cmark <<'EOF'
258## Shell Compatibility Reports
259
260These reports are based on [spec tests written for Oils][spec-tests].
261
262Here are some summary tables. **Click** on the column headers to sort:
263
264- [Total Passing](PASSING.html)
265 - Each shell gets 1 point for each case we marked passing.
266 - Our assertions are usually based on a **survey** of `bash`, `dash`, `mksh`,
267 `zsh`, and other shells. Assertions from 2016-2018 may favor OSH, but
268 there shouldn't be many of them.
269- [Delta bash](DELTA-bash.html)
270 - Compare each shell's passing count vs. bash
271- [Delta OSH](DELTA-osh.html)
272 - Compare each shell's passing count vs. OSH
273
274[tree.html](tree.html)
275
276### Notes and Caveats
277
278- Some tests may fail for innocuous reasons, e.g. printing `'$'` versus `\$`
279 - Shell authors are welcome to use our test suite, and add assertions.
280- OSH has some features that bash doesn't have.
281 - e.g. I removed `spec/strict-options` so shells aren't penalized for not
282 having these features.
283 - But I left `spec/errexit-osh` in because I think new shells should provide
284 alternatives to the **bugs** in POSIX:
285 [YSH Fixes Shell's Error Handling (`errexit`)](https://oils.pub/release/latest/doc/error-handling.html)
286 - I also think shells should adopt [Simple Word Evaluation in Unix Shell](https://oils.pub/release/latest/doc/simple-word-eval.html) (i.e. deprecate `$IFS`, more so than `zsh`)
287- Other ideas
288 - We could add a "majority agreement" metric, for a more neutral report.
289 - We could add the Smoosh test suite. Results are published on our [quality
290page](https://oils.pub/release/latest/quality.html).
291
292### Shells Compared
293
294- GNU `bash`
295 - <https://www.gnu.org/software/bash/>
296 - running fixed version built for Oils
297- `mksh` - shell on Android, derivative of `pdksh`
298 - <http://www.mirbsd.org/mksh.htm>
299 - running fixed version built for Oils
300- AT&T `ksh`
301 - <https://github.com/ksh93/ksh>
302 - running distro package
303- `toysh`
304 - <https://landley.net/toybox/>
305 - running tarball release
306- `sush`
307 - <https://github.com/shellgei/rusty_bash>
308 - running git HEAD
309- `brush`
310 - <https://github.com/reubeno/brush>
311 - running git HEAD
312- `osh`
313 - <https://github.com/oils-for-unix/oils>
314 - running git HEAD
315
316TODO: Add other shells, and be more specific about versions.
317
318### More Comparisons
319
320- [Binary Sizes](../../binary-sizes.txt)
321
322Possibly TODO
323
324- Build times
325- Lines of code?
326 - [Oils has a "compressed" implementation](https://www.oilshell.org/blog/2024/09/line-counts.html)
327- Memory safety
328- Oils has many other:
329 - test suites - `test/gold`, `test/wild`, ...
330 - benchmarks - parse time, runtime, ...
331
332### Links
333
334- Wiki:
335 - [Spec Tests][spec-tests]
336 - [Contributing](https://github.com/oils-for-unix/oils/wiki/Contributing)
337- Zulip: <https://oilshell.zulipchat.com/>
338 - Feel free to send feedback, and ask questions!
339
340[spec-tests]: https://github.com/oils-for-unix/oils/wiki/Spec-Tests
341
342### Features Not Yet Implemented in OSH
343
344We know about these gaps:
345
346- `kill` builtin, `let` keyword - we do have some spec tests for them
347- `coproc` keyword
348
349EOF
350
351 echo '
352 </body>
353</html>
354'
355
356# Notes on big files:
357# - spec/strict-options, errexit-osh - could be in the YSH suite
358# - but then that messes up our historical metrics
359# - or we create a new 'spec-compat' suite?
360# - spec/globignore - a big one for OSH
361
362}
363
364write-compare-html() {
365 local spec_subdir=${1:-'compat'}
366 local dir=_tmp/spec/$spec_subdir
367
368 local out=$dir/TOP.html
369 top-html >$out
370 log "Top-level index: file://$REPO_ROOT/$out"
371
372 if test -n "${QUICKLY:-}"; then
373 return
374 fi
375
376 write-summary-html PASSING "$spec_subdir"
377 write-summary-html DELTA-osh "$spec_subdir"
378 write-summary-html DELTA-bash "$spec_subdir"
379
380 echo
381 test/spec-compat.sh binary-sizes | tee _tmp/binary-sizes.txt
382}
383
384write-tree-html() {
385 local dir=${1:-_tmp/spec/compat}
386 tree -H './' -T 'Files in spec-compat Report' --charset=ascii $dir \
387 > $dir/tree.html
388}
389
390# TODO: Publish this script
391multi() { ~/git/tree-tools/bin/multi "$@"; }
392
393deploy() {
394 local epoch=${1:-2025-11-02}
395
396 local dest=$PWD/../pages/spec-compat/$epoch
397
398 local web_dir=$dest/web
399 #rm -r -f $web_dir
400
401 #mkdir -p $web_dir
402
403 find web/ -name '*.js' -o -name '*.css' | multi cp $dest
404
405 pushd _tmp
406 find spec/compat -name '*.html' -o -name '*.tsv' | multi cp $dest/renamed-tmp
407 cp -v binary-sizes.txt $dest/renamed-tmp/
408 popd
409
410 write-tree-html $dest/renamed-tmp/spec/compat
411
412 # Work around Jekyll rule for Github pages
413 #mv -v $dest/_tmp $dest/renamed-tmp
414
415 tree $dest/
416}
417
418task-five "$@"