OILS / test / spec-compat.sh View on Github | oils.pub

351 lines, 176 significant
1#!/usr/bin/env bash
2#
3# Test OSH against any shell
4#
5# Usage:
6# test/spec-compat.sh <function name>
7#
8# Examples:
9# $0 build-static-oils # Build Oils binaries
10# $0 build-brush T # Pull and build
11# $0 build-sush T # Pull and build
12# $0 osh-all # Run spec tests, and make report
13# # Includes binary size report
14
15: ${LIB_OSH=stdlib/osh}
16source $LIB_OSH/bash-strict.sh
17source $LIB_OSH/task-five.sh
18
19REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
20
21source build/dev-shell.sh # put mksh etc. in $PATH
22source test/common.sh
23source test/spec-common.sh
24
25OSH_TARGET=_bin/cxx-asan/osh
26OSH=$PWD/$OSH_TARGET
27
28# Metrics
29# - binary size - stripped
30# - lines of source code - I think we get this from DWARF debug info
31# - https://claude.ai/chat/40597e2e-4d1e-42b4-a756-7a265f01cc5a shows options
32# - llvm-dwarfdump
33# - Python lib https://github.com/eliben/pyelftools/
34# - right now this isn't worth it - spec tests are more important
35# - unsafe functions / methods?
36# - cargo geiger is also hard to parse
37
38readonly TOYBOX_DIR=~/src/toybox-0.8.12
39
40readonly SUSH_DIR=../../shells/rusty_bash
41readonly BRUSH_DIR=../../shells/brush
42
43readonly SUSH=$PWD/$SUSH_DIR/target/release/sush
44readonly BRUSH=$PWD/$BRUSH_DIR/target/release/brush
45
46# these are all roughly ksh compatible
47readonly -a SHELLS=(bash dash ash zsh mksh ksh $TOYBOX_DIR/sh $SUSH $BRUSH $OSH)
48
49download-toybox() {
50 #mkdir -p ~/src
51 wget --directory-prefix ~/src --no-clobber \
52 https://landley.net/toybox/downloads/toybox-0.8.12.tar.gz
53}
54
55build-toybox() {
56 pushd $TOYBOX_DIR
57
58 make toybox
59 # warning: using unfinished code
60 make sh
61
62 popd
63}
64
65update-rust() {
66 . ~/.cargo/env
67 time rustup update
68}
69
70build-brush() {
71 local pull=${1:-}
72
73 pushd ../../shells/brush
74
75 if test -n "$pull"; then
76 git pull
77 fi
78
79 . ~/.cargo/env
80
81 # Test incremental build speed
82 # - debug: 3.8 seconds
83 # - release: 1:06 minutes !
84 # touch brush-core/src/shell.rs
85
86 # 41s
87 time cargo build
88 echo
89
90 # 1m 49s
91 # It builds a stripped binary by default - disable that for metrics
92 RUSTFLAGS='-C strip=none' time cargo build --release
93 echo
94
95 popd
96}
97
98build-sush() {
99 local pull=${1:-}
100
101 pushd ../../shells/rusty_bash
102
103 if test -n "$pull"; then
104 git pull
105 fi
106
107 . ~/.cargo/env
108
109 # Test incremental build speed
110 # - debug: 1 second
111 # - release: 6 seconds
112 #touch src/core.rs
113
114 # 10 seconds
115 time cargo build
116 echo
117
118 # 15 seconds
119 time cargo build --release
120 echo
121
122 popd
123}
124
125build-static-oils() {
126 devtools/release-native.sh make-tar
127
128 # Static binary with glibc
129 build/static-oils.sh
130
131 # Static binary with musl libc
132 test/alpine.sh build-static-musl
133}
134
135binary-sizes() {
136 local oils_dynamic=_bin/cxx-opt/bin/oils_for_unix.mycpp.stripped
137 ninja $oils_dynamic
138
139 pushd $BRUSH_DIR >/dev/null
140 local out=target/release/brush.stripped
141 strip -o $out target/release/brush
142 local brush=$BRUSH_DIR/$out
143 popd >/dev/null
144
145 pushd $SUSH_DIR >/dev/null
146 local out=target/release/sush.stripped
147 strip -o $out target/release/sush
148 local sush=$SUSH_DIR/$out
149 popd >/dev/null
150
151 local -a oils=(
152 $oils_dynamic
153
154 # 2.4 M musl libc
155 _tmp/musl-libc/oils-for-unix-static.stripped
156
157 # 3.4 MB with glibc
158 _bin/cxx-opt-sh/oils-for-unix-static.stripped
159 )
160 local -a other=(
161
162 $brush $sush $TOYBOX_DIR/sh
163 # our bash isn't stripped
164 # $(which bash) $(which dash) $(which mksh) $(which ksh)
165 )
166
167 echo
168 ls -l --si --sort=none "${oils[@]}"
169 echo
170 ls -l --si --sort=none "${other[@]}"
171
172 # Rust binaries aren't dynamically linked to GNU readline, or libstdc++
173 echo
174 ldd "${oils[@]}" || true # fails if not dynamic
175 echo
176 ldd "${other[@]}"
177}
178
179symbols() {
180 pushd ../../shells/brush
181 #file target/release/brush
182
183 echo 'BRUSH'
184 # 6272
185 nm target/release/brush | wc -l
186 popd
187
188 pushd ../../shells/rusty_bash
189 # Not stripped
190 #file target/release/sush
191
192 echo 'SUSH'
193 # 4413
194 nm target/release/sush | wc -l
195 # More symbols
196 # nm target/debug/sush | wc -l
197 popd
198
199 #local osh=_bin/cxx-opt/bin/oils_for_unix.mycpp.stripped
200 local osh=_bin/cxx-opt/bin/oils_for_unix.mycpp
201 local dbg=_bin/cxx-dbg/bin/oils_for_unix.mycpp
202 ninja $osh
203
204 echo 'OSH'
205 # 9857 - lots of string literals?
206 nm $osh | wc -l
207 #nm $osh | less
208
209 #ninja $dbg
210 # 17570
211 #nm $dbg | wc -l
212}
213
214install-geiger() {
215 # https://github.com/geiger-rs/cargo-geiger
216 . ~/.cargo/env
217
218 # 2:34 minutes
219 cargo install --locked cargo-geiger
220}
221
222# This is DESTRUCTIVE for some reason.
223# It cleans the build and rebuilds every time.
224geiger-brush() {
225 . ~/.cargo/env
226
227 pushd ../../shells/brush
228
229 pushd brush-core
230 cargo geiger
231 popd
232
233 pushd brush-interactive
234 cargo geiger
235 popd
236
237 pushd brush-parser
238 cargo geiger
239 popd
240
241 pushd brush-shell
242 cargo geiger
243 popd
244
245 # doesn't work
246 #time cargo geiger --workspace
247 #time cargo geiger --package brush-core --package brush-parser
248
249 #time cargo geiger --workspace
250
251 popd
252}
253
254geiger-sush() {
255 . ~/.cargo/env
256
257 pushd ../../shells/rusty_bash
258
259 # this cleans the build
260 #
261 # Functions Expressions Impls Traits Methods
262 # 181/1056 9377/45040 114/158 30/32 463/2887
263 #
264 # x/y
265 # x = unsafe used by build
266 # y = unsafe in crate
267
268 # ~7 seconds
269 time cargo geiger
270
271 popd
272}
273
274#
275# Spec Tests
276#
277
278run-file() {
279 local spec_name=${1:-smoke}
280 shift # Pass list of shells
281
282 local spec_subdir='compat'
283 local base_dir=_tmp/spec/$spec_subdir
284 mkdir -v -p $base_dir
285
286 # spec/tilde hangs under toysh - need timeout
287 sh-spec spec/$spec_name.test.sh \
288 --tsv-output $base_dir/${spec_name}.result.tsv \
289 --timeout 1 \
290 "$@" \
291 "${SHELLS[@]}"
292}
293
294osh-all() {
295 # Since we're publishing these, make sure we start with a clean slate
296 rm -r -f -v _tmp/spec
297
298 ninja $OSH_TARGET
299
300 test/spec-runner.sh shell-sanity-check "${SHELLS[@]}"
301
302 local spec_subdir=compat
303
304 local status
305 set +o errexit
306 # $suite $compare_mode
307 test/spec-runner.sh all-parallel \
308 compat spec-compat $spec_subdir "$@"
309 status=$?
310 set -o errexit
311
312 # Write comparison even if we failed
313 test/spec-compat-html.sh write-compare-html $spec_subdir
314
315 return $status
316}
317
318#
319# Misc
320#
321
322list() {
323 mkdir -p _tmp/spec # _all-parallel also does this
324 test/spec-runner.sh write-suite-manifests
325 wc -l _tmp/spec/SUITE-*
326
327 # TODO:
328 # - Remove zsh test files?
329 # - What about *-bash test cases? These aren't clearly organized
330
331 cat _tmp/spec/SUITE-osh.txt
332}
333
334readonly ERRORS=(
335 'echo )' # parse error
336 'cd -z' # usage error
337 'cd /zzz' # runtime error
338)
339
340survey-errors() {
341 set +o errexit
342 for sh in "${SHELLS[@]}"; do
343 echo
344 echo " === $sh"
345 for code in "${ERRORS[@]}"; do
346 $sh -c "$code"
347 done
348 done
349}
350
351task-five "$@"