OILS / mycpp / TEST.sh View on Github | oilshell.org

455 lines, 302 significant
1#!/usr/bin/env bash
2#
3# Run tests in this directory.
4#
5# Usage:
6# mycpp/TEST.sh <function name>
7
8: ${LIB_OSH=stdlib/osh}
9source $LIB_OSH/bash-strict.sh
10source $LIB_OSH/task-five.sh
11
12REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
13
14source build/common.sh
15source build/ninja-rules-cpp.sh
16source devtools/common.sh
17source test/common.sh # run-test-bin, can-compile-32-bit
18
19# in case binaries weren't built
20shopt -s failglob
21
22# Will be needed to pass ASAN leak detector? Or only do this for the main binary?
23# export OILS_GC_ON_EXIT=1
24
25examples-variant() {
26 ### Run all examples using a variant -- STATS only
27
28 local compiler=${1:-cxx}
29 local variant=${2:-asan+gcalways}
30 local do_benchmark=${3:-}
31
32 banner "$0 examples-variant $compiler $variant"
33
34 ninja mycpp-examples-$compiler-$variant
35
36 local num_tests=0
37 local num_failed=0
38 local status=0
39
40 local log_dir=_test/$compiler-$variant/mycpp/examples
41 mkdir -p $log_dir
42
43 for b in _bin/$compiler-$variant/mycpp/examples/*; do
44 case $b in
45 *.pea) # for now, don't run pea_hello.pea, it fails on purpose
46 continue
47 ;;
48 *.stripped) # just run the unstripped binary
49 continue
50 ;;
51 esac
52
53 local prefix="$log_dir/$(basename $b)"
54
55 case $variant in
56 coverage)
57 export LLVM_PROFILE_FILE=$prefix.profraw
58 ;;
59 esac
60
61 local log="${prefix}${do_benchmark}.log"
62
63 log "RUN $b > $log"
64
65 local test_name=$(basename $b)
66 if test -n "$do_benchmark" && [[ $test_name == test_* ]]; then
67 log "Skipping $test_name in benchmark mode"
68 continue
69 fi
70
71 set +o errexit
72 BENCHMARK="$do_benchmark" $b >$log 2>&1
73 status=$?
74 set -o errexit
75
76 if test "$status" -eq 0; then
77 log 'OK'
78 else
79 log "FAIL with status $?"
80 log ''
81 #return $status
82 num_failed=$((num_failed + 1))
83 fi
84
85 num_tests=$((num_tests + 1))
86 done
87
88 log ''
89 log "$num_failed of $num_tests tests failed"
90 log ''
91
92 if test $num_failed -ne 0; then
93 echo "FAIL: Expected no failures, got $num_failed"
94 return 1
95 fi
96
97 return 0
98}
99
100#
101# 3 Variants x {test, benchmark}
102#
103
104ex-gcalways() {
105 local compiler=${1:-}
106 examples-variant "$compiler" asan+gcalways
107}
108
109# TOO SLOW to run. It's garbage collecting all the time.
110ex-gcalways-bench() {
111 local compiler=${1:-}
112 examples-variant "$compiler" asan+gcalways '.BENCHMARK'
113}
114
115ex-asan() {
116 local compiler=${1:-}
117 examples-variant "$compiler" asan
118}
119
120# 2 of 18 tests failed: cartesian, parse
121# So it does not catch the 10 segfaults that 'asan+gcalways' catches with a few
122# iterations!
123ex-asan-bench() {
124 local compiler=${1:-}
125 examples-variant "$compiler" asan '.BENCHMARK'
126}
127
128# PASS! Under both clang and GCC.
129ex-ubsan() {
130 local compiler=${1:-}
131 examples-variant "$compiler" ubsan
132}
133
134# same as ASAN: 2 of 18
135ex-ubsan-bench() {
136 local compiler=${1:-}
137 examples-variant "$compiler" ubsan '.BENCHMARK'
138}
139
140# PASS!
141ex-opt() {
142 local compiler=${1:-}
143 examples-variant "$compiler" opt
144}
145
146# 2 of 18 tests failed
147ex-opt-bench() {
148 local compiler=${1:-}
149 examples-variant "$compiler" opt '.BENCHMARK'
150}
151
152#
153# Unit Tests
154#
155
156run-unit-tests() {
157
158 local compiler=${1:-cxx}
159 local variant=${2:-asan+gcalways}
160
161 log ''
162 log "$0 run-unit-tests $compiler $variant"
163 log ''
164
165 ninja mycpp-unit-$compiler-$variant
166
167 local -a binaries=(_bin/$compiler-$variant/mycpp/*)
168
169 # Add these files if they exist in the variant
170 if test -d _bin/$compiler-$variant/mycpp/demo; then
171 binaries+=(_bin/$compiler-$variant/mycpp/demo/*)
172 fi
173
174 for b in "${binaries[@]}"; do
175 if ! test -f $b; then
176 continue
177 fi
178
179 local prefix=${b//_bin/_test/}
180 local log=$prefix.log
181 mkdir -p $(dirname $log)
182
183 local asan_options=''
184 case $b in
185 # leaks with malloc
186 */demo/hash_table|*/demo/target_lang|*/demo/gc_header|*/small_str_test)
187 asan_options='detect_leaks=0'
188 ;;
189 esac
190
191 ASAN_OPTIONS="$asan_options" run-test-bin $b
192
193 done
194}
195
196#
197# Test failures
198#
199
200translate-example() {
201 local ex=$1
202
203 local mycpp=_bin/shwrap/mycpp_main
204 $mycpp '.:pyext' _tmp/mycpp-invalid $ex
205}
206
207test-invalid-examples() {
208 local mycpp=_bin/shwrap/mycpp_main
209 ninja $mycpp
210 for ex in mycpp/examples/invalid_*; do
211
212 banner "$ex"
213
214 set +o errexit
215 translate-example $ex
216 local status=$?
217 set -o errexit
218
219 local expected_status=1
220
221 case $ex in
222 */invalid_condition.py)
223 expected_status=8
224 ;;
225 */invalid_other.py)
226 expected_status=6
227 ;;
228 */invalid_default_args.py)
229 expected_status=5
230 ;;
231 */invalid_try_else.py)
232 expected_status=3
233 ;;
234 */invalid_except.py)
235 expected_status=4
236 ;;
237 */invalid_global.py)
238 expected_status=2
239 ;;
240 */invalid_python.py)
241 expected_status=5
242 ;;
243 */invalid_switch.py)
244 expected_status=5
245 ;;
246 esac
247
248 if test $status -ne $expected_status; then
249 die "mycpp $ex: expected status $expected_status, got $status"
250 fi
251
252 done
253}
254
255test-control-flow-graph() {
256 local mycpp=_bin/shwrap/mycpp_main
257 ninja $mycpp
258 for ex in mycpp/examples/*.py; do
259 local data_dir=testdata/control-flow-graph/$(basename -s .py $ex)
260 if ! test -d $data_dir; then
261 continue
262 fi
263 banner "$ex"
264
265 translate-example $ex
266 for fact_path in $data_dir/*.facts; do
267 local fact_file=$(basename $fact_path)
268 diff -u $data_dir/$fact_file _tmp/mycpp-facts/$fact_file
269 done
270 done
271}
272
273# TODO: Run with Clang UBSAN in CI as well
274readonly UBSAN_COMPILER=cxx
275
276unit() {
277 ### Run by test/cpp-unit.sh
278
279 # Run other unit tests, e.g. the GC tests
280
281 if can-compile-32-bit; then
282 run-unit-tests '' asan32+gcalways # ASAN on 32-bit
283 else
284 log ''
285 log "*** Can't compile 32-bit binaries (gcc-multilib g++-multilib needed on Debian)"
286 log ''
287 fi
288
289 # Run other tests with all variants
290
291 run-unit-tests $UBSAN_COMPILER ubsan
292
293 run-unit-tests '' asan
294 run-unit-tests '' asan+gcalways
295 run-unit-tests '' opt
296 run-unit-tests '' asan+bigint
297
298 bump-leak-heap-test
299}
300
301bump-leak-heap-test() {
302 for config in cxx-asan+bumpleak $UBSAN_COMPILER-ubsan+bumpleak; do
303 local bin=_bin/$config/mycpp/bump_leak_heap_test
304 ninja $bin
305 run-test-bin $bin
306 done
307}
308
309#
310# Translator
311#
312
313test-translator() {
314 ### Invoked by soil/worker.sh
315
316 examples-variant '' asan
317
318 # Test with more collections
319 examples-variant '' asan+gcalways
320
321 run-test-func test-invalid-examples _test/mycpp/test-invalid-examples.log
322
323 run-test-func test-control-flow-graph _test/mycpp/test-cfg-examples.log
324
325 # Runs tests in cxx-asan variant, and benchmarks in cxx-opt variant
326 if ! ninja mycpp-logs-equal; then
327 log 'FAIL mycpp-logs-equal'
328 return 1
329 fi
330}
331
332soil-run() {
333 set +o errexit
334 $0 test-translator
335 local status=$?
336 set -o errexit
337
338 return $status
339}
340
341unit-test-coverage() {
342 ### Invoked by Soil
343
344 local bin=_bin/clang-coverage+bumpleak/mycpp/bump_leak_heap_test
345 ninja $bin
346 run-test-bin $bin
347
348 run-unit-tests clang coverage
349
350 local out_dir=_test/clang-coverage/mycpp
351 test/coverage.sh html-report $out_dir \
352 clang-coverage/mycpp clang-coverage+bumpleak/mycpp
353}
354
355examples-coverage() {
356 ### Invoked by Soil
357
358 examples-variant clang coverage
359
360 local out_dir=_test/clang-coverage/mycpp/examples
361 test/coverage.sh html-report $out_dir clang-coverage/mycpp/examples
362}
363
364mycpp-check() {
365 if true; then
366 #python3 -m mypy --strict --follow-imports=silent "$@"
367 python3 -m mypy --strict --no-strict-optional --follow-imports=silent "$@"
368 else
369 # Disable --strict until more passes - 308 -> 181 errors
370 # That means that types are optional
371 # Also add --no-strict-optional
372 python3 -m mypy --no-strict-optional --follow-imports=silent "$@"
373 fi
374}
375
376check-types() {
377 local p=".:$MYPY_WEDGE:$PY3_LIBS_WEDGE"
378
379 #local -a files=( mycpp/*.py )
380 local -a files=(
381 mycpp/{pass_state,util,crash,format_strings,visitor,const_pass,control_flow_pass,mycpp_main,cppgen_pass,conversion_pass}.py
382 )
383
384 # the path is fiddly
385 PYTHONPATH=$p MYPYPATH=$MYPY_WEDGE \
386 mycpp-check "${files[@]}"
387}
388
389files() {
390 wc -l mycpp/*.py | sort -n
391}
392
393copy-golden() {
394 local dir=testdata/mycpp
395 mkdir -p $dir
396 cp -v _gen/bin/oils_for_unix.mycpp.cc $dir
397}
398
399compare-golden() {
400 local -a files=(
401 testdata/mycpp/oils_for_unix.mycpp.cc _gen/bin/oils_for_unix.mycpp.cc
402 )
403
404 wc -l "${files[@]}"
405 echo
406
407 if diff -u "${files[@]}"; then
408 echo EQUAL
409 else
410 echo 'NOT EQUAL'
411 fi
412}
413
414compare-souffle() {
415 # Show less rooting in examples
416 ninja _bin/cxx-asan/mycpp/examples/test_iterators.mycpp{,-souffle}
417
418 local -a files=(
419 _gen/mycpp/examples/test_iterators.mycpp{,-souffle}.cc
420 )
421 if diff -u "${files[@]}"; then
422 die 'Should not be equal'
423 fi
424
425 ninja _bin/cxx-asan/mycpp-souffle/osh
426 local -a files=(
427 _gen/bin/oils_for_unix.mycpp{,-souffle}.cc
428 )
429 if diff -u "${files[@]}"; then
430 die 'Should not be equal'
431 fi
432}
433
434const-pass() {
435 python3 mycpp/const_pass.py "$@"
436}
437
438str-hash-demo() {
439 local file=benchmarks/testdata/configure-coreutils
440
441 # We have ~1600 strings - let's say it doubles
442 #
443 # 1613 unique strings -> 34 collisions of length 2, 1 of length 3
444 # 2618 unique strings -> 108 collisions of length 2, 6 of length 3
445 #
446 # So yes 3 is good
447
448 for n in 180 1800 3600 18000; do
449 echo "=== Number of strings $n ==="
450 head -n $n $file | const-pass
451 echo
452 done
453}
454
455task-five "$@"