OILS / mycpp / TEST.sh View on Github | oilshell.org

361 lines, 233 significant
1#!/usr/bin/env bash
2#
3# Run tests in this directory.
4#
5# Usage:
6# mycpp/TEST.sh <function name>
7
8set -o nounset
9set -o pipefail
10set -o errexit
11
12REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
13source build/common.sh
14source build/ninja-rules-cpp.sh
15source devtools/common.sh
16source test/common.sh # run-test-bin, can-compile-32-bit
17
18# in case binaries weren't built
19shopt -s failglob
20
21# Will be needed to pass ASAN leak detector? Or only do this for the main binary?
22# export OILS_GC_ON_EXIT=1
23
24examples-variant() {
25 ### Run all examples using a variant -- STATS only
26
27 local compiler=${1:-cxx}
28 local variant=${2:-asan+gcalways}
29 local do_benchmark=${3:-}
30
31 banner "$0 examples-variant $compiler $variant"
32
33 ninja mycpp-examples-$compiler-$variant
34
35 local num_tests=0
36 local num_failed=0
37 local status=0
38
39 local log_dir=_test/$compiler-$variant/mycpp/examples
40 mkdir -p $log_dir
41
42 for b in _bin/$compiler-$variant/mycpp/examples/*; do
43 case $b in
44 (*.stripped) # just run the unstripped binary
45 continue
46 ;;
47 esac
48
49 local prefix="$log_dir/$(basename $b)"
50
51 case $variant in
52 (coverage)
53 export LLVM_PROFILE_FILE=$prefix.profraw
54 ;;
55 esac
56
57 local log="${prefix}${do_benchmark}.log"
58
59 log "RUN $b > $log"
60
61 local test_name=$(basename $b)
62 if test -n "$do_benchmark" && [[ $test_name == test_* ]]; then
63 log "Skipping $test_name in benchmark mode"
64 continue
65 fi
66
67 set +o errexit
68 BENCHMARK="$do_benchmark" $b >$log 2>&1
69 status=$?
70 set -o errexit
71
72 if test "$status" -eq 0; then
73 log 'OK'
74 else
75 log "FAIL with status $?"
76 log ''
77 #return $status
78 num_failed=$((num_failed + 1))
79 fi
80
81 num_tests=$((num_tests + 1))
82 done
83
84 log ''
85 log "$num_failed of $num_tests tests failed"
86 log ''
87
88 if test $num_failed -ne 0; then
89 echo "FAIL: Expected no failures, got $num_failed"
90 return 1
91 fi
92
93 return 0
94}
95
96#
97# 3 Variants x {test, benchmark}
98#
99
100ex-gcalways() {
101 local compiler=${1:-}
102 examples-variant "$compiler" asan+gcalways
103}
104
105# TOO SLOW to run. It's garbage collecting all the time.
106ex-gcalways-bench() {
107 local compiler=${1:-}
108 examples-variant "$compiler" asan+gcalways '.BENCHMARK'
109}
110
111ex-asan() {
112 local compiler=${1:-}
113 examples-variant "$compiler" asan
114}
115
116# 2 of 18 tests failed: cartesian, parse
117# So it does not catch the 10 segfaults that 'asan+gcalways' catches with a few
118# iterations!
119ex-asan-bench() {
120 local compiler=${1:-}
121 examples-variant "$compiler" asan '.BENCHMARK'
122}
123
124# PASS! Under both clang and GCC.
125ex-ubsan() {
126 local compiler=${1:-}
127 examples-variant "$compiler" ubsan
128}
129
130# same as ASAN: 2 of 18
131ex-ubsan-bench() {
132 local compiler=${1:-}
133 examples-variant "$compiler" ubsan '.BENCHMARK'
134}
135
136# PASS!
137ex-opt() {
138 local compiler=${1:-}
139 examples-variant "$compiler" opt
140}
141
142# 2 of 18 tests failed
143ex-opt-bench() {
144 local compiler=${1:-}
145 examples-variant "$compiler" opt '.BENCHMARK'
146}
147
148#
149# Unit Tests
150#
151
152run-unit-tests() {
153
154 local compiler=${1:-cxx}
155 local variant=${2:-asan+gcalways}
156
157 log ''
158 log "$0 run-unit-tests $compiler $variant"
159 log ''
160
161 ninja mycpp-unit-$compiler-$variant
162
163 local -a binaries=(_bin/$compiler-$variant/mycpp/*)
164
165 # Add these files if they exist in the variant
166 if test -d _bin/$compiler-$variant/mycpp/demo; then
167 binaries+=(_bin/$compiler-$variant/mycpp/demo/*)
168 fi
169
170 for b in "${binaries[@]}"; do
171 if ! test -f $b; then
172 continue
173 fi
174
175 local prefix=${b//_bin/_test/}
176 local log=$prefix.log
177 mkdir -p $(dirname $log)
178
179 local asan_options=''
180 case $b in
181 # leaks with malloc
182 (*/demo/hash_table|*/demo/target_lang|*/demo/gc_header|*/small_str_test)
183 asan_options='detect_leaks=0'
184 ;;
185 esac
186
187 ASAN_OPTIONS="$asan_options" run-test-bin $b
188
189 done
190}
191
192#
193# Test failures
194#
195
196translate-example() {
197 local ex=$1
198
199 local mycpp=_bin/shwrap/mycpp_main
200 $mycpp '.:pyext' _tmp/mycpp-invalid $ex
201}
202
203test-invalid-examples() {
204 local mycpp=_bin/shwrap/mycpp_main
205 ninja $mycpp
206 for ex in mycpp/examples/invalid_*; do
207
208 banner "$ex"
209
210 set +o errexit
211 translate-example $ex
212 local status=$?
213 set -o errexit
214
215 local expected_status=1
216
217 case $ex in
218 */invalid_condition.py)
219 expected_status=8
220 ;;
221 */invalid_default_args.py)
222 expected_status=4
223 ;;
224 */invalid_try_else.py)
225 expected_status=3
226 ;;
227 */invalid_except.py)
228 expected_status=3
229 ;;
230 */invalid_global.py)
231 expected_status=2
232 ;;
233 */invalid_python.py)
234 expected_status=5
235 ;;
236 */invalid_switch.py)
237 expected_status=5
238 ;;
239 esac
240
241 if test $status -ne $expected_status; then
242 die "mycpp $ex: expected status $expected_status, got $status"
243 fi
244
245 done
246}
247
248test-control-flow-graph() {
249 local mycpp=_bin/shwrap/mycpp_main
250 ninja $mycpp
251 for ex in mycpp/examples/*.py; do
252 local data_dir=testdata/control-flow-graph/$(basename -s .py $ex)
253 if ! test -d $data_dir; then
254 continue
255 fi
256 banner "$ex"
257
258 translate-example $ex
259 for fact_path in $data_dir/*.facts; do
260 local fact_file=$(basename $fact_path)
261 diff -u $data_dir/$fact_file _tmp/mycpp-facts/$fact_file
262 done
263 done
264}
265
266# TODO: Run with Clang UBSAN in CI as well
267readonly UBSAN_COMPILER=cxx
268
269unit() {
270 ### Run by test/cpp-unit.sh
271
272 # Run other unit tests, e.g. the GC tests
273
274 if can-compile-32-bit; then
275 run-unit-tests '' asan32+gcalways # ASAN on 32-bit
276 else
277 log ''
278 log "*** Can't compile 32-bit binaries (gcc-multilib g++-multilib needed on Debian)"
279 log ''
280 fi
281
282 # Run other tests with all variants
283
284 run-unit-tests $UBSAN_COMPILER ubsan
285
286 run-unit-tests '' asan
287 run-unit-tests '' asan+gcalways
288 run-unit-tests '' opt
289 run-unit-tests '' asan+bigint
290
291 bump-leak-heap-test
292}
293
294bump-leak-heap-test() {
295 for config in cxx-asan+bumpleak $UBSAN_COMPILER-ubsan+bumpleak; do
296 local bin=_bin/$config/mycpp/bump_leak_heap_test
297 ninja $bin
298 run-test-bin $bin
299 done
300}
301
302#
303# Translator
304#
305
306test-translator() {
307 ### Invoked by soil/worker.sh
308
309 examples-variant '' asan
310
311 # Test with more collections
312 examples-variant '' asan+gcalways
313
314 run-test-func test-invalid-examples _test/mycpp/test-invalid-examples.log
315
316 run-test-func test-control-flow-graph _test/mycpp/test-cfg-examples.log
317
318 # Runs tests in cxx-asan variant, and benchmarks in cxx-opt variant
319 if ! ninja mycpp-logs-equal; then
320 log 'FAIL mycpp-logs-equal'
321 return 1
322 fi
323}
324
325soil-run() {
326 set +o errexit
327 $0 test-translator
328 local status=$?
329 set -o errexit
330
331 return $status
332}
333
334unit-test-coverage() {
335 ### Invoked by Soil
336
337 local bin=_bin/clang-coverage+bumpleak/mycpp/bump_leak_heap_test
338 ninja $bin
339 run-test-bin $bin
340
341 run-unit-tests clang coverage
342
343 local out_dir=_test/clang-coverage/mycpp
344 test/coverage.sh html-report $out_dir \
345 clang-coverage/mycpp clang-coverage+bumpleak/mycpp
346}
347
348examples-coverage() {
349 ### Invoked by Soil
350
351 examples-variant clang coverage
352
353 local out_dir=_test/clang-coverage/mycpp/examples
354 test/coverage.sh html-report $out_dir clang-coverage/mycpp/examples
355}
356
357# Call function $1 with arguments $2 $3 $4
358#
359# mycpp/TEST.sh examples-variant '' asan
360
361"$@"