OILS / benchmarks / gc.sh View on Github | oilshell.org

737 lines, 410 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# benchmarks/gc.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
11
12source benchmarks/common.sh # benchmark-html-head
13source benchmarks/cachegrind.sh # with-cachegrind
14source build/dev-shell.sh # R_LIBS_USER
15source test/tsv-lib.sh
16
17readonly BASE_DIR=_tmp/gc
18
19# duplicated in benchmarks/gc-cachegrind.sh
20readonly BASE_DIR_CACHEGRIND=_tmp/gc-cachegrind
21
22# See benchmarks/gperftools.sh. I think the Ubuntu package is very old
23
24download-tcmalloc() {
25 # TODO: move this to ../oil_DEPS ?
26 wget --directory _deps \
27 https://github.com/gperftools/gperftools/releases/download/gperftools-2.10/gperftools-2.10.tar.gz
28
29 # Then ./configure; make; sudo make install
30 # installs in /usr/local/lib
31
32 # Note: there's a warning about libunwind -- maybe install that first. Does
33 # it only apply to CPU profiles?
34}
35
36debug-tcmalloc() {
37 touch mycpp/marksweep_heap.cc
38
39 # No evidence of difference
40 for bin in _bin/cxx-{opt,opt+tcmalloc}/osh; do
41 echo $bin
42 ninja $bin
43
44 ldd $bin
45 echo
46
47 ls -l $bin
48 echo
49
50 # Check what we're linking against
51 nm $bin | egrep -i 'malloc|calloc'
52 #wc -l
53 echo
54 done
55}
56
57install-m32() {
58 # needed to compile with -m32
59 sudo apt-get install gcc-multilib g++-multilib
60}
61
62max-rss() {
63 # %e is real time
64 /usr/bin/time --format '%e %M' -- "$@"
65}
66
67compare-m32() {
68 for bin in _bin/cxx-opt{,32}/osh; do
69 echo $bin
70 ninja $bin
71
72 ldd $bin
73 echo
74
75 file $bin
76 echo
77
78 ls -l $bin
79 echo
80
81 # 141136 KiB vs. 110924 KiB. Significant savings, but it's slower.
82 max-rss $bin --ast-format none -n benchmarks/testdata/configure-coreutils
83
84 done
85}
86
87banner() {
88 echo -----
89 echo "$@"
90}
91
92print-tasks() {
93 local -a workloads=(
94 parse.configure-coreutils
95 parse.configure-cpython
96 parse.abuild
97 ex.bashcomp-parse-help # only runs with bash
98 ex.abuild-print-help # bash / dash / zsh
99 ex.compute-fib # bash / dash / zsh
100 )
101
102 local -a shells=(
103 "bash$TAB-"
104 "dash$TAB-"
105 "zsh$TAB-"
106
107 "_bin/cxx-opt+bumpleak/osh${TAB}mut"
108 "_bin/cxx-opt+bumproot/osh${TAB}mut"
109
110 "_bin/cxx-opt+bumpsmall/osh${TAB}mut+alloc"
111 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc"
112 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc+free+gc"
113
114 # these have trivial GC stats
115 "_bin/cxx-opt/osh${TAB}mut+alloc"
116 "_bin/cxx-opt/osh${TAB}mut+alloc+free"
117 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc"
118 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free"
119 # good GC stats
120 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc"
121 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc+exit"
122 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free+gc"
123 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free+gc+exit"
124 )
125
126 if test -n "${TCMALLOC:-}"; then
127 shells+=(
128 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc"
129 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc+free"
130 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc+free+gc"
131 )
132 fi
133
134 local id=0
135
136 for workload in "${workloads[@]}"; do
137 for shell in "${shells[@]}"; do
138 local row_part="$workload${TAB}$shell"
139
140 # Skip these rows
141 case $row_part in
142 "ex.bashcomp-parse-help${TAB}dash"*)
143 continue
144 ;;
145 "ex.bashcomp-parse-help${TAB}zsh"*)
146 continue
147 ;;
148 esac
149
150 local join_id="gc-$id"
151 local row="$join_id${TAB}$row_part"
152 echo "$row"
153
154 id=$((id + 1))
155
156 done
157
158 # Run a quick 10 tasks
159 if test -n "${QUICKLY:-}" && test $id -gt 10; then
160 break
161 fi
162 done
163}
164
165print-cachegrind-tasks() {
166 local -a workloads=(
167 # coreutils is on osh-parser
168 #parse.configure-coreutils
169
170 #parse.configure-cpython
171
172 # Faster tasks, like benchmarks/uftrace, which is instrumented
173 parse.abuild
174 ex.compute-fib
175 )
176
177 local -a shells=(
178 "bash${TAB}-"
179 "_bin/cxx-opt+bumpleak/osh${TAB}mut"
180 "_bin/cxx-opt+bumproot/osh${TAB}mut"
181
182 "_bin/cxx-opt+bumpsmall/osh${TAB}mut+alloc"
183 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc"
184 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc+free+gc"
185
186 "_bin/cxx-opt/osh${TAB}mut+alloc"
187 "_bin/cxx-opt/osh${TAB}mut+alloc+free"
188 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc"
189 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc+exit"
190
191 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc"
192 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free"
193 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free+gc"
194 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free+gc+exit"
195 )
196
197 local id=0
198 for workload in "${workloads[@]}"; do
199 for shell in "${shells[@]}"; do
200 local row_part="$workload${TAB}$shell"
201
202 local join_id="cachegrind-$id"
203 local row="$join_id${TAB}$row_part"
204 echo "$row"
205
206 id=$((id + 1))
207 done
208 done
209 #print-tasks | egrep 'configure-coreutils' | egrep osh
210}
211
212
213readonly BIG_THRESHOLD=$(( 1 * 1000 * 1000 * 1000 )) # 1 B
214
215run-tasks() {
216 local tsv_out=$1
217 local mode=${2:-time}
218
219 while read -r join_id task sh_path shell_runtime_opts; do
220
221 # Parse different files
222 case $task in
223 parse.configure-coreutils)
224 data_file='benchmarks/testdata/configure-coreutils'
225 ;;
226 parse.configure-cpython)
227 data_file='Python-2.7.13/configure'
228 ;;
229 parse.abuild)
230 data_file='benchmarks/testdata/abuild'
231 ;;
232 esac
233
234 # Construct argv for each task
235 local -a argv
236 case $task in
237 parse.*)
238 argv=( -n $data_file )
239
240 case $sh_path in
241 _bin/*/osh)
242 argv=( --ast-format none "${argv[@]}" )
243 ;;
244 esac
245 ;;
246
247 ex.bashcomp-parse-help)
248 argv=( benchmarks/parse-help/pure-excerpt.sh parse_help_file
249 benchmarks/parse-help/clang.txt )
250 ;;
251
252 ex.abuild-print-help)
253 argv=( testdata/osh-runtime/abuild -h )
254 ;;
255
256 ex.compute-fib)
257 # fewer iterations when instrumented
258 local iters
259 if test $mode = time; then
260 iters=100
261 else
262 iters=10
263 fi
264
265 argv=( benchmarks/compute/fib.sh $iters 44 )
266 ;;
267
268 *)
269 die "Invalid task $task"
270 ;;
271 esac
272
273 echo $join_id $task $sh_path $shell_runtime_opts
274
275 argv=( $sh_path "${argv[@]}" )
276 #echo + "${argv[@]}"
277 #set -x
278
279 if test $mode = cachegrind; then
280 # Add prefix
281 argv=( $0 with-cachegrind $BASE_DIR_CACHEGRIND/raw/$join_id.txt "${argv[@]}" )
282 fi
283
284 # Wrap in a command that writes one row of a TSV
285 # Note: for cachegrind, we need the join ID, but the --rusage is meaningless
286 local -a instrumented=(
287 time-tsv -o $tsv_out --append
288 --rusage
289 --field "$join_id" --field "$task" --field "$sh_path"
290 --field "$shell_runtime_opts"
291 -- "${argv[@]}"
292 )
293
294 # Run with the right environment variables
295
296 case $shell_runtime_opts in
297 -)
298 "${instrumented[@]}" > /dev/null
299 ;;
300 mut)
301 OILS_GC_STATS=1 \
302 "${instrumented[@]}" > /dev/null
303 ;;
304 mut+alloc)
305 # disable GC with big threshold
306 OILS_GC_STATS=1 OILS_GC_THRESHOLD=$BIG_THRESHOLD \
307 "${instrumented[@]}" > /dev/null
308 ;;
309 mut+alloc+free)
310 # do a single GC on exit
311 OILS_GC_STATS=1 OILS_GC_THRESHOLD=$BIG_THRESHOLD OILS_GC_ON_EXIT=1 \
312 "${instrumented[@]}" > /dev/null
313 ;;
314 mut+alloc+free+gc)
315 # Default configuration
316 #
317 # Save the GC stats here. None of the other runtime options are that
318 # interesting.
319
320 if test $mode = 'time' && test $sh_path != _bin/cxx-opt+nopool/osh; then
321 OILS_GC_STATS_FD=99 \
322 "${instrumented[@]}" > /dev/null 99>$BASE_DIR/raw/$join_id.txt
323 else
324 "${instrumented[@]}" > /dev/null
325 fi
326 ;;
327 mut+alloc+free+gc+exit)
328 # also GC on exit
329 OILS_GC_STATS=1 OILS_GC_ON_EXIT=1 \
330 "${instrumented[@]}" > /dev/null
331 ;;
332
333 *)
334 die "Invalid shell runtime opts $shell_runtime_opts"
335 ;;
336 esac
337
338 done
339
340 # TODO: OILS_GC_STATS_FD and tsv_column_from_files.py
341}
342
343fd-demo() {
344 local out=_tmp/gc/demo.txt
345
346 local bin=_bin/cxx-dbg/oils-for-unix
347 ninja $bin
348
349 # Hm you can't do $fd>out.txt, but that's OK
350 local fd=99
351
352 OILS_GC_STATS_FD=$fd 99>$out \
353 $bin --ast-format none -n benchmarks/testdata/configure
354
355 ls -l $out
356 cat $out
357}
358
359more-variants() {
360 # TODO: could revive this
361
362 case $compare_more in
363 (*m32*)
364 # Surprisingly, -m32 is SLOWER, even though it allocates less.
365 # My guess is because less work is going into maintaining this code path in
366 # GCC.
367
368 # 223 ms
369 # 61.9 MB bytes allocated
370 local bin=_bin/cxx-opt32/oils-for-unix
371 OILS_GC_THRESHOLD=$big_threshold \
372 run-osh $tsv_out $bin 'm32 mutator+malloc' $file
373
374 # 280 ms
375 OILS_GC_STATS=1 \
376 run-osh $tsv_out $bin 'm32 mutator+malloc+free+gc' $file
377 ;;
378 esac
379
380 # Show log of GC
381 case $compare_more in
382 (*gcverbose*)
383 local bin=_bin/cxx-gcverbose/oils-for-unix
384 # 280 ms
385 OILS_GC_STATS=1 OILS_GC_ON_EXIT=1 \
386 run-osh $tsv_out $bin 'gcverbose mutator+malloc+free+gc' $file
387 ;;
388 esac
389
390 if command -v pretty-tsv; then
391 pretty-tsv $tsv_out
392 fi
393}
394
395build-binaries() {
396 if true; then
397
398 soil/cpp-tarball.sh build-like-ninja \
399 opt{,+bumpleak,+bumproot,+bumpsmall,+nopool}
400
401 OILS_TRANSLATOR=mycpp-souffle soil/cpp-tarball.sh build-like-ninja opt
402
403 else
404
405 # Old Ninja build
406 local -a bin=( _bin/cxx-opt{,+bumpleak,+bumproot,+bumpsmall,+nopool}/osh )
407 bin+=( _bin/cxx-opt/mycpp-souffle/osh )
408
409 if test -n "${TCMALLOC:-}"; then
410 bin+=( _bin/cxx-opt+tcmalloc/osh )
411 fi
412 ninja "${bin[@]}"
413 fi
414}
415
416measure-all() {
417 build-binaries
418
419 local tsv_out=${1:-$BASE_DIR/raw/times.tsv}
420 mkdir -p $(dirname $tsv_out)
421
422 # Make the header
423 time-tsv -o $tsv_out --print-header \
424 --rusage --field join_id --field task --field sh_path --field shell_runtime_opts
425
426 time print-tasks | run-tasks $tsv_out
427
428 if command -v pretty-tsv; then
429 pretty-tsv $tsv_out
430 fi
431}
432
433measure-cachegrind() {
434 build-binaries
435
436 local tsv_out=${1:-$BASE_DIR_CACHEGRIND/raw/times.tsv}
437
438 mkdir -p $(dirname $tsv_out)
439
440 # Make the header
441 time-tsv -o $tsv_out --print-header \
442 --rusage --field join_id --field task --field sh_path --field shell_runtime_opts
443
444 print-cachegrind-tasks | run-tasks $tsv_out cachegrind
445
446 # TODO: join cachegrind columns
447
448 if command -v pretty-tsv; then
449 pretty-tsv $tsv_out
450 fi
451}
452
453print-report() {
454 local in_dir=$1
455
456 benchmark-html-head 'Memory Management Overhead'
457
458 cat <<EOF
459 <body class="width60">
460 <p id="home-link">
461 <a href="/">oilshell.org</a>
462 </p>
463EOF
464
465 cmark << 'EOF'
466## Memory Management Overhead
467
468Source code: [oil/benchmarks/gc.sh](https://github.com/oilshell/oil/tree/master/benchmarks/gc.sh)
469EOF
470
471 cmark << 'EOF'
472### GC Stats
473
474EOF
475
476 tsv2html $in_dir/gc_stats.tsv
477
478 cmark << 'EOF'
479
480- Underlying data: [stage2/gc_stats.tsv](stage2/gc_stats.tsv)
481- More columns: [stage1/gc_stats.tsv](stage1/gc_stats.tsv)
482
483### Resource Usage
484
485#### parse.configure-cpython
486
487EOF
488
489 tsv2html $in_dir/parse.configure-cpython.tsv
490
491 cmark << 'EOF'
492#### parse.configure-coreutils
493
494Parsing the autoconf-generated `configure` script from GNU coreutils.
495
496Note that unlike other shells, `osh -n` retains all nodes on purpose. (See the
497[parser benchmark](../osh-parser/index.html)).
498
499EOF
500
501 tsv2html $in_dir/parse.configure-coreutils.tsv
502
503 cmark <<'EOF'
504#### parse.abuild
505
506Parsing `abuild` from Alpine Linux.
507EOF
508
509 tsv2html $in_dir/parse.abuild.tsv
510
511 cmark <<'EOF'
512#### ex.compute-fib
513
514A synthetic benchmark for POSIX shell arithmetic.
515EOF
516
517 tsv2html $in_dir/ex.compute-fib.tsv
518
519 cmark <<'EOF'
520#### ex.bashcomp-parse-help
521
522A realistic `bash-completion` workload.
523EOF
524
525 tsv2html $in_dir/ex.bashcomp-parse-help.tsv
526
527 cmark <<'EOF'
528#### ex.abuild-print-help
529
530Running `abuild -h` from Alpine Linux.
531
532EOF
533
534 tsv2html $in_dir/ex.abuild-print-help.tsv
535
536 cmark << 'EOF'
537- Underlying data: [stage2/times.tsv](stage2/times.tsv)
538EOF
539
540 cat <<EOF
541
542 </body>
543</html>
544EOF
545}
546
547make-report() {
548 mkdir -p $BASE_DIR/{stage1,stage2}
549
550 # Concatenate tiny files
551 benchmarks/gc_stats_to_tsv.py $BASE_DIR/raw/gc-*.txt \
552 > $BASE_DIR/stage1/gc_stats.tsv
553
554 # Make TSV files
555 benchmarks/report.R gc $BASE_DIR $BASE_DIR/stage2
556
557 # Make HTML
558 benchmarks/report.sh stage3 $BASE_DIR
559}
560
561soil-run() {
562 ### Run in soil/benchmarks
563
564 measure-all
565
566 make-report
567}
568
569#
570# Misc Tests
571#
572
573gc-parse-smoke() {
574 local variant=${1:-opt}
575 local file=${2:-configure}
576
577 local bin=_bin/cxx-$variant/osh
578 ninja $bin
579
580 # OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 \
581 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
582 $bin --ast-format none -n $file
583
584 # No leaks
585 # OILS_GC_STATS=1 OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 $bin -n -c '('
586}
587
588gc-parse-big() {
589 local variant=${1:-opt}
590
591 gc-parse-smoke $variant benchmarks/testdata/configure-coreutils
592}
593
594gc-run-smoke() {
595 local variant=${1:-opt}
596
597 local bin=_bin/cxx-$variant/oils-for-unix
598 ninja $bin
599
600 # expose a bug with printf
601 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=500 OILS_GC_ON_EXIT=1 \
602 $bin -c 'for i in $(seq 100); do printf "%s\\n" "-- $i"; done'
603}
604
605gc-run-oil() {
606 ### Run some scripts from the repo
607
608 local variant=${1:-opt}
609
610 local bin=_bin/cxx-$variant/oils-for-unix
611 ninja $bin
612
613 local i=0
614 for script in */*.sh; do
615 case $script in
616 (build/clean.sh|build/common.sh|build/dev.sh)
617 # Top level does something!
618 echo "=== SKIP $script"
619 continue
620 ;;
621 esac
622
623 echo
624 echo "=== ($i) $script"
625
626 # Just run the top level, which (hopefully) does nothing
627 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 \
628 $bin $script
629
630 i=$((i + 1))
631 if test $i -gt 60; then
632 break
633 fi
634 done
635}
636
637gc-run-big() {
638 local variant=${1:-opt}
639
640 local target=_bin/cxx-$variant/oils-for-unix
641 ninja $target
642
643 local osh=$REPO_ROOT/$target
644
645 local dir=_tmp/gc-run-big
646 rm -r -f -v $dir
647 mkdir -v -p $dir
648
649 pushd $dir
650 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=100000 OILS_GC_ON_EXIT=1 \
651 $osh ../../Python-2.7.13/configure
652 popd
653}
654
655run-verbose() {
656 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
657 /usr/bin/time --format '*** MAX RSS KiB = %M' -- \
658 "$@"
659}
660
661# This hit the 24-bit object ID limitation in 2.5 seconds
662# Should be able to run indefinitely.
663run-for-a-long-time() {
664 local bin=_bin/cxx-opt/osh
665 ninja $bin
666 run-verbose $bin benchmarks/compute/fib.sh 10000
667
668 # time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 _bin/cxx-opt/osh benchmarks/compute/fib.sh 10000
669}
670
671while-loop() {
672 local i=0
673 while test $i -lt 10000; do
674 if ((i % 1000 == 0)) ; then
675 echo $i
676 fi
677 i=$((i + 1))
678 continue # BUG: skipped GC point
679 done
680}
681
682for-loop() {
683 for i in $(seq 10000); do
684 if ((i % 1000 == 0)) ; then
685 echo $i
686 fi
687 continue
688 done
689}
690
691recurse() {
692 local n=${1:-3000}
693
694 if ((n % 100 == 0)) ; then
695 echo $n
696 fi
697
698 if test $n = 0; then
699 return
700 fi
701
702 recurse $((n - 1))
703}
704
705test-loops() {
706 ### Regression for leak
707
708 local bin=_bin/cxx-opt/osh
709 ninja $bin
710
711 run-verbose $bin $0 recurse
712 echo
713
714 run-verbose $bin $0 while-loop
715 echo
716
717 run-verbose $bin $0 for-loop
718}
719
720expand-loop() {
721 local n=$1
722
723 local bin=_bin/cxx-opt/osh
724 ninja $bin
725
726 set -x
727 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
728 $bin -c "for i in {1..$n}; do echo \$i; done > /dev/null"
729 set +x
730}
731
732test-brace-exp() {
733 expand-loop 330000
734 expand-loop 340000
735}
736
737"$@"