OILS / benchmarks / gc.sh View on Github | oilshell.org

760 lines, 426 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# benchmarks/gc.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
11
12source benchmarks/common.sh # benchmark-html-head
13source benchmarks/cachegrind.sh # with-cachegrind
14source build/dev-shell.sh # R_LIBS_USER
15source test/tsv-lib.sh
16
17readonly BASE_DIR=_tmp/gc
18
19# duplicated in benchmarks/gc-cachegrind.sh
20readonly BASE_DIR_CACHEGRIND=_tmp/gc-cachegrind
21
22# See benchmarks/gperftools.sh. I think the Ubuntu package is very old
23
24download-tcmalloc() {
25 # TODO: move this to ../oil_DEPS ?
26 wget --directory _deps \
27 https://github.com/gperftools/gperftools/releases/download/gperftools-2.10/gperftools-2.10.tar.gz
28
29 # Then ./configure; make; sudo make install
30 # installs in /usr/local/lib
31
32 # Note: there's a warning about libunwind -- maybe install that first. Does
33 # it only apply to CPU profiles?
34}
35
36debug-tcmalloc() {
37 touch mycpp/marksweep_heap.cc
38
39 # No evidence of difference
40 for bin in _bin/cxx-{opt,opt+tcmalloc}/osh; do
41 echo $bin
42 ninja $bin
43
44 ldd $bin
45 echo
46
47 ls -l $bin
48 echo
49
50 # Check what we're linking against
51 nm $bin | egrep -i 'malloc|calloc'
52 #wc -l
53 echo
54 done
55}
56
57install-m32() {
58 # needed to compile with -m32
59 sudo apt-get install gcc-multilib g++-multilib
60}
61
62max-rss() {
63 # %e is real time
64 /usr/bin/time --format '%e %M' -- "$@"
65}
66
67compare-m32() {
68 for bin in _bin/cxx-opt{,32}/osh; do
69 echo $bin
70 ninja $bin
71
72 ldd $bin
73 echo
74
75 file $bin
76 echo
77
78 ls -l $bin
79 echo
80
81 # 141136 KiB vs. 110924 KiB. Significant savings, but it's slower.
82 max-rss $bin --ast-format none -n benchmarks/testdata/configure-coreutils
83
84 done
85}
86
87banner() {
88 echo -----
89 echo "$@"
90}
91
92print-tasks() {
93 local mycpp_souffle=${1:-}
94
95 local -a workloads=(
96 parse.configure-coreutils
97 parse.configure-cpython
98 parse.abuild
99 ex.bashcomp-parse-help # only runs with bash
100 ex.abuild-print-help # bash / dash / zsh
101 ex.compute-fib # bash / dash / zsh
102 )
103
104 local -a shells=(
105 "bash$TAB-"
106 "dash$TAB-"
107 "zsh$TAB-"
108
109 "_bin/cxx-opt+bumpleak/osh${TAB}mut"
110 "_bin/cxx-opt+bumproot/osh${TAB}mut"
111
112 "_bin/cxx-opt+bumpsmall/osh${TAB}mut+alloc"
113 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc"
114 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc+free+gc"
115
116 # these have trivial GC stats
117 "_bin/cxx-opt/osh${TAB}mut+alloc"
118 "_bin/cxx-opt/osh${TAB}mut+alloc+free"
119 # good GC stats
120 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc"
121 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc+exit"
122 )
123
124 if test -n "$mycpp_souffle"; then
125 shells+=(
126 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc"
127 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free"
128 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free+gc"
129 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free+gc+exit"
130 )
131 fi
132
133 if test -n "${TCMALLOC:-}"; then
134 shells+=(
135 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc"
136 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc+free"
137 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc+free+gc"
138 )
139 fi
140
141 local id=0
142
143 for workload in "${workloads[@]}"; do
144 for shell in "${shells[@]}"; do
145 local row_part="$workload${TAB}$shell"
146
147 # Skip these rows
148 case $row_part in
149 "ex.bashcomp-parse-help${TAB}dash"*)
150 continue
151 ;;
152 "ex.bashcomp-parse-help${TAB}zsh"*)
153 continue
154 ;;
155 esac
156
157 local join_id="gc-$id"
158 local row="$join_id${TAB}$row_part"
159 echo "$row"
160
161 id=$((id + 1))
162
163 done
164
165 # Run a quick 10 tasks
166 if test -n "${QUICKLY:-}" && test $id -gt 10; then
167 break
168 fi
169 done
170}
171
172print-cachegrind-tasks() {
173 local mycpp_souffle=${1:-}
174
175 local -a workloads=(
176 # coreutils is on osh-parser
177 #parse.configure-coreutils
178
179 #parse.configure-cpython
180
181 # Faster tasks, like benchmarks/uftrace, which is instrumented
182 parse.abuild
183 ex.compute-fib
184 )
185
186 local -a shells=(
187 "bash${TAB}-"
188 "_bin/cxx-opt+bumpleak/osh${TAB}mut"
189 "_bin/cxx-opt+bumproot/osh${TAB}mut"
190
191 "_bin/cxx-opt+bumpsmall/osh${TAB}mut+alloc"
192 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc"
193 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc+free+gc"
194
195 "_bin/cxx-opt/osh${TAB}mut+alloc"
196 "_bin/cxx-opt/osh${TAB}mut+alloc+free"
197 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc"
198 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc+exit"
199 )
200
201 if test -n "$mycpp_souffle"; then
202 shells+=(
203 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc"
204 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free"
205 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free+gc"
206 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free+gc+exit"
207 )
208 fi
209
210 local id=0
211 for workload in "${workloads[@]}"; do
212 for shell in "${shells[@]}"; do
213 local row_part="$workload${TAB}$shell"
214
215 local join_id="cachegrind-$id"
216 local row="$join_id${TAB}$row_part"
217 echo "$row"
218
219 id=$((id + 1))
220 done
221 done
222 #print-tasks | egrep 'configure-coreutils' | egrep osh
223}
224
225
226readonly BIG_THRESHOLD=$(( 1 * 1000 * 1000 * 1000 )) # 1 B
227
228run-tasks() {
229 local tsv_out=$1
230 local mode=${2:-time}
231
232 while read -r join_id task sh_path shell_runtime_opts; do
233
234 # Parse different files
235 case $task in
236 parse.configure-coreutils)
237 data_file='benchmarks/testdata/configure-coreutils'
238 ;;
239 parse.configure-cpython)
240 data_file='Python-2.7.13/configure'
241 ;;
242 parse.abuild)
243 data_file='benchmarks/testdata/abuild'
244 ;;
245 esac
246
247 # Construct argv for each task
248 local -a argv
249 case $task in
250 parse.*)
251 argv=( -n $data_file )
252
253 case $sh_path in
254 _bin/*/osh)
255 argv=( --ast-format none "${argv[@]}" )
256 ;;
257 esac
258 ;;
259
260 ex.bashcomp-parse-help)
261 argv=( benchmarks/parse-help/pure-excerpt.sh parse_help_file
262 benchmarks/parse-help/clang.txt )
263 ;;
264
265 ex.abuild-print-help)
266 argv=( testdata/osh-runtime/abuild -h )
267 ;;
268
269 ex.compute-fib)
270 # fewer iterations when instrumented
271 local iters
272 if test $mode = time; then
273 iters=100
274 else
275 iters=10
276 fi
277
278 argv=( benchmarks/compute/fib.sh $iters 44 )
279 ;;
280
281 *)
282 die "Invalid task $task"
283 ;;
284 esac
285
286 echo $join_id $task $sh_path $shell_runtime_opts
287
288 argv=( $sh_path "${argv[@]}" )
289 #echo + "${argv[@]}"
290 #set -x
291
292 if test $mode = cachegrind; then
293 # Add prefix
294 argv=( $0 with-cachegrind $BASE_DIR_CACHEGRIND/raw/$join_id.txt "${argv[@]}" )
295 fi
296
297 # Wrap in a command that writes one row of a TSV
298 # Note: for cachegrind, we need the join ID, but the --rusage is meaningless
299 local -a instrumented=(
300 time-tsv -o $tsv_out --append
301 --rusage
302 --field "$join_id" --field "$task" --field "$sh_path"
303 --field "$shell_runtime_opts"
304 -- "${argv[@]}"
305 )
306
307 # Run with the right environment variables
308
309 case $shell_runtime_opts in
310 -)
311 "${instrumented[@]}" > /dev/null
312 ;;
313 mut)
314 OILS_GC_STATS=1 \
315 "${instrumented[@]}" > /dev/null
316 ;;
317 mut+alloc)
318 # disable GC with big threshold
319 OILS_GC_STATS=1 OILS_GC_THRESHOLD=$BIG_THRESHOLD \
320 "${instrumented[@]}" > /dev/null
321 ;;
322 mut+alloc+free)
323 # do a single GC on exit
324 OILS_GC_STATS=1 OILS_GC_THRESHOLD=$BIG_THRESHOLD OILS_GC_ON_EXIT=1 \
325 "${instrumented[@]}" > /dev/null
326 ;;
327 mut+alloc+free+gc)
328 # Default configuration
329 #
330 # Save the GC stats here. None of the other runtime options are that
331 # interesting.
332
333 if test $mode = 'time' && test $sh_path != _bin/cxx-opt+nopool/osh; then
334 OILS_GC_STATS_FD=99 \
335 "${instrumented[@]}" > /dev/null 99>$BASE_DIR/raw/$join_id.txt
336 else
337 "${instrumented[@]}" > /dev/null
338 fi
339 ;;
340 mut+alloc+free+gc+exit)
341 # also GC on exit
342 OILS_GC_STATS=1 OILS_GC_ON_EXIT=1 \
343 "${instrumented[@]}" > /dev/null
344 ;;
345
346 *)
347 die "Invalid shell runtime opts $shell_runtime_opts"
348 ;;
349 esac
350
351 done
352
353 # TODO: OILS_GC_STATS_FD and tsv_column_from_files.py
354}
355
356fd-demo() {
357 local out=_tmp/gc/demo.txt
358
359 local bin=_bin/cxx-dbg/oils-for-unix
360 ninja $bin
361
362 # Hm you can't do $fd>out.txt, but that's OK
363 local fd=99
364
365 OILS_GC_STATS_FD=$fd 99>$out \
366 $bin --ast-format none -n benchmarks/testdata/configure
367
368 ls -l $out
369 cat $out
370}
371
372more-variants() {
373 # TODO: could revive this
374
375 case $compare_more in
376 (*m32*)
377 # Surprisingly, -m32 is SLOWER, even though it allocates less.
378 # My guess is because less work is going into maintaining this code path in
379 # GCC.
380
381 # 223 ms
382 # 61.9 MB bytes allocated
383 local bin=_bin/cxx-opt32/oils-for-unix
384 OILS_GC_THRESHOLD=$big_threshold \
385 run-osh $tsv_out $bin 'm32 mutator+malloc' $file
386
387 # 280 ms
388 OILS_GC_STATS=1 \
389 run-osh $tsv_out $bin 'm32 mutator+malloc+free+gc' $file
390 ;;
391 esac
392
393 # Show log of GC
394 case $compare_more in
395 (*gcverbose*)
396 local bin=_bin/cxx-gcverbose/oils-for-unix
397 # 280 ms
398 OILS_GC_STATS=1 OILS_GC_ON_EXIT=1 \
399 run-osh $tsv_out $bin 'gcverbose mutator+malloc+free+gc' $file
400 ;;
401 esac
402
403 if command -v pretty-tsv; then
404 pretty-tsv $tsv_out
405 fi
406}
407
408build-binaries() {
409 if true; then
410
411 soil/cpp-tarball.sh build-like-ninja \
412 opt{,+bumpleak,+bumproot,+bumpsmall,+nopool}
413
414 OILS_TRANSLATOR=mycpp-souffle soil/cpp-tarball.sh build-like-ninja opt
415
416 else
417
418 # Old Ninja build
419 local -a bin=( _bin/cxx-opt{,+bumpleak,+bumproot,+bumpsmall,+nopool}/osh )
420 bin+=( _bin/cxx-opt/mycpp-souffle/osh )
421
422 if test -n "${TCMALLOC:-}"; then
423 bin+=( _bin/cxx-opt+tcmalloc/osh )
424 fi
425 ninja "${bin[@]}"
426 fi
427}
428
429measure-all() {
430 local tsv_out=${1:-$BASE_DIR/raw/times.tsv}
431 local mycpp_souffle=${2:-}
432
433 build-binaries
434
435 mkdir -p $(dirname $tsv_out)
436
437 # Make the header
438 time-tsv -o $tsv_out --print-header \
439 --rusage --field join_id --field task --field sh_path --field shell_runtime_opts
440
441 # Pass through args, which may include mycpp-souffle
442 time print-tasks "$mycpp_souffle" | run-tasks $tsv_out
443
444 if command -v pretty-tsv; then
445 pretty-tsv $tsv_out
446 fi
447}
448
449measure-cachegrind() {
450 local tsv_out=${1:-$BASE_DIR_CACHEGRIND/raw/times.tsv}
451 local mycpp_souffle=${2:-}
452
453 build-binaries
454
455 mkdir -p $(dirname $tsv_out)
456
457 # Make the header
458 time-tsv -o $tsv_out --print-header \
459 --rusage --field join_id --field task --field sh_path --field shell_runtime_opts
460
461 print-cachegrind-tasks "$mycpp_souffle" | run-tasks $tsv_out cachegrind
462
463 # TODO: join cachegrind columns
464
465 if command -v pretty-tsv; then
466 pretty-tsv $tsv_out
467 fi
468}
469
470print-report() {
471 local in_dir=$1
472
473 benchmark-html-head 'Memory Management Overhead'
474
475 cat <<EOF
476 <body class="width60">
477 <p id="home-link">
478 <a href="/">oilshell.org</a>
479 </p>
480EOF
481
482 cmark << 'EOF'
483## Memory Management Overhead
484
485Source code: [oil/benchmarks/gc.sh](https://github.com/oilshell/oil/tree/master/benchmarks/gc.sh)
486EOF
487
488 cmark << 'EOF'
489### GC Stats
490
491EOF
492
493 tsv2html $in_dir/gc_stats.tsv
494
495 cmark << 'EOF'
496
497- Underlying data: [stage2/gc_stats.tsv](stage2/gc_stats.tsv)
498- More columns: [stage1/gc_stats.tsv](stage1/gc_stats.tsv)
499
500### Resource Usage
501
502#### parse.configure-cpython
503
504EOF
505
506 tsv2html $in_dir/parse.configure-cpython.tsv
507
508 cmark << 'EOF'
509#### parse.configure-coreutils
510
511Parsing the autoconf-generated `configure` script from GNU coreutils.
512
513Note that unlike other shells, `osh -n` retains all nodes on purpose. (See the
514[parser benchmark](../osh-parser/index.html)).
515
516EOF
517
518 tsv2html $in_dir/parse.configure-coreutils.tsv
519
520 cmark <<'EOF'
521#### parse.abuild
522
523Parsing `abuild` from Alpine Linux.
524EOF
525
526 tsv2html $in_dir/parse.abuild.tsv
527
528 cmark <<'EOF'
529#### ex.compute-fib
530
531A synthetic benchmark for POSIX shell arithmetic.
532EOF
533
534 tsv2html $in_dir/ex.compute-fib.tsv
535
536 cmark <<'EOF'
537#### ex.bashcomp-parse-help
538
539A realistic `bash-completion` workload.
540EOF
541
542 tsv2html $in_dir/ex.bashcomp-parse-help.tsv
543
544 cmark <<'EOF'
545#### ex.abuild-print-help
546
547Running `abuild -h` from Alpine Linux.
548
549EOF
550
551 tsv2html $in_dir/ex.abuild-print-help.tsv
552
553 cmark << 'EOF'
554- Underlying data: [stage2/times.tsv](stage2/times.tsv)
555EOF
556
557 cat <<EOF
558
559 </body>
560</html>
561EOF
562}
563
564make-report() {
565 mkdir -p $BASE_DIR/{stage1,stage2}
566
567 # Concatenate tiny files
568 benchmarks/gc_stats_to_tsv.py $BASE_DIR/raw/gc-*.txt \
569 > $BASE_DIR/stage1/gc_stats.tsv
570
571 # Make TSV files
572 benchmarks/report.R gc $BASE_DIR $BASE_DIR/stage2
573
574 # Make HTML
575 benchmarks/report.sh stage3 $BASE_DIR
576}
577
578soil-run() {
579 ### Run in soil/benchmarks
580
581 measure-all '' mycpp-souffle
582
583 make-report
584}
585
586run-for-release() {
587 measure-all ''
588
589 make-report
590}
591
592#
593# Misc Tests
594#
595
596gc-parse-smoke() {
597 local variant=${1:-opt}
598 local file=${2:-configure}
599
600 local bin=_bin/cxx-$variant/osh
601 ninja $bin
602
603 # OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 \
604 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
605 $bin --ast-format none -n $file
606
607 # No leaks
608 # OILS_GC_STATS=1 OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 $bin -n -c '('
609}
610
611gc-parse-big() {
612 local variant=${1:-opt}
613
614 gc-parse-smoke $variant benchmarks/testdata/configure-coreutils
615}
616
617gc-run-smoke() {
618 local variant=${1:-opt}
619
620 local bin=_bin/cxx-$variant/oils-for-unix
621 ninja $bin
622
623 # expose a bug with printf
624 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=500 OILS_GC_ON_EXIT=1 \
625 $bin -c 'for i in $(seq 100); do printf "%s\\n" "-- $i"; done'
626}
627
628gc-run-oil() {
629 ### Run some scripts from the repo
630
631 local variant=${1:-opt}
632
633 local bin=_bin/cxx-$variant/oils-for-unix
634 ninja $bin
635
636 local i=0
637 for script in */*.sh; do
638 case $script in
639 (build/clean.sh|build/common.sh|build/dev.sh)
640 # Top level does something!
641 echo "=== SKIP $script"
642 continue
643 ;;
644 esac
645
646 echo
647 echo "=== ($i) $script"
648
649 # Just run the top level, which (hopefully) does nothing
650 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 \
651 $bin $script
652
653 i=$((i + 1))
654 if test $i -gt 60; then
655 break
656 fi
657 done
658}
659
660gc-run-big() {
661 local variant=${1:-opt}
662
663 local target=_bin/cxx-$variant/oils-for-unix
664 ninja $target
665
666 local osh=$REPO_ROOT/$target
667
668 local dir=_tmp/gc-run-big
669 rm -r -f -v $dir
670 mkdir -v -p $dir
671
672 pushd $dir
673 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=100000 OILS_GC_ON_EXIT=1 \
674 $osh ../../Python-2.7.13/configure
675 popd
676}
677
678run-verbose() {
679 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
680 /usr/bin/time --format '*** MAX RSS KiB = %M' -- \
681 "$@"
682}
683
684# This hit the 24-bit object ID limitation in 2.5 seconds
685# Should be able to run indefinitely.
686run-for-a-long-time() {
687 local bin=_bin/cxx-opt/osh
688 ninja $bin
689 run-verbose $bin benchmarks/compute/fib.sh 10000
690
691 # time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 _bin/cxx-opt/osh benchmarks/compute/fib.sh 10000
692}
693
694while-loop() {
695 local i=0
696 while test $i -lt 10000; do
697 if ((i % 1000 == 0)) ; then
698 echo $i
699 fi
700 i=$((i + 1))
701 continue # BUG: skipped GC point
702 done
703}
704
705for-loop() {
706 for i in $(seq 10000); do
707 if ((i % 1000 == 0)) ; then
708 echo $i
709 fi
710 continue
711 done
712}
713
714recurse() {
715 local n=${1:-3000}
716
717 if ((n % 100 == 0)) ; then
718 echo $n
719 fi
720
721 if test $n = 0; then
722 return
723 fi
724
725 recurse $((n - 1))
726}
727
728test-loops() {
729 ### Regression for leak
730
731 local bin=_bin/cxx-opt/osh
732 ninja $bin
733
734 run-verbose $bin $0 recurse
735 echo
736
737 run-verbose $bin $0 while-loop
738 echo
739
740 run-verbose $bin $0 for-loop
741}
742
743expand-loop() {
744 local n=$1
745
746 local bin=_bin/cxx-opt/osh
747 ninja $bin
748
749 set -x
750 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
751 $bin -c "for i in {1..$n}; do echo \$i; done > /dev/null"
752 set +x
753}
754
755test-brace-exp() {
756 expand-loop 330000
757 expand-loop 340000
758}
759
760"$@"