OILS / benchmarks / gc.sh View on Github | oils.pub

755 lines, 422 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# benchmarks/gc.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
11
12source benchmarks/common.sh # benchmark-html-head
13source benchmarks/cachegrind.sh # with-cachegrind
14source build/dev-shell.sh # R_LIBS_USER
15source test/tsv-lib.sh
16
17readonly BASE_DIR=_tmp/gc
18
19# duplicated in benchmarks/gc-cachegrind.sh
20readonly BASE_DIR_CACHEGRIND=_tmp/gc-cachegrind
21
22# See benchmarks/gperftools.sh. I think the Ubuntu package is very old
23
24download-tcmalloc() {
25 # TODO: move this to ../oil_DEPS ?
26 wget --directory _deps \
27 https://github.com/gperftools/gperftools/releases/download/gperftools-2.10/gperftools-2.10.tar.gz
28
29 # Then ./configure; make; sudo make install
30 # installs in /usr/local/lib
31
32 # Note: there's a warning about libunwind -- maybe install that first. Does
33 # it only apply to CPU profiles?
34}
35
36debug-tcmalloc() {
37 touch mycpp/marksweep_heap.cc
38
39 # No evidence of difference
40 for bin in _bin/cxx-{opt,opt+tcmalloc}/osh; do
41 echo $bin
42 ninja $bin
43
44 ldd $bin
45 echo
46
47 ls -l $bin
48 echo
49
50 # Check what we're linking against
51 nm $bin | egrep -i 'malloc|calloc'
52 #wc -l
53 echo
54 done
55}
56
57install-m32() {
58 # needed to compile with -m32
59 sudo apt-get install gcc-multilib g++-multilib
60}
61
62max-rss() {
63 # %e is real time
64 /usr/bin/time --format '%e %M' -- "$@"
65}
66
67compare-m32() {
68 for bin in _bin/cxx-opt{,32}/osh; do
69 echo $bin
70 ninja $bin
71
72 ldd $bin
73 echo
74
75 file $bin
76 echo
77
78 ls -l $bin
79 echo
80
81 # 141136 KiB vs. 110924 KiB. Significant savings, but it's slower.
82 max-rss $bin --ast-format none -n benchmarks/testdata/configure-coreutils
83
84 done
85}
86
87print-tasks() {
88 local mycpp_souffle=${1:-}
89
90 local -a workloads=(
91 parse.configure-coreutils
92 parse.configure-cpython
93 parse.abuild
94 ex.bashcomp-parse-help # only runs with bash
95 ex.abuild-print-help # bash / dash / zsh
96 ex.compute-fib # bash / dash / zsh
97 )
98
99 local -a shells=(
100 "bash$TAB-"
101 "dash$TAB-"
102 "zsh$TAB-"
103
104 "_bin/cxx-opt+bumpleak/osh${TAB}mut"
105 "_bin/cxx-opt+bumproot/osh${TAB}mut"
106
107 "_bin/cxx-opt+bumpsmall/osh${TAB}mut+alloc"
108 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc"
109 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc+free+gc"
110
111 # these have trivial GC stats
112 "_bin/cxx-opt/osh${TAB}mut+alloc"
113 "_bin/cxx-opt/osh${TAB}mut+alloc+free"
114 # good GC stats
115 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc"
116 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc+exit"
117 )
118
119 if test -n "$mycpp_souffle"; then
120 shells+=(
121 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc"
122 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free"
123 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free+gc"
124 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free+gc+exit"
125 )
126 fi
127
128 if test -n "${TCMALLOC:-}"; then
129 shells+=(
130 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc"
131 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc+free"
132 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc+free+gc"
133 )
134 fi
135
136 local id=0
137
138 for workload in "${workloads[@]}"; do
139 for shell in "${shells[@]}"; do
140 local row_part="$workload${TAB}$shell"
141
142 # Skip these rows
143 case $row_part in
144 "ex.bashcomp-parse-help${TAB}dash"*)
145 continue
146 ;;
147 "ex.bashcomp-parse-help${TAB}zsh"*)
148 continue
149 ;;
150 esac
151
152 local join_id="gc-$id"
153 local row="$join_id${TAB}$row_part"
154 echo "$row"
155
156 id=$((id + 1))
157
158 done
159
160 # Run a quick 10 tasks
161 if test -n "${QUICKLY:-}" && test $id -gt 10; then
162 break
163 fi
164 done
165}
166
167print-cachegrind-tasks() {
168 local mycpp_souffle=${1:-}
169
170 local -a workloads=(
171 # coreutils is on osh-parser
172 #parse.configure-coreutils
173
174 #parse.configure-cpython
175
176 # Faster tasks, like benchmarks/uftrace, which is instrumented
177 parse.abuild
178 ex.compute-fib
179 )
180
181 local -a shells=(
182 "bash${TAB}-"
183 "_bin/cxx-opt+bumpleak/osh${TAB}mut"
184 "_bin/cxx-opt+bumproot/osh${TAB}mut"
185
186 "_bin/cxx-opt+bumpsmall/osh${TAB}mut+alloc"
187 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc"
188 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc+free+gc"
189
190 "_bin/cxx-opt/osh${TAB}mut+alloc"
191 "_bin/cxx-opt/osh${TAB}mut+alloc+free"
192 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc"
193 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc+exit"
194 )
195
196 if test -n "$mycpp_souffle"; then
197 shells+=(
198 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc"
199 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free"
200 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free+gc"
201 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free+gc+exit"
202 )
203 fi
204
205 local id=0
206 for workload in "${workloads[@]}"; do
207 for shell in "${shells[@]}"; do
208 local row_part="$workload${TAB}$shell"
209
210 local join_id="cachegrind-$id"
211 local row="$join_id${TAB}$row_part"
212 echo "$row"
213
214 id=$((id + 1))
215 done
216 done
217 #print-tasks | egrep 'configure-coreutils' | egrep osh
218}
219
220
221readonly BIG_THRESHOLD=$(( 1 * 1000 * 1000 * 1000 )) # 1 B
222
223run-tasks() {
224 local tsv_out=$1
225 local mode=${2:-time}
226
227 while read -r join_id task sh_path shell_runtime_opts; do
228
229 # Parse different files
230 case $task in
231 parse.configure-coreutils)
232 data_file='benchmarks/testdata/configure-coreutils'
233 ;;
234 parse.configure-cpython)
235 data_file='Python-2.7.13/configure'
236 ;;
237 parse.abuild)
238 data_file='benchmarks/testdata/abuild'
239 ;;
240 esac
241
242 # Construct argv for each task
243 local -a argv
244 case $task in
245 parse.*)
246 argv=( -n $data_file )
247
248 case $sh_path in
249 _bin/*/osh)
250 argv=( --ast-format none "${argv[@]}" )
251 ;;
252 esac
253 ;;
254
255 ex.bashcomp-parse-help)
256 argv=( benchmarks/parse-help/pure-excerpt.sh parse_help_file
257 benchmarks/parse-help/clang.txt )
258 ;;
259
260 ex.abuild-print-help)
261 argv=( testdata/osh-runtime/abuild -h )
262 ;;
263
264 ex.compute-fib)
265 # fewer iterations when instrumented
266 local iters
267 if test $mode = time; then
268 iters=100
269 else
270 iters=10
271 fi
272
273 argv=( benchmarks/compute/fib.sh $iters 44 )
274 ;;
275
276 *)
277 die "Invalid task $task"
278 ;;
279 esac
280
281 echo $join_id $task $sh_path $shell_runtime_opts
282
283 argv=( $sh_path "${argv[@]}" )
284 #echo + "${argv[@]}"
285 #set -x
286
287 if test $mode = cachegrind; then
288 # Add prefix
289 argv=( $0 with-cachegrind $BASE_DIR_CACHEGRIND/raw/$join_id.txt "${argv[@]}" )
290 fi
291
292 # Wrap in a command that writes one row of a TSV
293 # Note: for cachegrind, we need the join ID, but the --rusage is meaningless
294 local -a instrumented=(
295 time-tsv -o $tsv_out --append
296 --rusage
297 --field "$join_id" --field "$task" --field "$sh_path"
298 --field "$shell_runtime_opts"
299 -- "${argv[@]}"
300 )
301
302 # Run with the right environment variables
303
304 case $shell_runtime_opts in
305 -)
306 "${instrumented[@]}" > /dev/null
307 ;;
308 mut)
309 OILS_GC_STATS=1 \
310 "${instrumented[@]}" > /dev/null
311 ;;
312 mut+alloc)
313 # disable GC with big threshold
314 OILS_GC_STATS=1 OILS_GC_THRESHOLD=$BIG_THRESHOLD \
315 "${instrumented[@]}" > /dev/null
316 ;;
317 mut+alloc+free)
318 # do a single GC on exit
319 OILS_GC_STATS=1 OILS_GC_THRESHOLD=$BIG_THRESHOLD OILS_GC_ON_EXIT=1 \
320 "${instrumented[@]}" > /dev/null
321 ;;
322 mut+alloc+free+gc)
323 # Default configuration
324 #
325 # Save the GC stats here. None of the other runtime options are that
326 # interesting.
327
328 if test $mode = 'time' && test $sh_path != _bin/cxx-opt+nopool/osh; then
329 OILS_GC_STATS_FD=99 \
330 "${instrumented[@]}" > /dev/null 99>$BASE_DIR/raw/$join_id.txt
331 else
332 "${instrumented[@]}" > /dev/null
333 fi
334 ;;
335 mut+alloc+free+gc+exit)
336 # also GC on exit
337 OILS_GC_STATS=1 OILS_GC_ON_EXIT=1 \
338 "${instrumented[@]}" > /dev/null
339 ;;
340
341 *)
342 die "Invalid shell runtime opts $shell_runtime_opts"
343 ;;
344 esac
345
346 done
347
348 # TODO: OILS_GC_STATS_FD and tsv_column_from_files.py
349}
350
351fd-demo() {
352 local out=_tmp/gc/demo.txt
353
354 local bin=_bin/cxx-dbg/oils-for-unix
355 ninja $bin
356
357 # Hm you can't do $fd>out.txt, but that's OK
358 local fd=99
359
360 OILS_GC_STATS_FD=$fd 99>$out \
361 $bin --ast-format none -n benchmarks/testdata/configure
362
363 ls -l $out
364 cat $out
365}
366
367more-variants() {
368 # TODO: could revive this
369
370 case $compare_more in
371 (*m32*)
372 # Surprisingly, -m32 is SLOWER, even though it allocates less.
373 # My guess is because less work is going into maintaining this code path in
374 # GCC.
375
376 # 223 ms
377 # 61.9 MB bytes allocated
378 local bin=_bin/cxx-opt32/oils-for-unix
379 OILS_GC_THRESHOLD=$big_threshold \
380 run-osh $tsv_out $bin 'm32 mutator+malloc' $file
381
382 # 280 ms
383 OILS_GC_STATS=1 \
384 run-osh $tsv_out $bin 'm32 mutator+malloc+free+gc' $file
385 ;;
386 esac
387
388 # Show log of GC
389 case $compare_more in
390 (*gcverbose*)
391 local bin=_bin/cxx-gcverbose/oils-for-unix
392 # 280 ms
393 OILS_GC_STATS=1 OILS_GC_ON_EXIT=1 \
394 run-osh $tsv_out $bin 'gcverbose mutator+malloc+free+gc' $file
395 ;;
396 esac
397
398 if command -v pretty-tsv; then
399 pretty-tsv $tsv_out
400 fi
401}
402
403build-binaries() {
404 if true; then
405
406 soil/cpp-tarball.sh build-like-ninja \
407 opt{,+bumpleak,+bumproot,+bumpsmall,+nopool}
408
409 OILS_TRANSLATOR=mycpp-souffle soil/cpp-tarball.sh build-like-ninja opt
410
411 else
412
413 # Old Ninja build
414 local -a bin=( _bin/cxx-opt{,+bumpleak,+bumproot,+bumpsmall,+nopool}/osh )
415 bin+=( _bin/cxx-opt/mycpp-souffle/osh )
416
417 if test -n "${TCMALLOC:-}"; then
418 bin+=( _bin/cxx-opt+tcmalloc/osh )
419 fi
420 ninja "${bin[@]}"
421 fi
422}
423
424measure-all() {
425 local tsv_out=${1:-$BASE_DIR/raw/times.tsv}
426 local mycpp_souffle=${2:-}
427
428 build-binaries
429
430 mkdir -p $(dirname $tsv_out)
431
432 # Make the header
433 time-tsv -o $tsv_out --print-header \
434 --rusage --field join_id --field task --field sh_path --field shell_runtime_opts
435
436 # Pass through args, which may include mycpp-souffle
437 time print-tasks "$mycpp_souffle" | run-tasks $tsv_out
438
439 if command -v pretty-tsv; then
440 pretty-tsv $tsv_out
441 fi
442}
443
444measure-cachegrind() {
445 local tsv_out=${1:-$BASE_DIR_CACHEGRIND/raw/times.tsv}
446 local mycpp_souffle=${2:-}
447
448 build-binaries
449
450 mkdir -p $(dirname $tsv_out)
451
452 # Make the header
453 time-tsv -o $tsv_out --print-header \
454 --rusage --field join_id --field task --field sh_path --field shell_runtime_opts
455
456 print-cachegrind-tasks "$mycpp_souffle" | run-tasks $tsv_out cachegrind
457
458 # TODO: join cachegrind columns
459
460 if command -v pretty-tsv; then
461 pretty-tsv $tsv_out
462 fi
463}
464
465print-report() {
466 local in_dir=$1
467
468 benchmark-html-head 'Memory Management Overhead'
469
470 cat <<EOF
471 <body class="width60">
472 <p id="home-link">
473 <a href="/">oils.pub</a>
474 </p>
475EOF
476
477 cmark << 'EOF'
478## Memory Management Overhead
479
480Source code: [oil/benchmarks/gc.sh](https://github.com/oilshell/oil/tree/master/benchmarks/gc.sh)
481EOF
482
483 cmark << 'EOF'
484### GC Stats
485
486EOF
487
488 tsv2html $in_dir/gc_stats.tsv
489
490 cmark << 'EOF'
491
492- Underlying data: [stage2/gc_stats.tsv](stage2/gc_stats.tsv)
493- More columns: [stage1/gc_stats.tsv](stage1/gc_stats.tsv)
494
495### Resource Usage
496
497#### parse.configure-cpython
498
499EOF
500
501 tsv2html $in_dir/parse.configure-cpython.tsv
502
503 cmark << 'EOF'
504#### parse.configure-coreutils
505
506Parsing the autoconf-generated `configure` script from GNU coreutils.
507
508Note that unlike other shells, `osh -n` retains all nodes on purpose. (See the
509[parser benchmark](../osh-parser/index.html)).
510
511EOF
512
513 tsv2html $in_dir/parse.configure-coreutils.tsv
514
515 cmark <<'EOF'
516#### parse.abuild
517
518Parsing `abuild` from Alpine Linux.
519EOF
520
521 tsv2html $in_dir/parse.abuild.tsv
522
523 cmark <<'EOF'
524#### ex.compute-fib
525
526A synthetic benchmark for POSIX shell arithmetic.
527EOF
528
529 tsv2html $in_dir/ex.compute-fib.tsv
530
531 cmark <<'EOF'
532#### ex.bashcomp-parse-help
533
534A realistic `bash-completion` workload.
535EOF
536
537 tsv2html $in_dir/ex.bashcomp-parse-help.tsv
538
539 cmark <<'EOF'
540#### ex.abuild-print-help
541
542Running `abuild -h` from Alpine Linux.
543
544EOF
545
546 tsv2html $in_dir/ex.abuild-print-help.tsv
547
548 cmark << 'EOF'
549- Underlying data: [stage2/times.tsv](stage2/times.tsv)
550EOF
551
552 cat <<EOF
553
554 </body>
555</html>
556EOF
557}
558
559make-report() {
560 mkdir -p $BASE_DIR/{stage1,stage2}
561
562 # Concatenate tiny files
563 benchmarks/gc_stats_to_tsv.py $BASE_DIR/raw/gc-*.txt \
564 > $BASE_DIR/stage1/gc_stats.tsv
565
566 # Make TSV files
567 benchmarks/report.R gc $BASE_DIR $BASE_DIR/stage2
568
569 # Make HTML
570 benchmarks/report.sh stage3 $BASE_DIR
571}
572
573soil-run() {
574 ### Run in soil/benchmarks
575
576 measure-all '' mycpp-souffle
577
578 make-report
579}
580
581run-for-release() {
582 measure-all ''
583
584 make-report
585}
586
587#
588# Misc Tests
589#
590
591gc-parse-smoke() {
592 local variant=${1:-opt}
593 local file=${2:-configure}
594
595 local bin=_bin/cxx-$variant/osh
596 ninja $bin
597
598 # OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 \
599 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
600 $bin --ast-format none -n $file
601
602 # No leaks
603 # OILS_GC_STATS=1 OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 $bin -n -c '('
604}
605
606gc-parse-big() {
607 local variant=${1:-opt}
608
609 gc-parse-smoke $variant benchmarks/testdata/configure-coreutils
610}
611
612gc-run-smoke() {
613 local variant=${1:-opt}
614
615 local bin=_bin/cxx-$variant/oils-for-unix
616 ninja $bin
617
618 # expose a bug with printf
619 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=500 OILS_GC_ON_EXIT=1 \
620 $bin -c 'for i in $(seq 100); do printf "%s\\n" "-- $i"; done'
621}
622
623gc-run-oil() {
624 ### Run some scripts from the repo
625
626 local variant=${1:-opt}
627
628 local bin=_bin/cxx-$variant/oils-for-unix
629 ninja $bin
630
631 local i=0
632 for script in */*.sh; do
633 case $script in
634 (build/clean.sh|build/common.sh|build/dev.sh)
635 # Top level does something!
636 echo "=== SKIP $script"
637 continue
638 ;;
639 esac
640
641 echo
642 echo "=== ($i) $script"
643
644 # Just run the top level, which (hopefully) does nothing
645 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 \
646 $bin $script
647
648 i=$((i + 1))
649 if test $i -gt 60; then
650 break
651 fi
652 done
653}
654
655gc-run-big() {
656 local variant=${1:-opt}
657
658 local target=_bin/cxx-$variant/oils-for-unix
659 ninja $target
660
661 local osh=$REPO_ROOT/$target
662
663 local dir=_tmp/gc-run-big
664 rm -r -f -v $dir
665 mkdir -v -p $dir
666
667 pushd $dir
668 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=100000 OILS_GC_ON_EXIT=1 \
669 $osh ../../Python-2.7.13/configure
670 popd
671}
672
673run-verbose() {
674 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
675 /usr/bin/time --format '*** MAX RSS KiB = %M' -- \
676 "$@"
677}
678
679# This hit the 24-bit object ID limitation in 2.5 seconds
680# Should be able to run indefinitely.
681run-for-a-long-time() {
682 local bin=_bin/cxx-opt/osh
683 ninja $bin
684 run-verbose $bin benchmarks/compute/fib.sh 10000
685
686 # time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 _bin/cxx-opt/osh benchmarks/compute/fib.sh 10000
687}
688
689while-loop() {
690 local i=0
691 while test $i -lt 10000; do
692 if ((i % 1000 == 0)) ; then
693 echo $i
694 fi
695 i=$((i + 1))
696 continue # BUG: skipped GC point
697 done
698}
699
700for-loop() {
701 for i in $(seq 10000); do
702 if ((i % 1000 == 0)) ; then
703 echo $i
704 fi
705 continue
706 done
707}
708
709recurse() {
710 local n=${1:-3000}
711
712 if ((n % 100 == 0)) ; then
713 echo $n
714 fi
715
716 if test $n = 0; then
717 return
718 fi
719
720 recurse $((n - 1))
721}
722
723test-loops() {
724 ### Regression for leak
725
726 local bin=_bin/cxx-opt/osh
727 ninja $bin
728
729 run-verbose $bin $0 recurse
730 echo
731
732 run-verbose $bin $0 while-loop
733 echo
734
735 run-verbose $bin $0 for-loop
736}
737
738expand-loop() {
739 local n=$1
740
741 local bin=_bin/cxx-opt/osh
742 ninja $bin
743
744 set -x
745 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
746 $bin -c "for i in {1..$n}; do echo \$i; done > /dev/null"
747 set +x
748}
749
750test-brace-exp() {
751 expand-loop 330000
752 expand-loop 340000
753}
754
755"$@"