OILS / benchmarks / gc.sh View on Github | oils.pub

738 lines, 413 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# benchmarks/gc.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
11
12source benchmarks/common.sh # benchmark-html-head
13source benchmarks/cachegrind.sh # with-cachegrind
14source build/dev-shell.sh # R_LIBS_USER
15source test/tsv-lib.sh
16
17readonly BASE_DIR=_tmp/gc
18
19# duplicated in benchmarks/gc-cachegrind.sh
20readonly BASE_DIR_CACHEGRIND=_tmp/gc-cachegrind
21
22# See benchmarks/gperftools.sh. I think the Ubuntu package is very old
23
24download-tcmalloc() {
25 wget --directory-prefix _deps \
26 https://github.com/gperftools/gperftools/releases/download/gperftools-2.10/gperftools-2.10.tar.gz
27
28 # Then ./configure; make; sudo make install
29 # installs in /usr/local/lib
30
31 # Note: there's a warning about libunwind -- maybe install that first. Does
32 # it only apply to CPU profiles?
33}
34
35debug-tcmalloc() {
36 touch mycpp/marksweep_heap.cc
37
38 # No evidence of difference
39 for bin in _bin/cxx-{opt,opt+tcmalloc}/osh; do
40 echo $bin
41 ninja $bin
42
43 ldd $bin
44 echo
45
46 ls -l $bin
47 echo
48
49 # Check what we're linking against
50 nm $bin | egrep -i 'malloc|calloc'
51 #wc -l
52 echo
53 done
54}
55
56install-m32() {
57 # needed to compile with -m32
58 sudo apt-get install gcc-multilib g++-multilib
59}
60
61max-rss() {
62 # %e is real time
63 /usr/bin/time --format '%e %M' -- "$@"
64}
65
66compare-m32() {
67 for bin in _bin/cxx-opt{,32}/osh; do
68 echo $bin
69 ninja $bin
70
71 ldd $bin
72 echo
73
74 file $bin
75 echo
76
77 ls -l $bin
78 echo
79
80 # 141136 KiB vs. 110924 KiB. Significant savings, but it's slower.
81 max-rss $bin --ast-format none -n benchmarks/testdata/configure-coreutils
82
83 done
84}
85
86print-tasks() {
87 local mycpp_nosouffle=${1:-}
88
89 local -a workloads=(
90 parse.configure-coreutils
91 parse.configure-cpython
92 parse.abuild
93 ex.bashcomp-parse-help # only runs with bash
94 ex.abuild-print-help # bash / dash / zsh
95 ex.compute-fib # bash / dash / zsh
96 )
97
98 local -a shells=(
99 "bash$TAB-"
100 "dash$TAB-"
101 "zsh$TAB-"
102
103 "_bin/cxx-opt+bumpleak/osh${TAB}mut"
104 "_bin/cxx-opt+bumproot/osh${TAB}mut"
105
106 "_bin/cxx-opt+bumpsmall/osh${TAB}mut+alloc"
107 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc"
108 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc+free+gc"
109
110 # these have trivial GC stats
111 "_bin/cxx-opt/osh${TAB}mut+alloc"
112 "_bin/cxx-opt/osh${TAB}mut+alloc+free"
113 # good GC stats
114 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc"
115 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc+exit"
116 )
117
118 if test -n "$mycpp_nosouffle"; then
119 shells+=(
120 "_bin/cxx-opt/mycpp-nosouffle/osh${TAB}mut+alloc"
121 "_bin/cxx-opt/mycpp-nosouffle/osh${TAB}mut+alloc+free"
122 "_bin/cxx-opt/mycpp-nosouffle/osh${TAB}mut+alloc+free+gc"
123 "_bin/cxx-opt/mycpp-nosouffle/osh${TAB}mut+alloc+free+gc+exit"
124 )
125 fi
126
127 if test -n "${TCMALLOC:-}"; then
128 shells+=(
129 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc"
130 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc+free"
131 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc+free+gc"
132 )
133 fi
134
135 local id=0
136
137 for workload in "${workloads[@]}"; do
138 for shell in "${shells[@]}"; do
139 local row_part="$workload${TAB}$shell"
140
141 # Skip these rows
142 case $row_part in
143 "ex.bashcomp-parse-help${TAB}dash"*)
144 continue
145 ;;
146 "ex.bashcomp-parse-help${TAB}zsh"*)
147 continue
148 ;;
149 esac
150
151 local join_id="gc-$id"
152 local row="$join_id${TAB}$row_part"
153 echo "$row"
154
155 id=$((id + 1))
156
157 done
158
159 # Run a quick 10 tasks
160 if test -n "${QUICKLY:-}" && test $id -gt 10; then
161 break
162 fi
163 done
164}
165
166print-cachegrind-tasks() {
167 local mycpp_nosouffle=${1:-}
168
169 local -a workloads=(
170 # coreutils is on osh-parser
171 #parse.configure-coreutils
172
173 #parse.configure-cpython
174
175 # Faster tasks, like benchmarks/uftrace, which is instrumented
176 parse.abuild
177 ex.compute-fib
178 )
179
180 local -a shells=(
181 "bash${TAB}-"
182 "_bin/cxx-opt+bumpleak/osh${TAB}mut"
183 "_bin/cxx-opt+bumproot/osh${TAB}mut"
184
185 "_bin/cxx-opt+bumpsmall/osh${TAB}mut+alloc"
186
187 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc+free+gc"
188
189 "_bin/cxx-opt/osh${TAB}mut+alloc"
190 "_bin/cxx-opt/osh${TAB}mut+alloc+free"
191 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc"
192 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc+exit"
193 )
194
195 if test -n "$mycpp_nosouffle"; then
196 shells+=(
197 "_bin/cxx-opt/mycpp-nosouffle/osh${TAB}mut+alloc+free+gc"
198 )
199 fi
200
201 local id=0
202 for workload in "${workloads[@]}"; do
203 for shell in "${shells[@]}"; do
204 local row_part="$workload${TAB}$shell"
205
206 local join_id="cachegrind-$id"
207 local row="$join_id${TAB}$row_part"
208 echo "$row"
209
210 id=$((id + 1))
211 done
212 done
213 #print-tasks | egrep 'configure-coreutils' | egrep osh
214}
215
216
217readonly BIG_THRESHOLD=$(( 1 * 1000 * 1000 * 1000 )) # 1 B
218
219run-tasks() {
220 local tsv_out=$1
221 local mode=${2:-time}
222
223 while read -r join_id task sh_path shell_runtime_opts; do
224
225 # Parse different files
226 case $task in
227 parse.configure-coreutils)
228 data_file='benchmarks/testdata/configure-coreutils'
229 ;;
230 parse.configure-cpython)
231 data_file='Python-2.7.13/configure'
232 ;;
233 parse.abuild)
234 data_file='benchmarks/testdata/abuild'
235 ;;
236 esac
237
238 # Construct argv for each task
239 local -a argv
240 case $task in
241 parse.*)
242 argv=( -n $data_file )
243
244 case $sh_path in
245 _bin/*/osh)
246 argv=( --ast-format none "${argv[@]}" )
247 ;;
248 esac
249 ;;
250
251 ex.bashcomp-parse-help)
252 argv=( benchmarks/parse-help/pure-excerpt.sh parse_help_file
253 benchmarks/parse-help/clang.txt )
254 ;;
255
256 ex.abuild-print-help)
257 argv=( testdata/osh-runtime/abuild -h )
258 ;;
259
260 ex.compute-fib)
261 # fewer iterations when instrumented
262 local iters
263 if test $mode = time; then
264 iters=100
265 else
266 iters=10
267 fi
268
269 argv=( benchmarks/compute/fib.sh $iters 44 )
270 ;;
271
272 *)
273 die "Invalid task $task"
274 ;;
275 esac
276
277 echo $join_id $task $sh_path $shell_runtime_opts
278
279 argv=( $sh_path "${argv[@]}" )
280 #echo + "${argv[@]}"
281 #set -x
282
283 if test $mode = cachegrind; then
284 # Add prefix
285 argv=( $0 with-cachegrind $BASE_DIR_CACHEGRIND/raw/$join_id.txt "${argv[@]}" )
286 fi
287
288 # Wrap in a command that writes one row of a TSV
289 # Note: for cachegrind, we need the join ID, but the --rusage is meaningless
290 local -a instrumented=(
291 time-tsv -o $tsv_out --append
292 --rusage
293 --field "$join_id" --field "$task" --field "$sh_path"
294 --field "$shell_runtime_opts"
295 -- "${argv[@]}"
296 )
297
298 # Run with the right environment variables
299
300 case $shell_runtime_opts in
301 -)
302 "${instrumented[@]}" > /dev/null
303 ;;
304 mut)
305 OILS_GC_STATS=1 \
306 "${instrumented[@]}" > /dev/null
307 ;;
308 mut+alloc)
309 # disable GC with big threshold
310 OILS_GC_STATS=1 OILS_GC_THRESHOLD=$BIG_THRESHOLD \
311 "${instrumented[@]}" > /dev/null
312 ;;
313 mut+alloc+free)
314 # do a single GC on exit
315 OILS_GC_STATS=1 OILS_GC_THRESHOLD=$BIG_THRESHOLD OILS_GC_ON_EXIT=1 \
316 "${instrumented[@]}" > /dev/null
317 ;;
318 mut+alloc+free+gc)
319 # Default configuration
320 #
321 # Save the GC stats here. None of the other runtime options are that
322 # interesting.
323
324 if test $mode = 'time' && test $sh_path != _bin/cxx-opt+nopool/osh; then
325 OILS_GC_STATS_FD=99 \
326 "${instrumented[@]}" > /dev/null 99>$BASE_DIR/raw/$join_id.txt
327 else
328 "${instrumented[@]}" > /dev/null
329 fi
330 ;;
331 mut+alloc+free+gc+exit)
332 # also GC on exit
333 OILS_GC_STATS=1 OILS_GC_ON_EXIT=1 \
334 "${instrumented[@]}" > /dev/null
335 ;;
336
337 *)
338 die "Invalid shell runtime opts $shell_runtime_opts"
339 ;;
340 esac
341
342 done
343
344 # TODO: OILS_GC_STATS_FD and tsv_column_from_files.py
345}
346
347fd-demo() {
348 local out=_tmp/gc/demo.txt
349
350 local bin=_bin/cxx-dbg/oils-for-unix
351 ninja $bin
352
353 # Hm you can't do $fd>out.txt, but that's OK
354 local fd=99
355
356 OILS_GC_STATS_FD=$fd 99>$out \
357 $bin --ast-format none -n benchmarks/testdata/configure
358
359 ls -l $out
360 cat $out
361}
362
363more-variants() {
364 # TODO: could revive this
365
366 case $compare_more in
367 (*m32*)
368 # Surprisingly, -m32 is SLOWER, even though it allocates less.
369 # My guess is because less work is going into maintaining this code path in
370 # GCC.
371
372 # 223 ms
373 # 61.9 MB bytes allocated
374 local bin=_bin/cxx-opt32/oils-for-unix
375 OILS_GC_THRESHOLD=$big_threshold \
376 run-osh $tsv_out $bin 'm32 mutator+malloc' $file
377
378 # 280 ms
379 OILS_GC_STATS=1 \
380 run-osh $tsv_out $bin 'm32 mutator+malloc+free+gc' $file
381 ;;
382 esac
383
384 # Show log of GC
385 case $compare_more in
386 (*gcverbose*)
387 local bin=_bin/cxx-gcverbose/oils-for-unix
388 # 280 ms
389 OILS_GC_STATS=1 OILS_GC_ON_EXIT=1 \
390 run-osh $tsv_out $bin 'gcverbose mutator+malloc+free+gc' $file
391 ;;
392 esac
393
394 if command -v pretty-tsv; then
395 pretty-tsv $tsv_out
396 fi
397}
398
399build-binaries() {
400 soil/cpp-tarball.sh build-like-ninja \
401 opt{,+bumpleak,+bumproot,+bumpsmall,+nopool}
402
403 OILS_TRANSLATOR=mycpp-nosouffle soil/cpp-tarball.sh build-like-ninja opt
404}
405
406measure-all() {
407 local tsv_out=${1:-$BASE_DIR/raw/times.tsv}
408 local mycpp_nosouffle=${2:-}
409
410 build-binaries
411
412 mkdir -p $(dirname $tsv_out)
413
414 # Make the header
415 time-tsv -o $tsv_out --print-header \
416 --rusage --field join_id --field task --field sh_path --field shell_runtime_opts
417
418 # Pass through args, which may include mycpp-nosouffle
419 time print-tasks "$mycpp_nosouffle" | run-tasks $tsv_out
420
421 if command -v pretty-tsv; then
422 pretty-tsv $tsv_out
423 fi
424}
425
426measure-cachegrind() {
427 local tsv_out=${1:-$BASE_DIR_CACHEGRIND/raw/times.tsv}
428 local mycpp_nosouffle=${2:-T}
429
430 build-binaries
431
432 mkdir -p $(dirname $tsv_out)
433
434 # Make the header
435 time-tsv -o $tsv_out --print-header \
436 --rusage --field join_id --field task --field sh_path --field shell_runtime_opts
437
438 print-cachegrind-tasks "$mycpp_nosouffle" | run-tasks $tsv_out cachegrind
439
440 # TODO: join cachegrind columns
441
442 if command -v pretty-tsv; then
443 pretty-tsv $tsv_out
444 fi
445}
446
447print-report() {
448 local in_dir=$1
449
450 benchmark-html-head 'Memory Management Overhead'
451
452 cat <<EOF
453 <body class="width60">
454 <p id="home-link">
455 <a href="/">oils.pub</a>
456 </p>
457EOF
458
459 cmark << 'EOF'
460## Memory Management Overhead
461
462Source code: [oils/benchmarks/gc.sh](https://github.com/oils-for-unix/oils/tree/master/benchmarks/gc.sh)
463EOF
464
465 cmark << 'EOF'
466### GC Stats
467
468EOF
469
470 tsv2html $in_dir/gc_stats.tsv
471
472 cmark << 'EOF'
473
474- Underlying data: [stage2/gc_stats.tsv](stage2/gc_stats.tsv)
475- More columns: [stage1/gc_stats.tsv](stage1/gc_stats.tsv)
476
477### Resource Usage
478
479#### parse.configure-cpython
480
481EOF
482
483 tsv2html $in_dir/parse.configure-cpython.tsv
484
485 cmark << 'EOF'
486#### parse.configure-coreutils
487
488Parsing the autoconf-generated `configure` script from GNU coreutils.
489
490Note that unlike other shells, `osh -n` retains all nodes on purpose. (See the
491[parser benchmark](../osh-parser/index.html)).
492
493EOF
494
495 tsv2html $in_dir/parse.configure-coreutils.tsv
496
497 cmark <<'EOF'
498#### parse.abuild
499
500Parsing `abuild` from Alpine Linux.
501EOF
502
503 tsv2html $in_dir/parse.abuild.tsv
504
505 cmark <<'EOF'
506#### ex.compute-fib
507
508A synthetic benchmark for POSIX shell arithmetic.
509EOF
510
511 tsv2html $in_dir/ex.compute-fib.tsv
512
513 cmark <<'EOF'
514#### ex.bashcomp-parse-help
515
516A realistic `bash-completion` workload.
517EOF
518
519 tsv2html $in_dir/ex.bashcomp-parse-help.tsv
520
521 cmark <<'EOF'
522#### ex.abuild-print-help
523
524Running `abuild -h` from Alpine Linux.
525
526EOF
527
528 tsv2html $in_dir/ex.abuild-print-help.tsv
529
530 cmark << 'EOF'
531- Underlying data: [stage2/times.tsv](stage2/times.tsv)
532EOF
533
534 cat <<EOF
535
536 </body>
537</html>
538EOF
539}
540
541make-report() {
542 mkdir -p $BASE_DIR/{stage1,stage2}
543
544 # Concatenate tiny files
545 benchmarks/gc_stats_to_tsv.py $BASE_DIR/raw/gc-*.txt \
546 > $BASE_DIR/stage1/gc_stats.tsv
547
548 # Make TSV files
549 benchmarks/report.R gc $BASE_DIR $BASE_DIR/stage2
550
551 # Make HTML
552 benchmarks/report.sh stage3 $BASE_DIR
553}
554
555soil-run() {
556 ### Run in soil/benchmarks
557
558 measure-all '' mycpp-nosouffle
559
560 make-report
561}
562
563run-for-release() {
564 # TODO: turn on nosouffle?
565 measure-all ''
566
567 make-report
568}
569
570#
571# Misc Tests
572#
573
574gc-parse-smoke() {
575 local variant=${1:-opt}
576 local file=${2:-configure}
577
578 local bin=_bin/cxx-$variant/osh
579 ninja $bin
580
581 # OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 \
582 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
583 $bin --ast-format none -n $file
584
585 # No leaks
586 # OILS_GC_STATS=1 OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 $bin -n -c '('
587}
588
589gc-parse-big() {
590 local variant=${1:-opt}
591
592 gc-parse-smoke $variant benchmarks/testdata/configure-coreutils
593}
594
595gc-run-smoke() {
596 local variant=${1:-opt}
597
598 local bin=_bin/cxx-$variant/oils-for-unix
599 ninja $bin
600
601 # expose a bug with printf
602 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=500 OILS_GC_ON_EXIT=1 \
603 $bin -c 'for i in $(seq 100); do printf "%s\\n" "-- $i"; done'
604}
605
606gc-run-oil() {
607 ### Run some scripts from the repo
608
609 local variant=${1:-opt}
610
611 local bin=_bin/cxx-$variant/oils-for-unix
612 ninja $bin
613
614 local i=0
615 for script in */*.sh; do
616 case $script in
617 (build/clean.sh|build/common.sh|build/dev.sh)
618 # Top level does something!
619 echo "=== SKIP $script"
620 continue
621 ;;
622 esac
623
624 echo
625 echo "=== ($i) $script"
626
627 # Just run the top level, which (hopefully) does nothing
628 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 \
629 $bin $script
630
631 i=$((i + 1))
632 if test $i -gt 60; then
633 break
634 fi
635 done
636}
637
638gc-run-big() {
639 local variant=${1:-opt}
640
641 local target=_bin/cxx-$variant/oils-for-unix
642 ninja $target
643
644 local osh=$REPO_ROOT/$target
645
646 local dir=_tmp/gc-run-big
647 rm -r -f -v $dir
648 mkdir -v -p $dir
649
650 pushd $dir
651 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=100000 OILS_GC_ON_EXIT=1 \
652 $osh ../../Python-2.7.13/configure
653 popd
654}
655
656run-verbose() {
657 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
658 /usr/bin/time --format '*** MAX RSS KiB = %M' -- \
659 "$@"
660}
661
662# This hit the 24-bit object ID limitation in 2.5 seconds
663# Should be able to run indefinitely.
664run-for-a-long-time() {
665 local bin=_bin/cxx-opt/osh
666 ninja $bin
667 run-verbose $bin benchmarks/compute/fib.sh 10000
668
669 # time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 _bin/cxx-opt/osh benchmarks/compute/fib.sh 10000
670}
671
672while-loop() {
673 local i=0
674 while test $i -lt 10000; do
675 if ((i % 1000 == 0)) ; then
676 echo $i
677 fi
678 i=$((i + 1))
679 continue # BUG: skipped GC point
680 done
681}
682
683for-loop() {
684 for i in $(seq 10000); do
685 if ((i % 1000 == 0)) ; then
686 echo $i
687 fi
688 continue
689 done
690}
691
692recurse() {
693 local n=${1:-3000}
694
695 if ((n % 100 == 0)) ; then
696 echo $n
697 fi
698
699 if test $n = 0; then
700 return
701 fi
702
703 recurse $((n - 1))
704}
705
706test-loops() {
707 ### Regression for leak
708
709 local bin=_bin/cxx-opt/osh
710 ninja $bin
711
712 run-verbose $bin $0 recurse
713 echo
714
715 run-verbose $bin $0 while-loop
716 echo
717
718 run-verbose $bin $0 for-loop
719}
720
721expand-loop() {
722 local n=$1
723
724 local bin=_bin/cxx-opt/osh
725 ninja $bin
726
727 set -x
728 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
729 $bin -c "for i in {1..$n}; do echo \$i; done > /dev/null"
730 set +x
731}
732
733test-brace-exp() {
734 expand-loop 330000
735 expand-loop 340000
736}
737
738"$@"