OILS / benchmarks / gc.sh View on Github | oilshell.org

753 lines, 420 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# benchmarks/gc.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
11
12source benchmarks/common.sh # benchmark-html-head
13source benchmarks/cachegrind.sh # with-cachegrind
14source build/dev-shell.sh # R_LIBS_USER
15source test/tsv-lib.sh
16
17readonly BASE_DIR=_tmp/gc
18
19# duplicated in benchmarks/gc-cachegrind.sh
20readonly BASE_DIR_CACHEGRIND=_tmp/gc-cachegrind
21
22# See benchmarks/gperftools.sh. I think the Ubuntu package is very old
23
24download-tcmalloc() {
25 # TODO: move this to ../oil_DEPS ?
26 wget --directory _deps \
27 https://github.com/gperftools/gperftools/releases/download/gperftools-2.10/gperftools-2.10.tar.gz
28
29 # Then ./configure; make; sudo make install
30 # installs in /usr/local/lib
31
32 # Note: there's a warning about libunwind -- maybe install that first. Does
33 # it only apply to CPU profiles?
34}
35
36debug-tcmalloc() {
37 touch mycpp/marksweep_heap.cc
38
39 # No evidence of difference
40 for bin in _bin/cxx-{opt,opt+tcmalloc}/osh; do
41 echo $bin
42 ninja $bin
43
44 ldd $bin
45 echo
46
47 ls -l $bin
48 echo
49
50 # Check what we're linking against
51 nm $bin | egrep -i 'malloc|calloc'
52 #wc -l
53 echo
54 done
55}
56
57install-m32() {
58 # needed to compile with -m32
59 sudo apt-get install gcc-multilib g++-multilib
60}
61
62max-rss() {
63 # %e is real time
64 /usr/bin/time --format '%e %M' -- "$@"
65}
66
67compare-m32() {
68 for bin in _bin/cxx-opt{,32}/osh; do
69 echo $bin
70 ninja $bin
71
72 ldd $bin
73 echo
74
75 file $bin
76 echo
77
78 ls -l $bin
79 echo
80
81 # 141136 KiB vs. 110924 KiB. Significant savings, but it's slower.
82 max-rss $bin --ast-format none -n benchmarks/testdata/configure-coreutils
83
84 done
85}
86
87banner() {
88 echo -----
89 echo "$@"
90}
91
92print-tasks() {
93 local mycpp_souffle=${1:-}
94
95 local -a workloads=(
96 parse.configure-coreutils
97 parse.configure-cpython
98 parse.abuild
99 ex.bashcomp-parse-help # only runs with bash
100 ex.abuild-print-help # bash / dash / zsh
101 ex.compute-fib # bash / dash / zsh
102 )
103
104 local -a shells=(
105 "bash$TAB-"
106 "dash$TAB-"
107 "zsh$TAB-"
108
109 "_bin/cxx-opt+bumpleak/osh${TAB}mut"
110 "_bin/cxx-opt+bumproot/osh${TAB}mut"
111
112 "_bin/cxx-opt+bumpsmall/osh${TAB}mut+alloc"
113 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc"
114 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc+free+gc"
115
116 # these have trivial GC stats
117 "_bin/cxx-opt/osh${TAB}mut+alloc"
118 "_bin/cxx-opt/osh${TAB}mut+alloc+free"
119 # good GC stats
120 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc"
121 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc+exit"
122 )
123
124 if test -n "$mycpp_souffle"; then
125 shells+=(
126 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc"
127 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free"
128 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free+gc"
129 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free+gc+exit"
130 )
131 fi
132
133 if test -n "${TCMALLOC:-}"; then
134 shells+=(
135 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc"
136 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc+free"
137 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc+free+gc"
138 )
139 fi
140
141 local id=0
142
143 for workload in "${workloads[@]}"; do
144 for shell in "${shells[@]}"; do
145 local row_part="$workload${TAB}$shell"
146
147 # Skip these rows
148 case $row_part in
149 "ex.bashcomp-parse-help${TAB}dash"*)
150 continue
151 ;;
152 "ex.bashcomp-parse-help${TAB}zsh"*)
153 continue
154 ;;
155 esac
156
157 local join_id="gc-$id"
158 local row="$join_id${TAB}$row_part"
159 echo "$row"
160
161 id=$((id + 1))
162
163 done
164
165 # Run a quick 10 tasks
166 if test -n "${QUICKLY:-}" && test $id -gt 10; then
167 break
168 fi
169 done
170}
171
172print-cachegrind-tasks() {
173 local -a workloads=(
174 # coreutils is on osh-parser
175 #parse.configure-coreutils
176
177 #parse.configure-cpython
178
179 # Faster tasks, like benchmarks/uftrace, which is instrumented
180 parse.abuild
181 ex.compute-fib
182 )
183
184 local -a shells=(
185 "bash${TAB}-"
186 "_bin/cxx-opt+bumpleak/osh${TAB}mut"
187 "_bin/cxx-opt+bumproot/osh${TAB}mut"
188
189 "_bin/cxx-opt+bumpsmall/osh${TAB}mut+alloc"
190 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc"
191 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc+free+gc"
192
193 "_bin/cxx-opt/osh${TAB}mut+alloc"
194 "_bin/cxx-opt/osh${TAB}mut+alloc+free"
195 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc"
196 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc+exit"
197
198 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc"
199 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free"
200 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free+gc"
201 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free+gc+exit"
202 )
203
204 local id=0
205 for workload in "${workloads[@]}"; do
206 for shell in "${shells[@]}"; do
207 local row_part="$workload${TAB}$shell"
208
209 local join_id="cachegrind-$id"
210 local row="$join_id${TAB}$row_part"
211 echo "$row"
212
213 id=$((id + 1))
214 done
215 done
216 #print-tasks | egrep 'configure-coreutils' | egrep osh
217}
218
219
220readonly BIG_THRESHOLD=$(( 1 * 1000 * 1000 * 1000 )) # 1 B
221
222run-tasks() {
223 local tsv_out=$1
224 local mode=${2:-time}
225
226 while read -r join_id task sh_path shell_runtime_opts; do
227
228 # Parse different files
229 case $task in
230 parse.configure-coreutils)
231 data_file='benchmarks/testdata/configure-coreutils'
232 ;;
233 parse.configure-cpython)
234 data_file='Python-2.7.13/configure'
235 ;;
236 parse.abuild)
237 data_file='benchmarks/testdata/abuild'
238 ;;
239 esac
240
241 # Construct argv for each task
242 local -a argv
243 case $task in
244 parse.*)
245 argv=( -n $data_file )
246
247 case $sh_path in
248 _bin/*/osh)
249 argv=( --ast-format none "${argv[@]}" )
250 ;;
251 esac
252 ;;
253
254 ex.bashcomp-parse-help)
255 argv=( benchmarks/parse-help/pure-excerpt.sh parse_help_file
256 benchmarks/parse-help/clang.txt )
257 ;;
258
259 ex.abuild-print-help)
260 argv=( testdata/osh-runtime/abuild -h )
261 ;;
262
263 ex.compute-fib)
264 # fewer iterations when instrumented
265 local iters
266 if test $mode = time; then
267 iters=100
268 else
269 iters=10
270 fi
271
272 argv=( benchmarks/compute/fib.sh $iters 44 )
273 ;;
274
275 *)
276 die "Invalid task $task"
277 ;;
278 esac
279
280 echo $join_id $task $sh_path $shell_runtime_opts
281
282 argv=( $sh_path "${argv[@]}" )
283 #echo + "${argv[@]}"
284 #set -x
285
286 if test $mode = cachegrind; then
287 # Add prefix
288 argv=( $0 with-cachegrind $BASE_DIR_CACHEGRIND/raw/$join_id.txt "${argv[@]}" )
289 fi
290
291 # Wrap in a command that writes one row of a TSV
292 # Note: for cachegrind, we need the join ID, but the --rusage is meaningless
293 local -a instrumented=(
294 time-tsv -o $tsv_out --append
295 --rusage
296 --field "$join_id" --field "$task" --field "$sh_path"
297 --field "$shell_runtime_opts"
298 -- "${argv[@]}"
299 )
300
301 # Run with the right environment variables
302
303 case $shell_runtime_opts in
304 -)
305 "${instrumented[@]}" > /dev/null
306 ;;
307 mut)
308 OILS_GC_STATS=1 \
309 "${instrumented[@]}" > /dev/null
310 ;;
311 mut+alloc)
312 # disable GC with big threshold
313 OILS_GC_STATS=1 OILS_GC_THRESHOLD=$BIG_THRESHOLD \
314 "${instrumented[@]}" > /dev/null
315 ;;
316 mut+alloc+free)
317 # do a single GC on exit
318 OILS_GC_STATS=1 OILS_GC_THRESHOLD=$BIG_THRESHOLD OILS_GC_ON_EXIT=1 \
319 "${instrumented[@]}" > /dev/null
320 ;;
321 mut+alloc+free+gc)
322 # Default configuration
323 #
324 # Save the GC stats here. None of the other runtime options are that
325 # interesting.
326
327 if test $mode = 'time' && test $sh_path != _bin/cxx-opt+nopool/osh; then
328 OILS_GC_STATS_FD=99 \
329 "${instrumented[@]}" > /dev/null 99>$BASE_DIR/raw/$join_id.txt
330 else
331 "${instrumented[@]}" > /dev/null
332 fi
333 ;;
334 mut+alloc+free+gc+exit)
335 # also GC on exit
336 OILS_GC_STATS=1 OILS_GC_ON_EXIT=1 \
337 "${instrumented[@]}" > /dev/null
338 ;;
339
340 *)
341 die "Invalid shell runtime opts $shell_runtime_opts"
342 ;;
343 esac
344
345 done
346
347 # TODO: OILS_GC_STATS_FD and tsv_column_from_files.py
348}
349
350fd-demo() {
351 local out=_tmp/gc/demo.txt
352
353 local bin=_bin/cxx-dbg/oils-for-unix
354 ninja $bin
355
356 # Hm you can't do $fd>out.txt, but that's OK
357 local fd=99
358
359 OILS_GC_STATS_FD=$fd 99>$out \
360 $bin --ast-format none -n benchmarks/testdata/configure
361
362 ls -l $out
363 cat $out
364}
365
366more-variants() {
367 # TODO: could revive this
368
369 case $compare_more in
370 (*m32*)
371 # Surprisingly, -m32 is SLOWER, even though it allocates less.
372 # My guess is because less work is going into maintaining this code path in
373 # GCC.
374
375 # 223 ms
376 # 61.9 MB bytes allocated
377 local bin=_bin/cxx-opt32/oils-for-unix
378 OILS_GC_THRESHOLD=$big_threshold \
379 run-osh $tsv_out $bin 'm32 mutator+malloc' $file
380
381 # 280 ms
382 OILS_GC_STATS=1 \
383 run-osh $tsv_out $bin 'm32 mutator+malloc+free+gc' $file
384 ;;
385 esac
386
387 # Show log of GC
388 case $compare_more in
389 (*gcverbose*)
390 local bin=_bin/cxx-gcverbose/oils-for-unix
391 # 280 ms
392 OILS_GC_STATS=1 OILS_GC_ON_EXIT=1 \
393 run-osh $tsv_out $bin 'gcverbose mutator+malloc+free+gc' $file
394 ;;
395 esac
396
397 if command -v pretty-tsv; then
398 pretty-tsv $tsv_out
399 fi
400}
401
402build-binaries() {
403 if true; then
404
405 soil/cpp-tarball.sh build-like-ninja \
406 opt{,+bumpleak,+bumproot,+bumpsmall,+nopool}
407
408 OILS_TRANSLATOR=mycpp-souffle soil/cpp-tarball.sh build-like-ninja opt
409
410 else
411
412 # Old Ninja build
413 local -a bin=( _bin/cxx-opt{,+bumpleak,+bumproot,+bumpsmall,+nopool}/osh )
414 bin+=( _bin/cxx-opt/mycpp-souffle/osh )
415
416 if test -n "${TCMALLOC:-}"; then
417 bin+=( _bin/cxx-opt+tcmalloc/osh )
418 fi
419 ninja "${bin[@]}"
420 fi
421}
422
423measure-all() {
424 local tsv_out=${1:-$BASE_DIR/raw/times.tsv}
425 local mycpp_souffle=${2:-}
426
427 build-binaries
428
429 mkdir -p $(dirname $tsv_out)
430
431 # Make the header
432 time-tsv -o $tsv_out --print-header \
433 --rusage --field join_id --field task --field sh_path --field shell_runtime_opts
434
435 # Pass through args, which may include mycpp-souffle
436 time print-tasks "$mycpp_souffle" | run-tasks $tsv_out
437
438 if command -v pretty-tsv; then
439 pretty-tsv $tsv_out
440 fi
441}
442
443measure-cachegrind() {
444 build-binaries
445
446 local tsv_out=${1:-$BASE_DIR_CACHEGRIND/raw/times.tsv}
447
448 mkdir -p $(dirname $tsv_out)
449
450 # Make the header
451 time-tsv -o $tsv_out --print-header \
452 --rusage --field join_id --field task --field sh_path --field shell_runtime_opts
453
454 print-cachegrind-tasks | run-tasks $tsv_out cachegrind
455
456 # TODO: join cachegrind columns
457
458 if command -v pretty-tsv; then
459 pretty-tsv $tsv_out
460 fi
461}
462
463print-report() {
464 local in_dir=$1
465
466 benchmark-html-head 'Memory Management Overhead'
467
468 cat <<EOF
469 <body class="width60">
470 <p id="home-link">
471 <a href="/">oilshell.org</a>
472 </p>
473EOF
474
475 cmark << 'EOF'
476## Memory Management Overhead
477
478Source code: [oil/benchmarks/gc.sh](https://github.com/oilshell/oil/tree/master/benchmarks/gc.sh)
479EOF
480
481 cmark << 'EOF'
482### GC Stats
483
484EOF
485
486 tsv2html $in_dir/gc_stats.tsv
487
488 cmark << 'EOF'
489
490- Underlying data: [stage2/gc_stats.tsv](stage2/gc_stats.tsv)
491- More columns: [stage1/gc_stats.tsv](stage1/gc_stats.tsv)
492
493### Resource Usage
494
495#### parse.configure-cpython
496
497EOF
498
499 tsv2html $in_dir/parse.configure-cpython.tsv
500
501 cmark << 'EOF'
502#### parse.configure-coreutils
503
504Parsing the autoconf-generated `configure` script from GNU coreutils.
505
506Note that unlike other shells, `osh -n` retains all nodes on purpose. (See the
507[parser benchmark](../osh-parser/index.html)).
508
509EOF
510
511 tsv2html $in_dir/parse.configure-coreutils.tsv
512
513 cmark <<'EOF'
514#### parse.abuild
515
516Parsing `abuild` from Alpine Linux.
517EOF
518
519 tsv2html $in_dir/parse.abuild.tsv
520
521 cmark <<'EOF'
522#### ex.compute-fib
523
524A synthetic benchmark for POSIX shell arithmetic.
525EOF
526
527 tsv2html $in_dir/ex.compute-fib.tsv
528
529 cmark <<'EOF'
530#### ex.bashcomp-parse-help
531
532A realistic `bash-completion` workload.
533EOF
534
535 tsv2html $in_dir/ex.bashcomp-parse-help.tsv
536
537 cmark <<'EOF'
538#### ex.abuild-print-help
539
540Running `abuild -h` from Alpine Linux.
541
542EOF
543
544 tsv2html $in_dir/ex.abuild-print-help.tsv
545
546 cmark << 'EOF'
547- Underlying data: [stage2/times.tsv](stage2/times.tsv)
548EOF
549
550 cat <<EOF
551
552 </body>
553</html>
554EOF
555}
556
557make-report() {
558 mkdir -p $BASE_DIR/{stage1,stage2}
559
560 # Concatenate tiny files
561 benchmarks/gc_stats_to_tsv.py $BASE_DIR/raw/gc-*.txt \
562 > $BASE_DIR/stage1/gc_stats.tsv
563
564 # Make TSV files
565 benchmarks/report.R gc $BASE_DIR $BASE_DIR/stage2
566
567 # Make HTML
568 benchmarks/report.sh stage3 $BASE_DIR
569}
570
571soil-run() {
572 ### Run in soil/benchmarks
573
574 measure-all '' mycpp-souffle
575
576 make-report
577}
578
579run-for-release() {
580 measure-all ''
581
582 make-report
583}
584
585#
586# Misc Tests
587#
588
589gc-parse-smoke() {
590 local variant=${1:-opt}
591 local file=${2:-configure}
592
593 local bin=_bin/cxx-$variant/osh
594 ninja $bin
595
596 # OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 \
597 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
598 $bin --ast-format none -n $file
599
600 # No leaks
601 # OILS_GC_STATS=1 OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 $bin -n -c '('
602}
603
604gc-parse-big() {
605 local variant=${1:-opt}
606
607 gc-parse-smoke $variant benchmarks/testdata/configure-coreutils
608}
609
610gc-run-smoke() {
611 local variant=${1:-opt}
612
613 local bin=_bin/cxx-$variant/oils-for-unix
614 ninja $bin
615
616 # expose a bug with printf
617 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=500 OILS_GC_ON_EXIT=1 \
618 $bin -c 'for i in $(seq 100); do printf "%s\\n" "-- $i"; done'
619}
620
621gc-run-oil() {
622 ### Run some scripts from the repo
623
624 local variant=${1:-opt}
625
626 local bin=_bin/cxx-$variant/oils-for-unix
627 ninja $bin
628
629 local i=0
630 for script in */*.sh; do
631 case $script in
632 (build/clean.sh|build/common.sh|build/dev.sh)
633 # Top level does something!
634 echo "=== SKIP $script"
635 continue
636 ;;
637 esac
638
639 echo
640 echo "=== ($i) $script"
641
642 # Just run the top level, which (hopefully) does nothing
643 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 \
644 $bin $script
645
646 i=$((i + 1))
647 if test $i -gt 60; then
648 break
649 fi
650 done
651}
652
653gc-run-big() {
654 local variant=${1:-opt}
655
656 local target=_bin/cxx-$variant/oils-for-unix
657 ninja $target
658
659 local osh=$REPO_ROOT/$target
660
661 local dir=_tmp/gc-run-big
662 rm -r -f -v $dir
663 mkdir -v -p $dir
664
665 pushd $dir
666 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=100000 OILS_GC_ON_EXIT=1 \
667 $osh ../../Python-2.7.13/configure
668 popd
669}
670
671run-verbose() {
672 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
673 /usr/bin/time --format '*** MAX RSS KiB = %M' -- \
674 "$@"
675}
676
677# This hit the 24-bit object ID limitation in 2.5 seconds
678# Should be able to run indefinitely.
679run-for-a-long-time() {
680 local bin=_bin/cxx-opt/osh
681 ninja $bin
682 run-verbose $bin benchmarks/compute/fib.sh 10000
683
684 # time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 _bin/cxx-opt/osh benchmarks/compute/fib.sh 10000
685}
686
687while-loop() {
688 local i=0
689 while test $i -lt 10000; do
690 if ((i % 1000 == 0)) ; then
691 echo $i
692 fi
693 i=$((i + 1))
694 continue # BUG: skipped GC point
695 done
696}
697
698for-loop() {
699 for i in $(seq 10000); do
700 if ((i % 1000 == 0)) ; then
701 echo $i
702 fi
703 continue
704 done
705}
706
707recurse() {
708 local n=${1:-3000}
709
710 if ((n % 100 == 0)) ; then
711 echo $n
712 fi
713
714 if test $n = 0; then
715 return
716 fi
717
718 recurse $((n - 1))
719}
720
721test-loops() {
722 ### Regression for leak
723
724 local bin=_bin/cxx-opt/osh
725 ninja $bin
726
727 run-verbose $bin $0 recurse
728 echo
729
730 run-verbose $bin $0 while-loop
731 echo
732
733 run-verbose $bin $0 for-loop
734}
735
736expand-loop() {
737 local n=$1
738
739 local bin=_bin/cxx-opt/osh
740 ninja $bin
741
742 set -x
743 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
744 $bin -c "for i in {1..$n}; do echo \$i; done > /dev/null"
745 set +x
746}
747
748test-brace-exp() {
749 expand-loop 330000
750 expand-loop 340000
751}
752
753"$@"