OILS / regtest / aports-run.sh View on Github | oils.pub

606 lines, 303 significant
1#!/usr/bin/env bash
2#
3# Build Alpine Linux packages: baseline, OSH as /bin/sh, OSH as /bin/bash
4# See regtest/aports.md
5#
6# Usage:
7# regtest/aports-run.sh <function name>
8#
9# Common usage:
10#
11# export APORTS_EPOCH=2025-08-04-foo # optional override
12# $0 build-many-shards shard{0..16} # build all 17 shards in 2 configs
13#
14# Also useful:
15#
16# $0 fetch-packages fetch $pkg_filter $a_repo # alpine repo is 'main' or 'community'
17#
18# $0 fetch-packages fetch 100,300p # packages 100-300
19# $0 fetch-packages fetch '.*' # all packages
20#
21# Look for results in _tmp/aports-build/
22#
23# Build many packages:
24#
25# $0 build-packages-overlayfs osh-as-sh shard9 community
26# $0 build-packages-overlayfs osh-as-sh shardA # main is default $a_repo
27#
28# Build a single package:
29#
30# $0 build-package-overlayfs osh-as-sh userspace-rcu
31# $0 build-package-overlayfs osh-as-sh xterm community # community repo
32#
33# Drop into a shell:
34# INTERACTIVE=1 $0 build-package-overlayfs osh-as-sh userspace-rcu
35#
36# PKG_FILTER
37# shard[0-9]+ - shard3 is packages 301 to 400
38# [0-9]+ - 42 means build the first 42 packages
39# [0-9]+,[0-9]+p - 100,300p packages 100 to 300 (sed syntax)
40# ALL - all packages
41# .* - egrep pattern matching all packages
42# curl - egrep pattern matching 'curl'
43#
44# Preview packages:
45#
46# $0 package-dirs shard9 community
47
48: ${LIB_OSH=stdlib/osh}
49source $LIB_OSH/bash-strict.sh
50source $LIB_OSH/task-five.sh
51
52source regtest/aports-common.sh
53
54#
55# Config
56#
57
58show-config() {
59 enter-rootfs sh -c '
60 ls -l /bin/sh /bin/ash /bin/bash
61 '
62}
63
64save-default-config() {
65 enter-rootfs sh -c '
66 set -x
67 dest=/bin/bash.ORIG
68 cp /bin/bash $dest
69 '
70 show-config
71}
72
73
74set-baseline() {
75 # ensure we have the default config
76 enter-rootfs sh -c '
77 set -x
78 ln -s -f /bin/busybox /bin/sh
79 ln -s -f /bin/busybox /bin/ash
80 cp /bin/bash.ORIG /bin/bash
81 '
82 show-config
83}
84
85set-osh-as-X() {
86 local x=$1
87
88 enter-rootfs sh -c '
89 x=$1
90 set -x
91 if ! test -f /usr/local/bin/oils-for-unix; then
92 echo "Build Oils first"
93 exit
94 fi
95 ln -s -f /usr/local/bin/oils-for-unix /bin/$x
96 ' dummy0 "$x"
97 show-config
98}
99
100set-osh-as-sh() {
101 set-osh-as-X sh
102}
103
104set-osh-as-ash() {
105 set-osh-as-X ash
106}
107
108set-osh-as-bash() {
109 set-osh-as-X bash
110}
111
112#
113# Run
114#
115
116package-dirs() {
117 # lz gives 5 packages: some fail at baseline
118 # lzip: a single fast package
119 # mpfr4: OSH bug, and big log
120 # yash: make sure it doesn't hang
121 local package_filter=${1:-'lz|mpfr|yash'}
122 local a_repo=${2:-main} # or 'community'
123
124 local -a prefix
125
126 if [[ $package_filter = 'ALL' ]]; then
127 prefix=( cat )
128
129 # 100 means 0 to 100
130 elif [[ $package_filter =~ ^[0-9]+$ ]]; then
131 prefix=( head -n $package_filter )
132
133 # 100,300p means lines 100 to 300
134 elif [[ $package_filter =~ ^[0-9]+,[0-9]+p$ ]]; then
135 prefix=( sed -n $package_filter )
136
137 elif [[ $package_filter =~ ^shard([0-9]+)$ ]]; then
138 # shards of 100 packages
139
140 local shard_num=${BASH_REMATCH[1]}
141 #echo shard=$shard_num
142
143 local range
144 # shard 0 is 0-99
145 # shard 9 is 900 to 999
146 # shard 10 is 1000 to 1099
147 case $shard_num in
148 # sed doesn't like 000,099
149 0) range='1,100p' ;;
150 *) range="${shard_num}01,$(( shard_num + 1))00p" ;;
151 esac
152
153 prefix=( sed -n "$range" )
154
155 # shardA, shardB For testing the combined report
156 elif [[ $package_filter =~ ^shard([A-Z]+)$ ]]; then
157 local shard_name=${BASH_REMATCH[1]}
158 case $a_repo in
159 main)
160 case $shard_name in
161 A) package_filter='^gzip' ;; # failure
162 B) package_filter='^xz' ;; # failure
163 C) package_filter='^lz' ;; # 3 packages
164 D) package_filter='^jq$' ;; # produces autotools test-suite.log
165 E) package_filter='^py3-p' ;; # many packages in parallel
166 P) package_filter='^xz$|^shorewall' ;; # patches
167 *) package_filter='^perl-http-daemon' ;; # test out perl
168 esac
169 ;;
170 community)
171 case $shard_name in
172 A) package_filter='^py3-zulip' ;; # one Python package
173 B) package_filter='^xterm' ;; # one C package
174 C) package_filter='^shfmt' ;; # one Go package
175 D) package_filter='^shellspec' ;; # OSH disagreement because of 'var'
176 *) package_filter='^shell' ;; # a bunch of packages
177 esac
178 ;;
179 *)
180 die "Invalid a_repo $a_repo"
181 ;;
182 esac
183
184 prefix=( egrep "$package_filter" )
185
186 elif [[ $package_filter =~ ^disagree-(.*)+$ ]]; then
187 local filename=${BASH_REMATCH[1]}
188 # A file of EXACT package names, not patterns
189 # See copy-disagree
190 local package_file="_tmp/$package_filter.txt"
191 comm -1 -2 <(sort $package_file) <(sort _tmp/apk-${a_repo}-manifest.txt)
192 return
193
194 else
195 prefix=( egrep "$package_filter" )
196
197 fi
198
199 "${prefix[@]}" _tmp/apk-${a_repo}-manifest.txt
200}
201
202copy-disagree() {
203 ### Determine what to run
204
205 local epoch=${1:-2025-09-18-bash}
206 cp -v \
207 _tmp/aports-report/$epoch/disagree-packages.txt \
208 _tmp/disagree-$epoch.txt
209}
210
211do-packages() {
212 ### Download sources - abuild puts it in /var/cahe/distfiles
213 local action=${1:-fetch}
214 local package_filter=${2:-}
215 local a_repo=${3:-main}
216 # flags to pass to the inner shell
217 local sh_flags=${4:-'-e -u'} # -u to disable -e
218
219 # 6 seconds for 10 packages
220 # There are ~1600 packages
221 # So if there are 20 shards, each shard could have 10?
222
223 local -a package_dirs
224 package_dirs=( $(package-dirs "$package_filter" "$a_repo") )
225
226 echo "${dirs[@]}"
227 #return
228
229 time enter-rootfs-user sh $sh_flags -c '
230
231 action=$1
232 a_repo=$2
233 shift 2
234 for dir in "$@"; do
235 time abuild -r -C aports/$a_repo/$dir "$action"
236 done
237 ' dummy0 "$action" "$a_repo" "${package_dirs[@]}"
238}
239
240fetch-packages() {
241 local package_filter=${1:-}
242 local a_repo=${2:-main}
243
244 # -u means we don't pass -e (and it's non-empty)
245 do-packages fetch "$package_filter" "$a_repo" '-u'
246}
247
248banner() {
249 echo
250 echo "=== $@"
251 echo
252}
253
254build-package-overlayfs() {
255 local config=${1:-baseline}
256 local pkg=${2:-lua5.4}
257 local a_repo=${3:-main}
258
259 # baseline stack:
260 # _chroot/aports-build
261 # _chroot/package-upper/baseline/gzip # upper dir / layer dir
262 #
263 # osh-as-sh stack:
264 # _chroot/aports-build
265 # _chroot/osh-as-sh.overlay/layer # this has the symlink
266 # _chroot/package-upper/osh-as-sh/gzip # upper dir / layer dir
267
268 # allow concurrency
269 local xargs_slot="${XARGS_SLOT:-99}"
270 local ov_base_dir=_chroot/package-slot${xargs_slot}.overlay
271
272 local merged=$ov_base_dir/merged
273 local work=$ov_base_dir/work
274
275 local layer_dir=_chroot/package-layers/$config/$pkg
276 mkdir -p $merged $work $layer_dir
277
278 local overlay_opts
279 case $config in
280 baseline)
281 overlay_opts="lowerdir=$CHROOT_DIR,upperdir=$layer_dir,workdir=$work"
282 ;;
283 osh-as-sh)
284 local osh_as_sh=_chroot/osh-as-sh.overlay/layer
285 overlay_opts="lowerdir=$osh_as_sh:$CHROOT_DIR,upperdir=$layer_dir,workdir=$work"
286 ;;
287 *)
288 die "Invalid config $config"
289 ;;
290 esac
291
292 sudo mount \
293 -t overlay \
294 aports-package \
295 -o "$overlay_opts" \
296 $merged
297
298 local -a prefix
299 if test -n "$XARGS_SLOT"; then
300 local x=$XARGS_SLOT
301
302 # run task 0 on cores 0 and 1
303 # run task 9 on cores 18 and 19
304 local cores="$(( x*2 )),$(( x*2 + 1 ))"
305 prefix=( taskset -c "$cores" )
306 fi
307
308 "${prefix[@]}" $merged/enter-chroot -u udu sh -c '
309 cd oils
310
311 # show the effect of the overlay
312 #ls -l /bin/sh
313
314 regtest/aports-guest.sh build-one-package "$@"
315 ' dummy0 "$pkg" "$a_repo" "$xargs_slot"
316
317 if test -n "$INTERACTIVE"; then
318 echo "Starting interactive shell in overlayfs environment for package $a_repo/$pkg"
319 echo "Rebuild: abuild -f -r -C ~/aports/$a_repo/$pkg"
320 echo " Help: abuild -h"
321 # If the last command in the child shell exited non-zero then ctrl-d/exit
322 # will report that error code to the parent. If we don't ignore that error
323 # we will exit early and leave the package overlay mounted.
324 set +o errexit
325 $merged/enter-chroot -u udu
326 set -o errexit
327 fi
328
329 unmount-loop $merged
330}
331
332# old serial version
333OLD-build-many-packages-overlayfs() {
334 local package_filter=${1:-}
335 local config=${2:-baseline}
336 local a_repo=${3:-main}
337
338 local -a package_dirs
339 package_dirs=( $(package-dirs "$package_filter" "$a_repo") )
340
341 banner "Building ${#package_dirs[@]} packages (filter=$package_filter a_repo=$a_repo)"
342
343 for pkg in "${package_dirs[@]}"; do
344 build-package-overlayfs "$config" "$pkg" "$a_repo"
345 done
346}
347
348build-pkg() {
349 ### trivial wrapper around build-package-overlayfs - change arg order for xargs
350 local config=${1:-baseline}
351 local a_repo=${2:-main}
352 local pkg=${3:-lua5.4}
353
354 build-package-overlayfs "$config" "$pkg" "$a_repo"
355}
356
357# 2 cores per job
358NUM_PAR=$(( $(nproc) / 2 ))
359
360# TODO: we ran into the env.sh race condition in the enter-chroot script
361# generated by alpine-chroot-install
362build-many-packages-overlayfs() {
363 local package_filter=${1:-}
364 local config=${2:-baseline}
365 local a_repo=${3:-main}
366 local parallel=${4:-T}
367
368 banner "Building packages (filter=$package_filter a_repo=$a_repo)"
369
370 local -a flags
371 if test -n "$parallel"; then
372 log "(with $NUM_PAR jobs in parallel)"
373 flags=( -P $NUM_PAR )
374 else
375 log '(serially)'
376 fi
377
378 package-dirs "$package_filter" $a_repo |
379 xargs "${flags[@]}" -n 1 --process-slot-var=XARGS_SLOT -- \
380 $0 build-pkg $config $a_repo
381}
382
383
384clean-host-and-guest() {
385 # host dir _tmp/aports-build
386 rm -r -f -v $BASE_DIR
387}
388
389clean-guest() {
390 # clean guest chroot
391 sudo rm -r -f -v $CHROOT_HOME_DIR/oils/_tmp
392}
393
394readonly -a CONFIGS=( baseline osh-as-sh )
395
396APORTS_EPOCH="${APORTS_EPOCH:-}"
397# default epoch
398if test -z "$APORTS_EPOCH"; then
399 APORTS_EPOCH=$(date '+%Y-%m-%d')
400fi
401
402_build-many-configs-overlayfs() {
403 local package_filter=${1:-}
404 local epoch=${2:-$APORTS_EPOCH}
405 local a_repo=${3:-main}
406
407 if test -z "$package_filter"; then
408 die "Package filter is required (e.g. shard3, ALL)"
409 fi
410
411 clean-guest
412
413 # See note about /etc/sudoers.d at top of file
414
415 local dest_dir="$BASE_DIR/$epoch/$package_filter" # e.g. shard10
416
417 for config in "${CONFIGS[@]}"; do
418 banner "$epoch: Using config $config"
419
420 build-many-packages-overlayfs "$package_filter" "$config" "$a_repo"
421 done
422}
423
424remove-shard-files() {
425 local shard_dir=${1:-_chroot/shardC}
426
427 log "Removing big files in shard $shard_dir"
428
429 # For all packages packages, for baseline and osh-as-sh, clean up the aports source dir
430 # For linux, clang, etc. it becomes MANY GIGABYTES
431 #
432 # 2025-09-12: ignore errors from rm; I think there was a race condition -
433 # processes could still be running and creating files
434 #
435 # rm: cannot remove '_chroot/shard6/baseline/llvm19/home/udu/aports/main/llvm19/src/llvm-project-19.1.7.src/build/lib': Directory not empty
436 # real 1041m46.464s
437
438 sudo rm -r -f $shard_dir/*/*/home/udu/aports/ || true
439}
440
441build-many-shards-overlayfs() {
442 sudo -k
443
444 local a_repo=${A_REPO:-main} # env var like $APORTS_EPOCH
445
446 # Clean up old runs
447 sudo rm -r -f _chroot/shard* _chroot/disagree*
448
449 banner "$APORTS_EPOCH $a_repo: building shards: $*"
450
451 time for shard_name in "$@"; do
452 _build-many-configs-overlayfs "$shard_name" "$APORTS_EPOCH" "$a_repo"
453
454 # Move to _chroot/shard10, etc.
455 mv -v --no-target-directory _chroot/package-layers _chroot/$shard_name
456
457 make-shard-tree $shard_name $a_repo
458
459 remove-shard-files _chroot/$shard_name
460 done
461}
462
463make-shard-tree() {
464 ### Put outputs in rsync-able format, for a SINGLE shard
465
466 # The dire structure is like this:
467 #
468 # _tmp/aports-build/
469 # 2025-09-10-overlayfs/
470 # shard0/
471 # baseline/
472 # apk.txt
473 # tasks.tsv
474 # log/
475 # gzip.log.txt
476 # xz.log.txt
477 # test-suite/ # autotools dir
478 # gzip/
479 # test-suite.log.txt
480 # osh-as-sh/
481 # apk.txt
482 # tasks.tsv
483 # log/
484 # gzip.log.txt
485 # xz.log.txt
486 # test-suite/
487 # gzip/
488 # test-suite.log.txt
489 # shard1/
490 # ...
491 # shard16/
492 # ...
493
494 local shard_name=$1
495 local a_repo=${2:-main}
496 local epoch=${3:-$APORTS_EPOCH}
497
498 local shard_dir=_chroot/$shard_name
499
500 for config in baseline osh-as-sh; do
501 local dest_dir=$BASE_DIR/$epoch/$shard_name/$config
502 mkdir -p $dest_dir
503 #ls -l $shard_dir/$config
504
505 time python3 devtools/tsv_concat.py \
506 $shard_dir/$config/*/home/udu/oils/_tmp/aports-guest/*.task.tsv > $dest_dir/tasks.tsv
507
508 # Allowed to fail if zero .apk are built
509 time md5sum $shard_dir/$config/*/home/udu/packages/$a_repo/x86_64/*.apk > $dest_dir/apk.txt \
510 || true
511
512 abridge-logs2 $shard_dir/$config $dest_dir
513
514 done
515}
516
517abridge-logs2() {
518 local config_src_dir=${1:-_chroot/shardD/osh-as-sh}
519 local dest_dir=${2:-$BASE_DIR/shardD/osh-as-sh}
520
521 local log_dest_dir=$dest_dir/log
522 local test_suite_dest_dir=$dest_dir/test-suite
523 mkdir -p $log_dest_dir $test_suite_dest_dir
524
525 local threshold=$(( 500 * 1000 )) # 500 KB
526
527 # this assumes the build process doesn't create *.log.txt
528 # test-suite.log is the name used by the autotools test runner - we want to save those too
529 # ignore permission errors with || true
530 { find $config_src_dir -name '*.log.txt' -a -printf '%s\t%P\n' || true; } |
531 while read -r size path; do
532 local src=$config_src_dir/$path
533 # Remove text until last slash (shortest match)
534 # like $(basename $path) but in bash, for speed
535 local filename=${path##*/}
536 local dest=$log_dest_dir/$filename
537
538 if test "$size" -lt "$threshold"; then
539 cp -v $src $dest
540 else
541 # Bug fix: abriding to 1000 lines isn't sufficient. We got some logs
542 # that were hundreds of MB, with less than 1000 lines!
543 { echo "*** This log is abridged to its last 500 KB:"
544 echo
545 tail --bytes 500000 $src
546 } > $dest
547 fi
548 done
549
550 { find $config_src_dir -name 'test-suite.log' -a -printf '%P\n' || true; } |
551 while read -r path; do
552 local src=$config_src_dir/$path
553
554 # Remove text after the first slash (shortest match)
555 local package_name=${path%%/*}
556 local dest=$test_suite_dest_dir/$package_name/test-suite.log.txt
557
558 mkdir -p "$(dirname $dest)"
559 cp -v --no-target-directory $src $dest
560 done
561
562 # 500K threshold: 76 MB
563 du --si -s $log_dest_dir
564}
565
566compare-speed() {
567 ### reusing the chroot reuses is a LITTLE faster, but not a lot
568
569 # single chroot
570 build-many-shards shardC
571
572 # 3 chroots + overlayfs mounts
573 build-many-shards2 shardC
574}
575
576demo-build() {
577 local pkg=${1:-gzip} # in shardA, uses many cores
578 local do_pin=${2:-}
579
580 local -a prefix
581 if test -n "$do_pin"; then
582 echo "*** Pinning to CPU 0 ***"
583 prefix=( taskset -c 0 )
584 fi
585
586 "${prefix[@]}" $CHROOT_DIR/enter-chroot -u udu sh -c '
587 pkg=$1
588
589 echo "nproc = $(nproc)"
590
591 cd oils
592 set -x
593
594 # Note the user / real ratio! How many cores did we use?
595 time regtest/aports-guest.sh build-one-package $pkg
596 ' dummy0 $pkg
597}
598
599test-taskset() {
600 local pkg=${1:-gzip} # in shardA, uses many cores
601
602 demo-build $pkg ''
603 demo-build $pkg T
604}
605
606task-five "$@"