OILS / regtest / aports-setup.sh View on Github | oils.pub

538 lines, 285 significant
1#!/usr/bin/env bash
2#
3# Set up Alpine Linux chroot. See regtest/aports.md
4#
5# Usage:
6# regtest/aports-setup.sh <function name>
7#
8# Examples:
9#
10# $0 fetch-all
11# $0 prepare-all
12#
13# Other commands:
14# $0 remove-chroot
15
16: ${LIB_OSH=stdlib/osh}
17source $LIB_OSH/bash-strict.sh
18source $LIB_OSH/task-five.sh
19
20source regtest/aports-common.sh
21
22clone-aports() {
23 local dir=../../alpinelinux
24
25 mkdir -p $dir
26 pushd $dir
27
28 # Took 1m 13s, at 27 MiB /ssec
29 time git clone \
30 https://gitlab.alpinelinux.org/alpine/aports.git || true
31 #git@gitlab.alpinelinux.org:alpine/aports.git || true
32
33 popd
34}
35
36clone-aci() {
37 # I FORKED this, because this script FUCKED UP my /dev dir and current directory!
38 # Sent patches upstream
39
40 pushd ..
41
42 time git clone \
43 git@github.com:oils-for-unix/alpine-chroot-install || true
44
45 popd
46}
47
48checkout-stable() {
49
50 pushd ../../alpinelinux/aports
51
52 # Stable release branch like 3.22-stable
53 # The branch is frqeuently patched, but I guess it just needs to match the
54 # dl-cdn.alpinelinux.org URL
55 # https://alpinelinux.org/releases/
56
57 local branch='3.22-stable'
58 git checkout $branch
59 git log -n 1
60 popd > /dev/null
61
62 echo
63 echo
64
65 pushd ../alpine-chroot-install
66 git checkout master
67 git log -n 1
68 popd > /dev/null
69}
70
71patch-aports() {
72 local cur_dir=$PWD
73
74 pushd ../../alpinelinux/aports
75
76 for patch_dir in $cur_dir/regtest/patches/*; do
77 local package_name
78 package_name=$(basename $patch_dir)
79
80 local apkbuild=main/$package_name/APKBUILD
81 git restore $apkbuild
82
83 for mod_file in $cur_dir/regtest/patches/$package_name/*; do
84 echo
85 echo "*** Processing $mod_file"
86
87 case $mod_file in
88 *.patch)
89 # A patch to the source code
90
91 # Add our patches alongside Alpine's own patches
92
93 cp -v $mod_file main/$package_name/
94
95 # abuild's default_prepare() applies all patches inside $srcdir,
96 # but they also need to be specified in the 'source=' and
97 # 'sha512sums=' lists in APKBUILD
98 local patch_name=$(basename $mod_file)
99 local shasum=$(sha512sum $mod_file | cut -d ' ' -f1)
100 sed -i "/source='*/ a $patch_name" $apkbuild
101 sed -i "/sha512sums='*/ a $shasum $patch_name" $apkbuild
102 ;;
103
104 *.apkbuild)
105 # A patch to the APKBUILD file
106 set -x
107 git apply $mod_file
108 set +x
109 ;;
110
111 *.copy)
112 # A file to copy
113 cp -v $mod_file main/$package_name/
114 ;;
115 esac
116 done
117 done
118
119 popd >/dev/null
120}
121
122
123# 2025-11-09, adding parallelism
124readonly TARBALL_ID='10772'
125
126download-oils() {
127 local tarball_id=${1:-$TARBALL_ID}
128
129 local url="https://op.oilshell.org/uuu/github-jobs/$tarball_id/cpp-tarball.wwz/_release/oils-for-unix.tar"
130
131 rm -f -v _tmp/oils-for-unix.tar
132
133 #wget --no-clobber --directory-prefix _tmp "$url"
134 wget --directory-prefix _tmp "$url"
135}
136
137make-chroot() {
138 local aci='../alpine-chroot-install/alpine-chroot-install'
139
140 $aci --help
141
142 # Notes:
143 # - $aci -d requires an ABSOLUTE path. With a relative path, it creates
144 # _chroot/aports-build/_chroot/aports-build Filed bug upstream.
145 # - The -n flag is a feature I added: do not mount host dirs
146 # - TODO: when you run it twice, it should abort if the directory is full
147 # - Takes ~8 seconds
148
149 # default packages: build-base ca-certificates ssl_client
150 #
151 # This is already 267 MB, 247 K files
152
153 mkdir -p $CHROOT_DIR # make it with normal permissions first
154
155 # "branch" is one of these: https://dl-cdn.alpinelinux.org/
156 # it's not the aports branch
157 local branch='v3.22'
158 time sudo $aci -n -d $PWD/$CHROOT_DIR -b $branch
159}
160
161make-user() {
162 enter-rootfs adduser -D udu || true
163
164 # put it in abuild group
165 enter-rootfs addgroup udu abuild || true
166 # 'wheel' is for 'sudo'
167 enter-rootfs addgroup udu wheel || true
168
169 # CHeck the state
170 enter-rootfs-user sh -c 'whoami; echo GROUPS; groups'
171}
172
173setup-doas() {
174 # Manual configuration for abuild-keygen
175
176 #sudo cat _chroot/aports-build/etc/doas.conf
177 sudo rm -f $CHROOT_DIR/etc/doas.conf
178
179 # no password
180 enter-rootfs sh -c 'echo "permit nopass :wheel" >> /etc/doas.conf'
181}
182
183config-chroot() {
184 make-user
185 setup-doas
186 keygen
187}
188
189add-build-deps() {
190 # Must be done as root; there is no 'sudo'
191
192 # alpine-sdk: abuild, etc.
193 # abuild-rootbld: package required for 'abuild rootbld'
194 # pigz: seems like it's optionally used by abuild - should probably speed
195 # things up
196 # doas: for abuild-keygen
197 # bash python3: for time-tsv
198 # findutils: for xargs --process-slot-var
199 enter-rootfs sh -c '
200 apk update
201 apk add alpine-sdk abuild-rootbld pigz doas bash python3 findutils readline-dev
202 '
203
204 # enter-rootfs -u udu bash -c 'echo "hi from bash"'
205}
206
207change-perms() {
208 # pass any number of args
209
210 # get uid from /home/udu
211 local uid
212 uid=$(stat -c '%u' $CHROOT_HOME_DIR)
213 sudo chown --verbose --recursive $uid "$@"
214}
215
216copy-aports() {
217 # 'main' and 'community' are two "Alpine repos" stored the 'aports' git repo
218 for a_repo in main community; do
219 local dest=$CHROOT_HOME_DIR/aports/$a_repo/
220
221 sudo mkdir -p $dest
222 sudo rsync --archive --verbose \
223 ../../alpinelinux/aports/$a_repo/ $dest
224
225 change-perms $dest
226 done
227}
228
229_patch-yash-to-disable-tests() {
230 ### disable tests that use job control, causing SIGTTOU bug
231
232 local apkbuild=$CHROOT_HOME_DIR/aports/main/yash/APKBUILD
233
234 # make it idempotent
235 if ! grep 'FOR OILS' "$apkbuild"; then
236 echo '
237 check() {
238 echo "=== yash tests DISABLED FOR OILS ==="
239 }
240 ' >> $apkbuild
241 fi
242}
243
244patch-yash-to-disable-tests() {
245 sudo $0 _patch-yash-to-disable-tests
246}
247
248code-manifest() {
249 # TODO: need per-file tree shaking of build/py.sh
250 local -a build_py=(
251 build/py.sh # to compile time-helper
252
253 build/common.sh
254 build/dev-shell.sh
255 stdlib/osh/bash-strict.sh
256 stdlib/osh/byo-server.sh
257 stdlib/osh/task-five.sh
258 stdlib/osh/two.sh
259 )
260 for path in \
261 benchmarks/time_.py \
262 benchmarks/time-helper.c \
263 regtest/aports-guest.sh \
264 "${build_py[@]}"
265 do
266 echo "$PWD/$path" "$path"
267 done
268}
269
270multi-cp() {
271 ### like multi cp, but works without python2
272
273 local dest=$1
274 while read -r abs_path rel_path; do
275 # -D to make dirs
276
277 # Hack: make everything executable for now
278 # I feel like this should be in 'multi cp'
279
280 install -m 755 -v -D --no-target-directory "$abs_path" "$dest/$rel_path"
281
282 # cp -v --parents doesn't work, because it requires a directory arg
283 done
284}
285
286copy-code() {
287 local dest=$CHROOT_HOME_DIR/oils
288 sudo mkdir -v -p $dest
289
290 code-manifest | sudo $0 multi-cp $dest
291
292 change-perms $dest
293}
294
295test-time-tsv() {
296 enter-rootfs-user sh -c '
297 cd oils
298 pwd
299 whoami
300 echo ---
301
302 build/py.sh time-helper
303 regtest/aports-guest.sh my-time-tsv-test
304 '
305}
306
307oils-in-chroot() {
308 copy-aports
309
310 patch-yash-to-disable-tests
311
312 copy-code
313 test-time-tsv
314
315 copy-oils
316 build-oils
317}
318
319copy-oils() {
320 local dest=$CHROOT_HOME_DIR
321
322 local tar=$PWD/_tmp/oils-for-unix.tar
323 pushd $dest
324 sudo tar -x < $tar
325 popd
326
327 change-perms $dest/oils-for-unix-*
328}
329
330keygen() {
331 enter-rootfs-user sh -c '
332 #abuild-keygen -h
333
334 # -n non-interactive
335 abuild-keygen --append --install -n
336 '
337}
338
339apk-manifest() {
340 # 1643 files - find a subset to build
341
342 for a_repo in main community; do
343 local out=$PWD/_tmp/apk-${a_repo}-manifest.txt
344 mkdir -p _tmp
345
346 pushd $CHROOT_HOME_DIR/aports/$a_repo >/dev/null
347 find . -name 'APKBUILD' -a -printf '%P\n' | sed 's,/APKBUILD$,,g' | LANG=C sort | tee $out
348 popd >/dev/null
349 done
350 wc -l _tmp/apk-*-manifest.txt
351}
352
353build-oils() {
354 enter-rootfs-user sh -c '
355 cd oils-for-unix-*
356 ./configure
357 _build/oils.sh # do not --skip-rebuild
358 doas ./install
359 '
360}
361
362save-default-config() {
363 regtest/aports-run.sh save-default-config
364}
365
366create-osh-overlay() {
367 # _chroot/
368 # aports-build/ # chroot image
369 # osh-as-sh.overlay/ # overlay
370 # merged/ # permanently mounted
371 # work/
372 # layer/ # has the OSH symlink
373
374 local osh_overlay=_chroot/osh-as-sh.overlay
375
376 mkdir -v -p $osh_overlay/{merged,work,layer}
377
378 sudo mount \
379 -t overlay \
380 osh-as-sh \
381 -o "lowerdir=$CHROOT_DIR,upperdir=$osh_overlay/layer,workdir=$osh_overlay/work" \
382 $osh_overlay/merged
383
384 $osh_overlay/merged/enter-chroot sh -c '
385 set -x
386 if ! test -f /usr/local/bin/oils-for-unix; then
387 echo "Build Oils first"
388 exit
389 fi
390 ln -s -f /usr/local/bin/oils-for-unix /bin/sh
391 ln -s -f /usr/local/bin/oils-for-unix /bin/ash
392 ln -s -f /usr/local/bin/oils-for-unix /bin/bash
393 ' dummy0
394}
395
396# Works?
397patch-overlay-with-ash() {
398 local dir=_chroot/osh-as-sh.overlay
399 pushd $dir/layer/bin
400 sudo ln -s -f -v /usr/local/bin/oils-for-unix ash
401 popd
402 ls -l $dir/layer/bin
403}
404
405remove-osh-overlay() {
406 local dir=_chroot/osh-as-sh.overlay
407 sudo umount -l $dir/merged
408 sudo rm -r -f $dir
409}
410
411remove-shard-layers() {
412 sudo rm -r -f _chroot/shard*
413}
414
415create-package-dirs() {
416 # _chroot/
417 # package.overlay/
418 # merged/ # mounted and unmounted each time
419 # work/
420 # package-layers/
421 # baseline/
422 # gzip/
423 # xz/
424 # osh-as-sh/
425 # gzip/
426 # xz/
427
428 # obsolete after $XARGS_SLOT
429 # mkdir -v -p _chroot/package.overlay/{merged,work}
430 true
431}
432
433archived-distfiles() {
434 local a_repo=$1 # 'main' or 'community'
435
436 local tar=_chroot/distfiles-${a_repo}.tar
437 tar --create --file $tar --directory $CHROOT_DIR/var/cache/distfiles .
438
439 tar --list < $tar
440 echo
441 ls -l --si $tar
442 echo
443}
444
445unpack-distfiles() {
446 local a_repo=$1 # 'main' or 'community'
447
448 local tar=_chroot/distfiles-${a_repo}.tar
449 sudo tar --verbose -x --directory $CHROOT_DIR/var/cache/distfiles < $tar
450}
451
452show-distfiles() {
453 ls -l $CHROOT_DIR/var/cache/distfiles
454}
455
456_install-hook() {
457 local bwrap=${1:-}
458
459 local out=enter-rootfs
460 ../alpine-chroot-install/alpine-chroot-install -g > $out
461 chmod +x $out
462 echo "Wrote $out"
463
464 local src
465 if test -n "$bwrap"; then
466 hook=regtest/aports/enter-hook-bwrap.sh
467 else
468 hook=../alpine-chroot-install/enter-hook-chroot
469 fi
470
471 cp -v $hook $CHROOT_DIR/enter-hook
472}
473
474install-hook() {
475 sudo $0 _install-hook "$@"
476}
477
478_install-enter-bwrap() {
479 # don't need this other bwrap script
480 rm -f -v $CHROOT_DIR/enter-hook
481
482 cp -v regtest/aports/enter-bwrap.sh $CHROOT_DIR
483}
484
485install-enter-bwrap() {
486 sudo $0 _install-enter-bwrap "$@"
487}
488
489remove-chroot() {
490 # This unmounts /dev /proc /sys/ properly!
491 $CHROOT_DIR/destroy --remove
492}
493
494remove-all() {
495 set -x
496 remove-chroot || true
497 remove-osh-overlay || true
498 remove-shard-layers || true
499}
500
501fetch-all() {
502 local tarball_id=${1:-$TARBALL_ID}
503
504 clone-aports
505 clone-aci
506 checkout-stable
507 patch-aports
508
509 download-oils "$tarball_id"
510}
511
512prepare-all() {
513 # $0 make-chroot # 267 MB, 247 K files
514 # # ncdu shows gcc is big, e.g. cc1plus, cc1, lto1 are
515 # # each 35-40 MB
516 # $0 add-build-deps # add packages that build packages
517 # # 281 MB, 248 K files
518 # $0 config-chroot # user/groups, keygen
519 # $0 oils-in-chroot # copy-aports: 307 MB, 251 K files
520
521 make-chroot
522 add-build-deps
523
524 config-chroot
525
526 oils-in-chroot
527
528 # TODO: Don't need this for overlayfs
529 save-default-config
530
531 create-osh-overlay
532 create-package-dirs
533
534 # makes a host file
535 apk-manifest
536}
537
538task-five "$@"