OILS / regtest / aports-setup.sh View on Github | oils.pub

511 lines, 277 significant
1#!/usr/bin/env bash
2#
3# Set up Alpine Linux chroot. See regtest/aports.md
4#
5# Usage:
6# regtest/aports-setup.sh <function name>
7#
8# Examples:
9#
10# $0 fetch-all
11# $0 prepare-all
12#
13# Other commands:
14# $0 remove-chroot
15
16: ${LIB_OSH=stdlib/osh}
17source $LIB_OSH/bash-strict.sh
18source $LIB_OSH/task-five.sh
19
20source regtest/aports-common.sh
21
22clone-aports() {
23 local dir=../../alpinelinux
24
25 mkdir -p $dir
26 pushd $dir
27
28 # Took 1m 13s, at 27 MiB /ssec
29 time git clone \
30 https://gitlab.alpinelinux.org/alpine/aports.git || true
31 #git@gitlab.alpinelinux.org:alpine/aports.git || true
32
33 popd
34}
35
36clone-aci() {
37 # I FORKED this, because this script FUCKED UP my /dev dir and current directory!
38 # Sent patches upstream
39
40 pushd ..
41
42 time git clone \
43 git@github.com:oils-for-unix/alpine-chroot-install || true
44
45 popd
46}
47
48checkout-stable() {
49
50 pushd ../../alpinelinux/aports
51
52 # Stable release branch like 3.22-stable
53 # The branch is frqeuently patched, but I guess it just needs to match the
54 # dl-cdn.alpinelinux.org URL
55 # https://alpinelinux.org/releases/
56
57 local branch='3.22-stable'
58 git checkout $branch
59 git log -n 1
60 popd > /dev/null
61
62 echo
63 echo
64
65 pushd ../alpine-chroot-install
66 git checkout master
67 git log -n 1
68 popd > /dev/null
69}
70
71patch-aports() {
72 local cur_dir=$PWD
73
74 pushd ../../alpinelinux/aports
75
76 for patch_dir in $cur_dir/regtest/patches/*; do
77 local package_name
78 package_name=$(basename $patch_dir)
79
80 local apkbuild=main/$package_name/APKBUILD
81 git restore $apkbuild
82
83 for mod_file in $cur_dir/regtest/patches/$package_name/*; do
84 echo
85 echo "*** Processing $mod_file"
86
87 case $mod_file in
88 *.patch)
89 # A patch to the source code
90
91 # Add our patches alongside Alpine's own patches
92
93 cp -v $mod_file main/$package_name/
94
95 # abuild's default_prepare() applies all patches inside $srcdir,
96 # but they also need to be specified in the 'source=' and
97 # 'sha512sums=' lists in APKBUILD
98 local patch_name=$(basename $mod_file)
99 local shasum=$(sha512sum $mod_file | cut -d ' ' -f1)
100 sed -i "/source='*/ a $patch_name" $apkbuild
101 sed -i "/sha512sums='*/ a $shasum $patch_name" $apkbuild
102 ;;
103
104 *.apkbuild)
105 # A patch to the APKBUILD file
106 set -x
107 git apply $mod_file
108 set +x
109 ;;
110
111 *.copy)
112 # A file to copy
113 cp -v $mod_file main/$package_name/
114 ;;
115 esac
116 done
117 done
118
119 popd >/dev/null
120}
121
122# 2025-11-16: fresh run
123readonly TARBALL_ID='10856'
124
125download-oils() {
126 local tarball_id=${1:-$TARBALL_ID}
127
128 local url="https://op.oilshell.org/uuu/github-jobs/$tarball_id/cpp-tarball.wwz/_release/oils-for-unix.tar"
129
130 rm -f -v _tmp/oils-for-unix.tar
131
132 #wget --no-clobber --directory-prefix _tmp "$url"
133 wget --directory-prefix _tmp "$url"
134}
135
136make-chroot() {
137 local aci='../alpine-chroot-install/alpine-chroot-install'
138
139 $aci --help
140
141 # Notes:
142 # - $aci -d requires an ABSOLUTE path. With a relative path, it creates
143 # _chroot/aports-build/_chroot/aports-build Filed bug upstream.
144 # - The -n flag is a feature I added: do not mount host dirs
145 # - TODO: when you run it twice, it should abort if the directory is full
146 # - Takes ~8 seconds
147
148 # default packages: build-base ca-certificates ssl_client
149 #
150 # This is already 267 MB, 247 K files
151
152 mkdir -p $CHROOT_DIR # make it with normal permissions first
153
154 # "branch" is one of these: https://dl-cdn.alpinelinux.org/
155 # it's not the aports branch
156 local branch='v3.22'
157 time sudo $aci -n -d $PWD/$CHROOT_DIR -b $branch
158}
159
160make-user() {
161 enter-rootfs adduser -D udu || true
162
163 # put it in abuild group
164 enter-rootfs addgroup udu abuild || true
165 # 'wheel' is for 'sudo'
166 enter-rootfs addgroup udu wheel || true
167
168 # CHeck the state
169 enter-rootfs-user sh -c 'whoami; echo GROUPS; groups'
170}
171
172setup-doas() {
173 # Manual configuration for abuild-keygen
174
175 #sudo cat _chroot/aports-build/etc/doas.conf
176 sudo rm -f $CHROOT_DIR/etc/doas.conf
177
178 # no password
179 enter-rootfs sh -c 'echo "permit nopass :wheel" >> /etc/doas.conf'
180}
181
182config-chroot() {
183 make-user
184 setup-doas
185 keygen
186}
187
188add-build-deps() {
189 # Must be done as root; there is no 'sudo'
190
191 # alpine-sdk: abuild, etc.
192 # abuild-rootbld: package required for 'abuild rootbld'
193 # pigz: seems like it's optionally used by abuild - should probably speed
194 # things up
195 # doas: for abuild-keygen
196 # bash python3: for time-tsv
197 # findutils: for xargs --process-slot-var
198 enter-rootfs sh -c '
199 apk update
200 apk add alpine-sdk abuild-rootbld pigz doas bash python3 findutils readline-dev
201 '
202
203 # enter-rootfs -u udu bash -c 'echo "hi from bash"'
204}
205
206change-perms() {
207 # pass any number of args
208
209 # get uid from /home/udu
210 local uid
211 uid=$(stat -c '%u' $CHROOT_HOME_DIR)
212 sudo chown --verbose --recursive $uid "$@"
213}
214
215copy-aports() {
216 # 'main' and 'community' are two "Alpine repos" stored the 'aports' git repo
217 for a_repo in main community; do
218 local dest=$CHROOT_HOME_DIR/aports/$a_repo/
219
220 sudo mkdir -p $dest
221 sudo rsync --archive --verbose \
222 ../../alpinelinux/aports/$a_repo/ $dest
223
224 change-perms $dest
225 done
226}
227
228_patch-yash-to-disable-tests() {
229 ### disable tests that use job control, causing SIGTTOU bug
230
231 local apkbuild=$CHROOT_HOME_DIR/aports/main/yash/APKBUILD
232
233 # make it idempotent
234 if ! grep 'FOR OILS' "$apkbuild"; then
235 echo '
236 check() {
237 echo "=== yash tests DISABLED FOR OILS ==="
238 }
239 ' >> $apkbuild
240 fi
241}
242
243patch-yash-to-disable-tests() {
244 sudo $0 _patch-yash-to-disable-tests
245}
246
247code-manifest() {
248 # TODO: need per-file tree shaking of build/py.sh
249 local -a build_py=(
250 build/py.sh # to compile time-helper
251
252 build/common.sh
253 build/dev-shell.sh
254 stdlib/osh/bash-strict.sh
255 stdlib/osh/byo-server.sh
256 stdlib/osh/task-five.sh
257 stdlib/osh/two.sh
258 )
259 for path in \
260 benchmarks/time_.py \
261 benchmarks/time-helper.c \
262 regtest/aports-guest.sh \
263 "${build_py[@]}"
264 do
265 echo "$PWD/$path" "$path"
266 done
267}
268
269multi-cp() {
270 ### like multi cp, but works without python2
271
272 local dest=$1
273 while read -r abs_path rel_path; do
274 # -D to make dirs
275
276 # Hack: make everything executable for now
277 # I feel like this should be in 'multi cp'
278
279 install -m 755 -v -D --no-target-directory "$abs_path" "$dest/$rel_path"
280
281 # cp -v --parents doesn't work, because it requires a directory arg
282 done
283}
284
285copy-code() {
286 local dest=$CHROOT_HOME_DIR/oils
287 sudo mkdir -v -p $dest
288
289 code-manifest | sudo $0 multi-cp $dest
290
291 change-perms $dest
292}
293
294test-time-tsv() {
295 enter-rootfs-user sh -c '
296 cd oils
297 pwd
298 whoami
299 echo ---
300
301 build/py.sh time-helper
302 regtest/aports-guest.sh my-time-tsv-test
303 '
304}
305
306oils-in-chroot() {
307 copy-aports
308
309 patch-yash-to-disable-tests
310
311 copy-code
312 test-time-tsv
313
314 copy-oils
315 build-oils
316}
317
318copy-oils() {
319 local dest=$CHROOT_HOME_DIR
320
321 local tar=$PWD/_tmp/oils-for-unix.tar
322 pushd $dest
323 sudo tar -x < $tar
324 popd
325
326 change-perms $dest/oils-for-unix-*
327}
328
329keygen() {
330 enter-rootfs-user sh -c '
331 #abuild-keygen -h
332
333 # -n non-interactive
334 abuild-keygen --append --install -n
335 '
336}
337
338apk-manifest() {
339 # 1643 files - find a subset to build
340
341 for a_repo in main community; do
342 local out=$PWD/_tmp/apk-${a_repo}-manifest.txt
343 mkdir -p _tmp
344
345 pushd $CHROOT_HOME_DIR/aports/$a_repo >/dev/null
346 find . -name 'APKBUILD' -a -printf '%P\n' | sed 's,/APKBUILD$,,g' | LANG=C sort | tee $out
347 popd >/dev/null
348 done
349 wc -l _tmp/apk-*-manifest.txt
350}
351
352build-oils() {
353 enter-rootfs-user sh -c '
354 cd oils-for-unix-*
355 ./configure
356 _build/oils.sh # do not --skip-rebuild
357 doas ./install
358 '
359}
360
361create-osh-overlay() {
362 # _chroot/
363 # aports-build/ # chroot image
364 # osh-as-sh.overlay/ # overlay
365 # merged/ # permanently mounted
366 # work/
367 # layer/ # has the OSH symlink
368
369 local osh_overlay=_chroot/osh-as-sh.overlay
370
371 mkdir -v -p $osh_overlay/{merged,work,layer}
372
373 sudo mount \
374 -t overlay \
375 osh-as-sh \
376 -o "lowerdir=$CHROOT_DIR,upperdir=$osh_overlay/layer,workdir=$osh_overlay/work" \
377 $osh_overlay/merged
378
379 $osh_overlay/merged/enter-chroot sh -c '
380 set -x
381 if ! test -f /usr/local/bin/oils-for-unix; then
382 echo "Build Oils first"
383 exit
384 fi
385 ln -s -f /usr/local/bin/oils-for-unix /bin/sh
386 ln -s -f /usr/local/bin/oils-for-unix /bin/ash
387 ln -s -f /usr/local/bin/oils-for-unix /bin/bash
388 ' dummy0
389}
390
391# Works?
392patch-overlay-with-ash() {
393 local dir=_chroot/osh-as-sh.overlay
394 pushd $dir/layer/bin
395 sudo ln -s -f -v /usr/local/bin/oils-for-unix ash
396 popd
397 ls -l $dir/layer/bin
398}
399
400remove-osh-overlay() {
401 local dir=_chroot/osh-as-sh.overlay
402 sudo umount -l $dir/merged
403 sudo rm -r -f $dir
404}
405
406remove-shard-layers() {
407 sudo rm -r -f _chroot/shard*
408}
409
410archived-distfiles() {
411 local a_repo=$1 # 'main' or 'community'
412
413 local tar=_chroot/distfiles-${a_repo}.tar
414 tar --create --file $tar --directory $CHROOT_DIR/var/cache/distfiles .
415
416 tar --list < $tar
417 echo
418 ls -l --si $tar
419 echo
420}
421
422unpack-distfiles() {
423 local a_repo=$1 # 'main' or 'community'
424
425 local tar=_chroot/distfiles-${a_repo}.tar
426 sudo tar --verbose -x --directory $CHROOT_DIR/var/cache/distfiles < $tar
427}
428
429show-distfiles() {
430 ls -l $CHROOT_DIR/var/cache/distfiles
431}
432
433_install-hook() {
434 local bwrap=${1:-}
435
436 local out=enter-rootfs
437 ../alpine-chroot-install/alpine-chroot-install -g > $out
438 chmod +x $out
439 echo "Wrote $out"
440
441 local src
442 if test -n "$bwrap"; then
443 hook=regtest/aports/enter-hook-bwrap.sh
444 else
445 hook=../alpine-chroot-install/enter-hook-chroot
446 fi
447
448 cp -v $hook $CHROOT_DIR/enter-hook
449}
450
451install-hook() {
452 sudo $0 _install-hook "$@"
453}
454
455_install-enter-bwrap() {
456 # don't need this other bwrap script
457 rm -f -v $CHROOT_DIR/enter-hook
458
459 cp -v regtest/aports/enter-bwrap.sh $CHROOT_DIR
460}
461
462install-enter-bwrap() {
463 sudo $0 _install-enter-bwrap "$@"
464}
465
466remove-chroot() {
467 # This unmounts /dev /proc /sys/ properly!
468 $CHROOT_DIR/destroy --remove
469}
470
471remove-all() {
472 set -x
473 remove-chroot || true
474 remove-osh-overlay || true
475 remove-shard-layers || true
476}
477
478fetch-all() {
479 local tarball_id=${1:-$TARBALL_ID}
480
481 clone-aports
482 clone-aci
483 checkout-stable
484 patch-aports
485
486 download-oils "$tarball_id"
487}
488
489prepare-all() {
490 # $0 make-chroot # 267 MB, 247 K files
491 # # ncdu shows gcc is big, e.g. cc1plus, cc1, lto1 are
492 # # each 35-40 MB
493 # $0 add-build-deps # add packages that build packages
494 # # 281 MB, 248 K files
495 # $0 config-chroot # user/groups, keygen
496 # $0 oils-in-chroot # copy-aports: 307 MB, 251 K files
497
498 make-chroot
499 add-build-deps
500
501 config-chroot
502
503 oils-in-chroot
504
505 create-osh-overlay
506
507 # makes a host file
508 apk-manifest
509}
510
511task-five "$@"