OILS / regtest / debian-setup.ysh View on Github | oils.pub

539 lines, 107 significant
1#!bin/ysh
2
3#
4# Set up Debian Linux chroot. See regtest/aports.md
5#
6# Usage:
7# regtest/aports-setup.sh <function name>
8
9# : ${LIB_OSH=stdlib/osh}
10# source $LIB_OSH/bash-strict.sh
11# source $LIB_OSH/task-five.sh
12
13# source regtest/aports-common.sh
14
15# clone-aports() {
16# local dir=../../alpinelinux
17
18# mkdir -p $dir
19# pushd $dir
20
21# # Took 1m 13s, at 27 MiB /ssec
22# time git clone \
23# https://gitlab.alpinelinux.org/alpine/aports.git || true
24# #git@gitlab.alpinelinux.org:alpine/aports.git || true
25
26# popd
27# }
28
29# clone-aci() {
30# # I FORKED this, because this script FUCKED UP my /dev dir and current directory!
31# # Sent patches upstream
32
33# pushd ..
34
35# time git clone \
36# git@github.com:oils-for-unix/alpine-chroot-install || true
37
38# popd
39# }
40
41# checkout-stable() {
42
43# pushd ../../alpinelinux/aports
44
45# # Stable release branch like 3.22-stable
46# # The branch is frqeuently patched, but I guess it just needs to match the
47# # dl-cdn.alpinelinux.org URL
48# # https://alpinelinux.org/releases/
49
50# local branch='3.22-stable'
51# git checkout $branch
52# git log -n 1
53# popd > /dev/null
54
55# echo
56# echo
57
58# pushd ../alpine-chroot-install
59# git checkout master
60# git log -n 1
61# popd > /dev/null
62# }
63
64# patch-aports() {
65# local cur_dir=$PWD
66
67# pushd ../../alpinelinux/aports
68
69# for patch_dir in $cur_dir/regtest/patches/*; do
70# local package_name
71# package_name=$(basename $patch_dir)
72
73# local apkbuild=main/$package_name/APKBUILD
74# git restore $apkbuild
75
76# for mod_file in $cur_dir/regtest/patches/$package_name/*; do
77# echo
78# echo "*** Processing $mod_file"
79
80# case $mod_file in
81# *.patch)
82# # A patch to the source code
83
84# # Add our patches alongside Alpine's own patches
85
86# cp -v $mod_file main/$package_name/
87
88# # abuild's default_prepare() applies all patches inside $srcdir,
89# # but they also need to be specified in the 'source=' and
90# # 'sha512sums=' lists in APKBUILD
91# local patch_name=$(basename $mod_file)
92# local shasum=$(sha512sum $mod_file | cut -d ' ' -f1)
93# sed -i "/source='*/ a $patch_name" $apkbuild
94# sed -i "/sha512sums='*/ a $shasum $patch_name" $apkbuild
95# ;;
96
97# *.apkbuild)
98# # A patch to the APKBUILD file
99# set -x
100# git apply $mod_file
101# set +x
102# ;;
103
104# *.copy)
105# # A file to copy
106# cp -v $mod_file main/$package_name/
107# ;;
108# esac
109# done
110# done
111
112# popd >/dev/null
113# }
114
115# # 2025-11-16: fresh run
116# readonly TARBALL_ID='10856'
117# const TARBALL_ID='11092'
118
119proc download-oils() {
120 var tarball_id=${1:-$TARBALL_ID}
121
122 # var url="https://op.oilshell.org/uuu/github-jobs/$tarball_id/cpp-tarball.wwz/_release/oils-for-unix.tar"
123 var url="https://op.oilshell.org/code/github-jobs/git-fd65318bebefcacf83f0ba8678575c4a8f56fa4b/oils-for-unix.tar"
124
125 rm -f -v _tmp/oils-for-unix.tar
126
127 #wget --no-clobber --directory-prefix _tmp "$url"
128 wget --directory-prefix _tmp "$url"
129}
130
131# make-chroot() {
132# local aci='../alpine-chroot-install/alpine-chroot-install'
133
134# $aci --help
135
136# # Notes:
137# # - $aci -d requires an ABSOLUTE path. With a relative path, it creates
138# # _chroot/aports-build/_chroot/aports-build Filed bug upstream.
139# # - The -n flag is a feature I added: do not mount host dirs
140# # - TODO: when you run it twice, it should abort if the directory is full
141# # - Takes ~8 seconds
142
143# # default packages: build-base ca-certificates ssl_client
144# #
145# # This is already 267 MB, 247 K files
146
147# mkdir -p $CHROOT_DIR # make it with normal permissions first
148
149# # "branch" is one of these: https://dl-cdn.alpinelinux.org/
150# # it's not the aports branch
151# local branch='v3.22'
152# time sudo $aci -n -d $PWD/$CHROOT_DIR -b $branch
153# }
154
155proc make-user() {
156 exec-chroot "useradd -m -s /bin/bash udu"
157
158 # 'wheel' is for 'sudo'
159 exec-chroot "groupadd -f wheel"
160 exec-chroot "adduser udu wheel"
161}
162
163proc setup-doas() {
164 #sudo cat _chroot/aports-build/etc/doas.conf
165 sudo rm -f $CHROOT_DIR/etc/doas.conf
166
167 # no password
168 exec-chroot u'echo "permit nopass :wheel" >> /etc/doas.conf'
169}
170
171
172# add-build-deps() {
173# # Must be done as root; there is no 'sudo'
174
175# # alpine-sdk: abuild, etc.
176# # abuild-rootbld: package required for 'abuild rootbld'
177# # pigz: seems like it's optionally used by abuild - should probably speed
178# # things up
179# # doas: for abuild-keygen
180# # bash python3: for time-tsv
181# # findutils: for xargs --process-slot-var
182# enter-rootfs sh -c '
183# apk update
184# apk add alpine-sdk abuild-rootbld pigz doas bash python3 findutils readline-dev
185# '
186
187# # enter-rootfs -u udu bash -c 'echo "hi from bash"'
188# }
189
190proc change-perms(path) {
191 # pass any number of args
192
193 # get uid from /home/udu
194 var uid=$(stat -c '%u' $CHROOT_HOME_DIR)
195 sudo chown --verbose --recursive $uid $path
196}
197
198# copy-aports() {
199# # 'main' and 'community' are two "Alpine repos" stored the 'aports' git repo
200# for a_repo in main community; do
201# local dest=$CHROOT_HOME_DIR/aports/$a_repo/
202
203# sudo mkdir -p $dest
204# sudo rsync --archive --verbose \
205# ../../alpinelinux/aports/$a_repo/ $dest
206
207# change-perms $dest
208# done
209# }
210
211# _patch-yash-to-disable-tests() {
212# ### disable tests that use job control, causing SIGTTOU bug
213
214# local apkbuild=$CHROOT_HOME_DIR/aports/main/yash/APKBUILD
215
216# # make it idempotent
217# if ! grep 'FOR OILS' "$apkbuild"; then
218# echo '
219# check() {
220# echo "=== yash tests DISABLED FOR OILS ==="
221# }
222# ' >> $apkbuild
223# fi
224# }
225
226# patch-yash-to-disable-tests() {
227# sudo $0 _patch-yash-to-disable-tests
228# }
229
230# code-manifest() {
231# # TODO: need per-file tree shaking of build/py.sh
232# local -a build_py=(
233# build/py.sh # to compile time-helper
234
235# build/common.sh
236# build/dev-shell.sh
237# stdlib/osh/bash-strict.sh
238# stdlib/osh/byo-server.sh
239# stdlib/osh/task-five.sh
240# stdlib/osh/two.sh
241# )
242# for path in \
243# benchmarks/time_.py \
244# benchmarks/time-helper.c \
245# regtest/aports-guest.sh \
246# "${build_py[@]}"
247# do
248# echo "$PWD/$path" "$path"
249# done
250# }
251
252# multi-cp() {
253# ### like multi cp, but works without python2
254
255# local dest=$1
256# while read -r abs_path rel_path; do
257# # -D to make dirs
258
259# # Hack: make everything executable for now
260# # I feel like this should be in 'multi cp'
261
262# install -m 755 -v -D --no-target-directory "$abs_path" "$dest/$rel_path"
263
264# # cp -v --parents doesn't work, because it requires a directory arg
265# done
266# }
267
268# copy-code() {
269# local dest=$CHROOT_HOME_DIR/oils
270# sudo mkdir -v -p $dest
271
272# code-manifest | sudo $0 multi-cp $dest
273
274# change-perms $dest
275# }
276
277# test-time-tsv() {
278# enter-rootfs-user sh -c '
279# cd oils
280# pwd
281# whoami
282# echo ---
283
284# build/py.sh time-helper
285# regtest/aports-guest.sh my-time-tsv-test
286# '
287# }
288
289proc oils-in-chroot() {
290 # copy-aports
291
292 # patch-yash-to-disable-tests
293
294 # copy-code
295 # test-time-tsv
296 copy-oils
297 build-oils
298}
299
300proc copy-oils() {
301 var dest=CHROOT_HOME_DIR
302
303 var tar=$(pwd) ++ '/_tmp/oils-for-unix.tar'
304 pushd $dest
305 sudo tar -x < $tar
306 popd
307
308 change-perms "$dest/oils-for-unix-0.37.0"
309}
310
311# keygen() {
312# enter-rootfs-user sh -c '
313# #abuild-keygen -h
314
315# # -n non-interactive
316# abuild-keygen --append --install -n
317# '
318# }
319
320# apk-manifest() {
321# # 1643 files - find a subset to build
322
323# for a_repo in main community; do
324# local out=$PWD/_tmp/apk-${a_repo}-manifest.txt
325# mkdir -p _tmp
326
327# pushd $CHROOT_HOME_DIR/aports/$a_repo >/dev/null
328# find . -name 'APKBUILD' -a -printf '%P\n' | sed 's,/APKBUILD$,,g' | LANG=C sort | tee $out
329# popd >/dev/null
330# done
331# wc -l _tmp/apk-*-manifest.txt
332# }
333
334build-oils() {
335 exec-chroot-user 'cd oils-for-unix-*
336 ./configure
337 _build/oils.sh # do not --skip-rebuild
338 doas ./install
339 '
340}
341
342proc create-osh-overlay() {
343 # _debian_chroot/
344 # debian-build/ # chroot image
345 # osh-as-sh.overlay/ # overlay
346 # merged/ # permanently mounted
347 # work/
348 # layer/ # has the OSH symlink
349
350 var osh_overlay='_debian_chroot/osh-as-sh.overlay'
351
352 mkdir -v -p $osh_overlay/{merged,work,layer}
353
354 # -o index=off fixes this error: fsconfig() failed: Stale file handle
355 # See also https://oilshell.zulipchat.com/#narrow/channel/522730-distros/topic/setting.20up.20regtest.2Faports-setup.2Esh/with/544318771
356 sudo mount \
357 -t overlay \
358 osh-as-sh \
359 -o index=off \
360 -o "lowerdir=$CHROOT_DIR,upperdir=$osh_overlay/layer,workdir=$osh_overlay/work" \
361 $osh_overlay/merged
362
363 exec-chroot-osh '
364 set -x
365 if ! test -f /usr/local/bin/oils-for-unix; then
366 echo "Build Oils first"
367 exit
368 fi
369 ln -s -f /usr/local/bin/oils-for-unix /bin/sh
370 ln -s -f /usr/local/bin/oils-for-unix /bin/dash
371 ln -s -f /usr/local/bin/oils-for-unix /bin/bash
372 '
373}
374
375# # Works?
376# patch-overlay-with-ash() {
377# local dir=_chroot/osh-as-sh.overlay
378# pushd $dir/layer/bin
379# sudo ln -s -f -v /usr/local/bin/oils-for-unix ash
380# popd
381# ls -l $dir/layer/bin
382# }
383
384proc remove-osh-overlay() {
385 var dir='_debian_chroot/osh-as-sh.overlay'
386 sudo umount $dir/merged
387 sudo rm -r -f $dir
388}
389
390# remove-shard-layers() {
391# sudo rm -r -f _chroot/shard*
392# }
393
394# make-distfiles-tar() {
395# local a_repo=$1 # 'main' or 'community'
396
397# local tar=_chroot/distfiles-${a_repo}.tar
398# tar --create --file $tar --directory $CHROOT_DIR/var/cache/distfiles .
399
400# tar --list < $tar
401# echo
402# ls -l --si $tar
403# echo
404# }
405
406# unpack-distfiles() {
407# local a_repo=$1 # 'main' or 'community'
408
409# local tar=_chroot/distfiles-${a_repo}.tar
410# sudo tar --verbose -x --directory $CHROOT_DIR/var/cache/distfiles < $tar
411# }
412
413
414# _install-hook() {
415# local bwrap=${1:-}
416
417# local out=enter-rootfs
418# ../alpine-chroot-install/alpine-chroot-install -g > $out
419# chmod +x $out
420# echo "Wrote $out"
421
422# local src
423# if test -n "$bwrap"; then
424# hook=regtest/aports/enter-hook-bwrap.sh
425# else
426# hook=../alpine-chroot-install/enter-hook-chroot
427# fi
428
429# cp -v $hook $CHROOT_DIR/enter-hook
430# }
431
432# install-hook() {
433# sudo $0 _install-hook "$@"
434# }
435
436# _install-enter-bwrap() {
437# # don't need this other bwrap script
438# rm -f -v $CHROOT_DIR/enter-hook
439
440# cp -v regtest/aports/enter-bwrap.sh $CHROOT_DIR
441# }
442
443# install-enter-bwrap() {
444# sudo $0 _install-enter-bwrap "$@"
445# }
446
447. regtest/debian-common.ysh
448
449proc remove-all() {
450 set -x
451 remove-chroot
452 remove-osh-overlay
453# remove-shard-layers || true
454}
455
456proc remove-chroot() {
457 try {
458 sudo umount -R $CHROOT_DIR 2>/dev/null
459 }
460 sudo rm -rf $CHROOT_DIR
461}
462
463proc clone-debootstrap() {
464 var dir = '../debianlinux'
465 mkdir -p $dir
466 rm -rf "$dir/debootstrap"
467 pushd $dir
468
469 # Took 1m 13s, at 27 MiB /ssec
470 git clone \
471 https://salsa.debian.org/installer-team/debootstrap.git
472
473 popd
474}
475
476proc make-chroot() {
477 var dir = '../debianlinux/debootstrap'
478 pushd $dir
479 var debootstrap_dir = $(pwd)
480
481 # This takes about 1m20s
482 # need to pass --arch, automatic detection doesnt work on non-debian systems
483 sudo DEBOOTSTRAP_DIR=$debootstrap_dir ./debootstrap --arch=amd64 stable $CHROOT_DIR http://deb.debian.org/debian/
484 popd
485}
486
487proc exec-chroot(cmd) {
488 sudo chroot $CHROOT_DIR /bin/bash -l -c $cmd
489}
490
491proc exec-chroot-osh(cmd) {
492 sudo chroot $DEBIAN_ROOT/osh-as-sh.overlay/merged /bin/bash -l -c $cmd
493}
494
495proc exec-chroot-user(cmd) {
496 sudo chroot $CHROOT_DIR su -l udu -c $cmd
497}
498
499proc add-build-deps() {
500 # takes around 2 minutes
501 exec-chroot "apt-get update; apt-get install doas dpkg-dev build-essential devscripts fakeroot apt-utils -y"
502}
503
504proc config-chroot() {
505 exec-chroot 'echo "deb-src http://deb.debian.org/debian stable main" >> /etc/apt/sources.list'
506 make-user
507 setup-doas
508}
509
510proc fetch-all() {
511 clone-debootstrap
512
513 download-oils
514}
515
516proc prepare-all() {
517 # $0 make-chroot
518 # $0 add-build-deps # add packages that build packages
519 # # 281 MB, 248 K files
520 # $0 config-chroot # user/groups, keygen
521 # $0 oils-in-chroot # copy-aports: 307 MB, 251 K files
522
523 make-chroot
524
525 add-build-deps
526
527 # TODO: mount /proc etc?
528 config-chroot
529
530 # TODO: implement enter-rootfs-user
531 oils-in-chroot
532
533 create-osh-overlay
534
535 # # makes a host file
536 # apk-manifest
537}
538
539runproc @ARGV