OILS / regtest / debian-setup.ysh View on Github | oils.pub

537 lines, 39 significant
1#!bin/ysh
2
3#
4# Set up Debian Linux chroot. See regtest/aports.md
5#
6# Usage:
7# regtest/aports-setup.sh <function name>
8
9# : ${LIB_OSH=stdlib/osh}
10# source $LIB_OSH/bash-strict.sh
11# source $LIB_OSH/task-five.sh
12
13# source regtest/aports-common.sh
14
15# clone-aports() {
16# local dir=../../alpinelinux
17
18# mkdir -p $dir
19# pushd $dir
20
21# # Took 1m 13s, at 27 MiB /ssec
22# time git clone \
23# https://gitlab.alpinelinux.org/alpine/aports.git || true
24# #git@gitlab.alpinelinux.org:alpine/aports.git || true
25
26# popd
27# }
28
29# clone-aci() {
30# # I FORKED this, because this script FUCKED UP my /dev dir and current directory!
31# # Sent patches upstream
32
33# pushd ..
34
35# time git clone \
36# git@github.com:oils-for-unix/alpine-chroot-install || true
37
38# popd
39# }
40
41# checkout-stable() {
42
43# pushd ../../alpinelinux/aports
44
45# # Stable release branch like 3.22-stable
46# # The branch is frqeuently patched, but I guess it just needs to match the
47# # dl-cdn.alpinelinux.org URL
48# # https://alpinelinux.org/releases/
49
50# local branch='3.22-stable'
51# git checkout $branch
52# git log -n 1
53# popd > /dev/null
54
55# echo
56# echo
57
58# pushd ../alpine-chroot-install
59# git checkout master
60# git log -n 1
61# popd > /dev/null
62# }
63
64# patch-aports() {
65# local cur_dir=$PWD
66
67# pushd ../../alpinelinux/aports
68
69# for patch_dir in $cur_dir/regtest/patches/*; do
70# local package_name
71# package_name=$(basename $patch_dir)
72
73# local apkbuild=main/$package_name/APKBUILD
74# git restore $apkbuild
75
76# for mod_file in $cur_dir/regtest/patches/$package_name/*; do
77# echo
78# echo "*** Processing $mod_file"
79
80# case $mod_file in
81# *.patch)
82# # A patch to the source code
83
84# # Add our patches alongside Alpine's own patches
85
86# cp -v $mod_file main/$package_name/
87
88# # abuild's default_prepare() applies all patches inside $srcdir,
89# # but they also need to be specified in the 'source=' and
90# # 'sha512sums=' lists in APKBUILD
91# local patch_name=$(basename $mod_file)
92# local shasum=$(sha512sum $mod_file | cut -d ' ' -f1)
93# sed -i "/source='*/ a $patch_name" $apkbuild
94# sed -i "/sha512sums='*/ a $shasum $patch_name" $apkbuild
95# ;;
96
97# *.apkbuild)
98# # A patch to the APKBUILD file
99# set -x
100# git apply $mod_file
101# set +x
102# ;;
103
104# *.copy)
105# # A file to copy
106# cp -v $mod_file main/$package_name/
107# ;;
108# esac
109# done
110# done
111
112# popd >/dev/null
113# }
114
115# # 2025-11-16: fresh run
116# readonly TARBALL_ID='10856'
117
118# download-oils() {
119# local tarball_id=${1:-$TARBALL_ID}
120
121# local url="https://op.oilshell.org/uuu/github-jobs/$tarball_id/cpp-tarball.wwz/_release/oils-for-unix.tar"
122
123# rm -f -v _tmp/oils-for-unix.tar
124
125# #wget --no-clobber --directory-prefix _tmp "$url"
126# wget --directory-prefix _tmp "$url"
127# }
128
129# make-chroot() {
130# local aci='../alpine-chroot-install/alpine-chroot-install'
131
132# $aci --help
133
134# # Notes:
135# # - $aci -d requires an ABSOLUTE path. With a relative path, it creates
136# # _chroot/aports-build/_chroot/aports-build Filed bug upstream.
137# # - The -n flag is a feature I added: do not mount host dirs
138# # - TODO: when you run it twice, it should abort if the directory is full
139# # - Takes ~8 seconds
140
141# # default packages: build-base ca-certificates ssl_client
142# #
143# # This is already 267 MB, 247 K files
144
145# mkdir -p $CHROOT_DIR # make it with normal permissions first
146
147# # "branch" is one of these: https://dl-cdn.alpinelinux.org/
148# # it's not the aports branch
149# local branch='v3.22'
150# time sudo $aci -n -d $PWD/$CHROOT_DIR -b $branch
151# }
152
153# make-user() {
154# enter-rootfs adduser -D udu || true
155
156# # put it in abuild group
157# enter-rootfs addgroup udu abuild || true
158# # 'wheel' is for 'sudo'
159# enter-rootfs addgroup udu wheel || true
160
161# # CHeck the state
162# enter-rootfs-user sh -c 'whoami; echo GROUPS; groups'
163# }
164
165# setup-doas() {
166# # Manual configuration for abuild-keygen
167
168# #sudo cat _chroot/aports-build/etc/doas.conf
169# sudo rm -f $CHROOT_DIR/etc/doas.conf
170
171# # no password
172# enter-rootfs sh -c 'echo "permit nopass :wheel" >> /etc/doas.conf'
173# }
174
175# config-chroot() {
176# make-user
177# setup-doas
178# keygen
179# }
180
181# add-build-deps() {
182# # Must be done as root; there is no 'sudo'
183
184# # alpine-sdk: abuild, etc.
185# # abuild-rootbld: package required for 'abuild rootbld'
186# # pigz: seems like it's optionally used by abuild - should probably speed
187# # things up
188# # doas: for abuild-keygen
189# # bash python3: for time-tsv
190# # findutils: for xargs --process-slot-var
191# enter-rootfs sh -c '
192# apk update
193# apk add alpine-sdk abuild-rootbld pigz doas bash python3 findutils readline-dev
194# '
195
196# # enter-rootfs -u udu bash -c 'echo "hi from bash"'
197# }
198
199# change-perms() {
200# # pass any number of args
201
202# # get uid from /home/udu
203# local uid
204# uid=$(stat -c '%u' $CHROOT_HOME_DIR)
205# sudo chown --verbose --recursive $uid "$@"
206# }
207
208# copy-aports() {
209# # 'main' and 'community' are two "Alpine repos" stored the 'aports' git repo
210# for a_repo in main community; do
211# local dest=$CHROOT_HOME_DIR/aports/$a_repo/
212
213# sudo mkdir -p $dest
214# sudo rsync --archive --verbose \
215# ../../alpinelinux/aports/$a_repo/ $dest
216
217# change-perms $dest
218# done
219# }
220
221# _patch-yash-to-disable-tests() {
222# ### disable tests that use job control, causing SIGTTOU bug
223
224# local apkbuild=$CHROOT_HOME_DIR/aports/main/yash/APKBUILD
225
226# # make it idempotent
227# if ! grep 'FOR OILS' "$apkbuild"; then
228# echo '
229# check() {
230# echo "=== yash tests DISABLED FOR OILS ==="
231# }
232# ' >> $apkbuild
233# fi
234# }
235
236# patch-yash-to-disable-tests() {
237# sudo $0 _patch-yash-to-disable-tests
238# }
239
240# code-manifest() {
241# # TODO: need per-file tree shaking of build/py.sh
242# local -a build_py=(
243# build/py.sh # to compile time-helper
244
245# build/common.sh
246# build/dev-shell.sh
247# stdlib/osh/bash-strict.sh
248# stdlib/osh/byo-server.sh
249# stdlib/osh/task-five.sh
250# stdlib/osh/two.sh
251# )
252# for path in \
253# benchmarks/time_.py \
254# benchmarks/time-helper.c \
255# regtest/aports-guest.sh \
256# "${build_py[@]}"
257# do
258# echo "$PWD/$path" "$path"
259# done
260# }
261
262# multi-cp() {
263# ### like multi cp, but works without python2
264
265# local dest=$1
266# while read -r abs_path rel_path; do
267# # -D to make dirs
268
269# # Hack: make everything executable for now
270# # I feel like this should be in 'multi cp'
271
272# install -m 755 -v -D --no-target-directory "$abs_path" "$dest/$rel_path"
273
274# # cp -v --parents doesn't work, because it requires a directory arg
275# done
276# }
277
278# copy-code() {
279# local dest=$CHROOT_HOME_DIR/oils
280# sudo mkdir -v -p $dest
281
282# code-manifest | sudo $0 multi-cp $dest
283
284# change-perms $dest
285# }
286
287# test-time-tsv() {
288# enter-rootfs-user sh -c '
289# cd oils
290# pwd
291# whoami
292# echo ---
293
294# build/py.sh time-helper
295# regtest/aports-guest.sh my-time-tsv-test
296# '
297# }
298
299# oils-in-chroot() {
300# copy-aports
301
302# patch-yash-to-disable-tests
303
304# copy-code
305# test-time-tsv
306
307# copy-oils
308# build-oils
309# }
310
311# copy-oils() {
312# local dest=$CHROOT_HOME_DIR
313
314# local tar=$PWD/_tmp/oils-for-unix.tar
315# pushd $dest
316# sudo tar -x < $tar
317# popd
318
319# change-perms $dest/oils-for-unix-*
320# }
321
322# keygen() {
323# enter-rootfs-user sh -c '
324# #abuild-keygen -h
325
326# # -n non-interactive
327# abuild-keygen --append --install -n
328# '
329# }
330
331# apk-manifest() {
332# # 1643 files - find a subset to build
333
334# for a_repo in main community; do
335# local out=$PWD/_tmp/apk-${a_repo}-manifest.txt
336# mkdir -p _tmp
337
338# pushd $CHROOT_HOME_DIR/aports/$a_repo >/dev/null
339# find . -name 'APKBUILD' -a -printf '%P\n' | sed 's,/APKBUILD$,,g' | LANG=C sort | tee $out
340# popd >/dev/null
341# done
342# wc -l _tmp/apk-*-manifest.txt
343# }
344
345# build-oils() {
346# enter-rootfs-user sh -c '
347# cd oils-for-unix-*
348# ./configure
349# _build/oils.sh # do not --skip-rebuild
350# doas ./install
351# '
352# }
353
354# create-osh-overlay() {
355# # _chroot/
356# # aports-build/ # chroot image
357# # osh-as-sh.overlay/ # overlay
358# # merged/ # permanently mounted
359# # work/
360# # layer/ # has the OSH symlink
361
362# local osh_overlay=_chroot/osh-as-sh.overlay
363
364# mkdir -v -p $osh_overlay/{merged,work,layer}
365
366# # -o index=off fixes this error: fsconfig() failed: Stale file handle
367# # See also https://oilshell.zulipchat.com/#narrow/channel/522730-distros/topic/setting.20up.20regtest.2Faports-setup.2Esh/with/544318771
368# sudo mount \
369# -t overlay \
370# osh-as-sh \
371# -o index=off \
372# -o "lowerdir=$CHROOT_DIR,upperdir=$osh_overlay/layer,workdir=$osh_overlay/work" \
373# $osh_overlay/merged
374
375# $osh_overlay/merged/enter-chroot sh -c '
376# set -x
377# if ! test -f /usr/local/bin/oils-for-unix; then
378# echo "Build Oils first"
379# exit
380# fi
381# ln -s -f /usr/local/bin/oils-for-unix /bin/sh
382# ln -s -f /usr/local/bin/oils-for-unix /bin/ash
383# ln -s -f /usr/local/bin/oils-for-unix /bin/bash
384# ' dummy0
385# }
386
387# # Works?
388# patch-overlay-with-ash() {
389# local dir=_chroot/osh-as-sh.overlay
390# pushd $dir/layer/bin
391# sudo ln -s -f -v /usr/local/bin/oils-for-unix ash
392# popd
393# ls -l $dir/layer/bin
394# }
395
396# remove-osh-overlay() {
397# local dir=_chroot/osh-as-sh.overlay
398# sudo umount -l $dir/merged
399# sudo rm -r -f $dir
400# }
401
402# remove-shard-layers() {
403# sudo rm -r -f _chroot/shard*
404# }
405
406# make-distfiles-tar() {
407# local a_repo=$1 # 'main' or 'community'
408
409# local tar=_chroot/distfiles-${a_repo}.tar
410# tar --create --file $tar --directory $CHROOT_DIR/var/cache/distfiles .
411
412# tar --list < $tar
413# echo
414# ls -l --si $tar
415# echo
416# }
417
418# unpack-distfiles() {
419# local a_repo=$1 # 'main' or 'community'
420
421# local tar=_chroot/distfiles-${a_repo}.tar
422# sudo tar --verbose -x --directory $CHROOT_DIR/var/cache/distfiles < $tar
423# }
424
425# show-distfiles() {
426# ls -l $CHROOT_DIR/var/cache/distfiles
427# }
428
429# _install-hook() {
430# local bwrap=${1:-}
431
432# local out=enter-rootfs
433# ../alpine-chroot-install/alpine-chroot-install -g > $out
434# chmod +x $out
435# echo "Wrote $out"
436
437# local src
438# if test -n "$bwrap"; then
439# hook=regtest/aports/enter-hook-bwrap.sh
440# else
441# hook=../alpine-chroot-install/enter-hook-chroot
442# fi
443
444# cp -v $hook $CHROOT_DIR/enter-hook
445# }
446
447# install-hook() {
448# sudo $0 _install-hook "$@"
449# }
450
451# _install-enter-bwrap() {
452# # don't need this other bwrap script
453# rm -f -v $CHROOT_DIR/enter-hook
454
455# cp -v regtest/aports/enter-bwrap.sh $CHROOT_DIR
456# }
457
458# install-enter-bwrap() {
459# sudo $0 _install-enter-bwrap "$@"
460# }
461
462remove-all() {
463 set -x
464 remove-chroot || true
465# remove-osh-overlay || true
466# remove-shard-layers || true
467}
468
469proc remove-chroot() {
470 var chroot = $(pwd) ++ "/_debian_chroot"
471
472 try {
473 sudo umount -R $chroot 2>/dev/null
474 }
475 sudo rm -rf $chroot
476}
477
478proc clone-debootstrap() {
479 var dir = '../debianlinux'
480
481 mkdir -p $dir
482 pushd $dir
483
484 # Took 1m 13s, at 27 MiB /ssec
485 git clone \
486 https://salsa.debian.org/installer-team/debootstrap.git
487
488 popd
489}
490
491proc make-chroot() {
492 var dir = '../debianlinux/debootstrap'
493 var chroot = $(pwd) ++ "/_debian_chroot"
494 pushd $dir
495 var debootstrap_dir = $(pwd)
496
497 # This takes about 1m20s
498 # need to pass --arch, automatic detection doesnt work on non-debian systems
499 sudo DEBOOTSTRAP_DIR=$debootstrap_dir ./debootstrap --arch=amd64 stable $chroot http://deb.debian.org/debian/
500 popd
501}
502
503proc add-build-deps() {
504 var chroot = $(pwd) ++ "/_debian_chroot"
505 # takes around 2 minutes
506 sudo chroot $chroot /bin/bash -c "apt-get update; apt-get install dpkg-dev build-essential devscripts fakeroot apt-utils -y"
507}
508
509proc fetch-all() {
510 clone-debootstrap
511
512 # download-oils "$tarball_id"
513}
514
515proc prepare-all() {
516 # $0 make-chroot
517 # $0 add-build-deps # add packages that build packages
518 # # 281 MB, 248 K files
519 # $0 config-chroot # user/groups, keygen
520 # $0 oils-in-chroot # copy-aports: 307 MB, 251 K files
521
522 make-chroot
523
524 add-build-deps
525
526 # config-chroot -> add debsrc to /etc/apt/sources.list
527 # -> mount /proc, maybe more
528
529 # oils-in-chroot
530
531 # create-osh-overlay
532
533 # # makes a host file
534 # apk-manifest
535}
536
537runproc @ARGV