OILS / soil / host-shim.sh View on Github | oils.pub

442 lines, 252 significant
1#!/usr/bin/env bash
2#
3# Shell functions run on the host machine, OUTSIDE the container.
4#
5# Usage:
6# soil/host-shim.sh <function name>
7#
8# Examples:
9# soil/host-shim.sh local-test-uke cpp-spec # run a job
10# soil/host-shim.sh local-test-uke cpp-spec '' T # get shell
11
12set -o nounset
13set -o pipefail
14set -o errexit
15
16REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
17
18source soil/common.sh
19source test/tsv-lib.sh
20
21readonly REGISTRY='ghcr.io'
22
23live-image-tag() {
24 ### image ID -> Docker tag name
25 local image_id=$1
26
27
28 case $image_id in
29 dummy)
30 # wedges 2025
31 echo 'v-2026-01-13'
32 ;;
33 *)
34 # ghcr.io migration
35 echo 'v-2025-12-21'
36 ;;
37
38 #
39 # UNUSED
40 #
41
42 app-tests)
43 # wedges 2025
44 echo 'v-2025-10-28'
45 ;;
46 wild)
47 # wedges 2025
48 echo 'v-2025-10-28'
49 ;;
50 bloaty)
51 # wedges 2025
52 echo 'v-2025-10-28'
53 ;;
54 benchmarks)
55 # wedges 2025
56 echo 'v-2025-10-28'
57 ;;
58 benchmarks2)
59 # wedges 2025
60 # rebuild cmark, NOT uftrace
61 echo 'v-2025-10-28'
62 ;;
63 cpp-spec)
64 # wedges 2025
65 echo 'v-2025-10-28'
66 ;;
67 pea)
68 # wedges 2025
69 echo 'v-2025-10-28'
70 ;;
71 cpp-small)
72 # wedges 2025
73 echo 'v-2025-10-28'
74 ;;
75 clang)
76 # wedges 2025
77 echo 'v-2025-10-28'
78 ;;
79 ovm-tarball)
80 # wedges 2025
81 echo 'v-2025-10-28'
82 ;;
83 other-tests)
84 # wedges 2025
85 echo 'v-2025-10-28'
86 ;;
87 dev-minimal)
88 # wedges 2025
89 echo 'v-2025-10-28'
90 ;;
91 *)
92 die "Invalid image $image"
93 ;;
94 esac
95}
96
97make-soil-dir() {
98 log-context 'make-soil-dir'
99
100 mkdir --verbose -p _tmp/soil
101 ls -l -d . _tmp _tmp/soil
102
103 # Match what mount-perms does
104 chmod --changes 777 _tmp _tmp/soil
105 ls -l -d . _tmp _tmp/soil
106}
107
108show-disk-info() {
109 # Debug 'no space left on device' issue
110 echo 'DISKS'
111 df -h
112 echo
113
114 # Useful but many permissions errors
115 if false; then
116 echo 'SPACE FOR IMAGES?'
117 du --si -s ~/.local/share/ || true
118 echo
119 fi
120}
121
122podman-prune() {
123 ### Should this work on Debian?
124
125 if ! command -v podman; then
126 echo 'no podman'
127 return
128 fi
129
130 echo 'IMAGES'
131 podman images --all
132 echo
133
134 if false; then
135 # This causes an interactive prompt
136 echo 'PRUNE'
137 podman system prune || true
138 echo
139
140 show-disk-info
141
142 echo 'PRUNE AS ROOT'
143 sudo podman system prune || true
144 echo
145
146 show-disk-info
147 fi
148}
149
150mount-perms() {
151 ### Ensure that the guest can write to bind mount
152
153 local repo_root=$1
154
155 #show-disk-info
156
157 log-context 'mount-perms'
158
159 # We have to chmod all dirs because 'build/py.sh all' creates
160 # build/temp.linux-*, for example. Also can't exclude .git/ because
161 # submodules need it.
162 time find "$repo_root" -type d -a -print \
163 | xargs -d $'\n' -- chmod --changes 777 \
164 | wc -l
165 echo
166}
167
168job-reset() {
169 ### Called between jobs
170
171 #show-disk-info
172
173 log-context 'job-reset'
174
175 # The VM runs as the 'build' user on sourcehut. The podman container runs as
176 # 'uke' user, which apparently gets UID 100999.
177 #
178 # Running as 'build', we can't remove files created by the guest, so use
179 # 'sudo'.
180 #
181 # It's really these three dirs.
182 # ls -l -d _tmp/soil _tmp/soil/logs _devbuild/bin || true
183
184 sudo $0 mount-perms $PWD
185 echo
186
187 git status .
188 echo
189
190 # Similar to functions in 'build/clean.sh'
191 local -a dirs=(_tmp _bin _build _devbuild _test)
192 #local -a dirs=(_tmp)
193
194 log 'Removing temp dirs'
195 log ''
196
197 du --si -s "${dirs[@]}" || true
198 rm -r -f "${dirs[@]}"
199 echo
200
201 show-disk-info
202}
203
204image-layers-tsv() {
205 local docker=${1:-podman}
206 local image=${2:-oils-for-unix/soil-dummy}
207 local tag=${3:-latest}
208
209 # --human=0 gives us raw bytes and ISO timestamps
210 # --no-trunc shows the full command line
211
212 # Removed .CreatedAt because of this issue
213 # https://github.com/containers/podman/issues/17738
214 #echo $'num_bytes\tcreated_at\tcreated_by'
215
216 echo $'num_bytes\tcreated_by'
217
218 $docker history \
219 --no-trunc --human=0 \
220 --format json \
221 $REGISTRY/$image:$tag | python3 -c '
222import sys, json
223array = json.load(sys.stdin)
224for row in array:
225 print("%s\t%s" % (row["size"], row["CreatedBy"]))
226'
227}
228
229save-image-stats() {
230 local soil_dir=${1:-_tmp/soil}
231 local docker=${2:-podman}
232 local image=${3:-oils-for-unix/soil-dummy}
233 local tag=${4:-latest}
234
235 log "save-image-stats with $(which $docker)"
236 $docker --version
237 echo
238
239 mkdir -p $soil_dir
240
241 # NOTE: Works on my dev machine, but produces an empty table on CI?
242 $docker images "$REGISTRY/$image:v-*" > $soil_dir/images-tagged.txt
243 log "Wrote $soil_dir/images-tagged.txt"
244
245 $docker history $REGISTRY/$image:$tag > $soil_dir/image-layers.txt
246 log "Wrote $soil_dir/image-layers.txt"
247
248 image-layers-tsv $docker $image $tag > $soil_dir/image-layers.tsv
249 log "Wrote $soil_dir/image-layers.tsv"
250
251 # TODO: sum into image-layers.json
252 # - total size
253 # - earliest and layer date?
254
255 here-schema-tsv >$soil_dir/image-layers.schema.tsv <<EOF
256column_name type
257num_bytes integer
258created_at string
259created_by string
260EOF
261
262 log "Wrote $soil_dir/image-layers.schema.tsv"
263}
264
265# for both ASAN/LeakSanitizer and GDB
266readonly PTRACE_FLAGS=( --cap-add=SYS_PTRACE --security-opt seccomp=unconfined )
267
268run-job-uke() {
269 local docker=$1 # docker or podman
270 local repo_root=$2
271 local job_name=$3 # e.g. dev-minimal
272 local debug_shell=${4:-}
273
274 log-context 'run-job-uke'
275
276 # Do this on the HOST because we write the pull time into it as well. It's
277 # shared between guest and host.
278 make-soil-dir
279 local soil_dir=$repo_root/_tmp/soil
280
281 local image_id=$job_name
282 # Some jobs don't have their own image, and some need docker -t
283 case $job_name in
284 app-tests)
285 ;;
286 cpp-coverage)
287 image_id='clang'
288 ;;
289 cpp-tarball)
290 image_id='cpp-small'
291 ;;
292 benchmarks3)
293 # for test/syscall
294 #image_id='ovm-tarball'
295
296 # TODO: use wait-for-tarball
297 #image_id='benchmarks2'
298
299 # Ninja build
300 image_id='benchmarks'
301 ;;
302 interactive)
303 # Reuse for now
304 image_id='benchmarks'
305 ;;
306 esac
307
308 local -a flags=()
309 case $job_name in
310 app-tests|interactive)
311 # app-tests: to run ble.sh tests
312 # interactive: to run 'interactive-osh' with job control enabled
313 flags+=( -t )
314 ;;
315 esac
316
317 case $job_name in
318 cpp-small|cpp-spec|cpp-tarball|ovm-tarball)
319 # podman requires an additional flag for ASAN, so it can use ptrace()
320 # Otherwise we get:
321 # ==1194==LeakSanitizer has encountered a fatal error.
322 # ==1194==HINT: LeakSanitizer does not work under ptrace (strace, gdb, etc)
323 #
324 # Note: somehow this isn't necessary locally: on podamn 4.3.1 on Debian 12
325 if test $docker = podman; then
326 flags+=( "${PTRACE_FLAGS[@]}" )
327 fi
328 esac
329
330 local image="ghcr.io/oils-for-unix/soil-$image_id"
331
332 local tag=$(live-image-tag $image_id)
333
334 local pull_status
335 # Use external time command in POSIX format, so it's consistent between hosts
336 set -o errexit
337 command time -p -o $soil_dir/image-pull-time.txt \
338 $docker pull $REGISTRY/$image:$tag
339 pull_status=$?
340 set +o errexit
341
342 if test $pull_status -ne 0; then
343 log "$docker pull failed with status $pull_status"
344
345 # Save status for a check later
346 mkdir -p _soil-jobs
347 echo "$pull_status" > _soil-jobs/$job_name.status.txt
348
349 # Return success
350 return
351 fi
352
353 save-image-stats $soil_dir $docker $image $tag
354
355 show-disk-info
356
357 podman-prune
358
359 local -a args
360 if test -n "$debug_shell"; then
361 # launch interactive shell
362 flags+=( -i -t )
363
364 # So we can run GDB
365 # https://stackoverflow.com/questions/35860527/warning-error-disabling-address-space-randomization-operation-not-permitted
366 flags+=( "${PTRACE_FLAGS[@]}" )
367
368 args=(bash)
369 else
370 args=(sh -c "cd /home/uke/oil; soil/worker.sh JOB-$job_name")
371 fi
372
373 set -x
374 $docker run "${flags[@]}" \
375 --mount "type=bind,source=$repo_root,target=/home/uke/oil" \
376 $REGISTRY/$image:$tag \
377 "${args[@]}"
378 set +x
379}
380
381did-all-succeed() {
382 ### Check if the given jobs succeeded
383
384 local max_status=0
385 for job_name in "$@"; do
386 local status
387 read status unused_job_id < "_soil-jobs/$job_name.status.txt"
388
389 echo "$job_name status: $status"
390 if test $status -gt $max_status; then
391 max_status=$status
392 fi
393 done
394
395 log ''
396 log "Exiting with max job status $max_status"
397
398 return "$max_status"
399}
400
401local-test-uke() {
402 ### Something I can run locally. This is fast.
403
404 # Simulate sourcehut with 'local-test-uke dummy dummy'
405 local job_name=${1:-dummy}
406 local job2=${2:-}
407 local debug_shell=${3:-} # add 'bash' to change it to a debug shell
408 local docker=${4:-podman}
409
410 local branch=$(git rev-parse --abbrev-ref HEAD)
411
412 local fresh_clone=/tmp/soil-$job_name
413 rm -r -f -v $fresh_clone
414
415 local this_repo=$PWD
416 git clone $this_repo $fresh_clone
417 cd $fresh_clone
418 git checkout $branch
419
420 sudo $0 mount-perms $fresh_clone
421 sudo $0 run-job-uke "$docker" $fresh_clone $job_name "$debug_shell"
422
423 # Run another job in the same container, to test interactions
424
425 if test -n "$job2"; then
426 $0 job-reset
427 sudo $0 run-job-uke "$docker" $fresh_clone $job2
428 fi
429}
430
431local-shell() {
432 local job_name=${1:-cpp}
433
434 # no job 2
435 local-test-uke $job_name '' bash
436}
437
438cleanup() {
439 sudo rm -r -f -v _tmp/soil /tmp/soil-*
440}
441
442"$@"