OILS / soil / host-shim.sh View on Github | oilshell.org

389 lines, 226 significant
1#!/usr/bin/env bash
2#
3# Shell functions run on the host machine, OUTSIDE the container.
4#
5# Usage:
6# soil/host-shim.sh <function name>
7#
8# Examples:
9# soil/host-shim.sh local-test-uke cpp-spec
10
11set -o nounset
12set -o pipefail
13set -o errexit
14
15REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
16
17source soil/common.sh
18source test/tsv-lib.sh
19
20live-image-tag() {
21 ### image ID -> Docker tag name
22 local image_id=$1
23
24 case $image_id in
25 (app-tests)
26 # rebuild with curl
27 echo 'v-2023-10-05'
28 ;;
29 (wild)
30 # rebuild with ca-certificates
31 echo 'v-2024-08-26'
32 ;;
33 (bloaty)
34 # rebuild with ca-certificates
35 echo 'v-2024-08-26'
36 ;;
37 (benchmarks)
38 # freshen up
39 echo 'v-2023-07-15'
40 ;;
41 (benchmarks2)
42 # debian 12, python3, new R-libs, cmark
43 # new uftrace version
44 echo 'v-2024-06-09'
45 ;;
46 (cpp-spec)
47 # Rebuild with jq, procps
48 echo 'v-2023-07-17'
49 ;;
50 (pea)
51 # freshen up
52 echo 'v-2024-06-08'
53 ;;
54 (cpp-small)
55 # Rebuild with Docker, remove dead code
56 echo 'v-2023-07-15'
57 ;;
58 (clang)
59 # Rebuild with wedges
60 echo 'v-2023-08-09'
61 ;;
62 (ovm-tarball)
63 # bash 5.2.21
64 echo 'v-2024-06-09b'
65 ;;
66 (other-tests)
67 # freshen up
68 echo 'v-2023-07-15'
69 ;;
70 (dummy)
71 # freshen up
72 echo 'v-2024-06-08'
73 ;;
74 (dev-minimal)
75 # Use python3 wedge and mypy-0.780 repo
76 echo 'v-2023-07-15'
77 ;;
78
79 # Not run directly
80 (common)
81 # Rebuild with wedges
82 echo 'v-2023-02-28f'
83 ;;
84 (*)
85 die "Invalid image $image"
86 ;;
87 esac
88}
89
90make-soil-dir() {
91 log-context 'make-soil-dir'
92
93 mkdir --verbose -p _tmp/soil
94 ls -l -d . _tmp _tmp/soil
95
96 # Match what mount-perms does
97 chmod --changes 777 _tmp _tmp/soil
98 ls -l -d . _tmp _tmp/soil
99}
100
101show-disk-info() {
102 # Debug 'no space left on device' issue
103 echo 'DISKS'
104 df -h
105 echo
106
107 # Useful but many permissions errors
108 if false; then
109 echo 'SPACE FOR IMAGES?'
110 du --si -s ~/.local/share/ || true
111 echo
112 fi
113}
114
115podman-prune() {
116 ### Should this work on Debian?
117
118 if ! command -v podman; then
119 echo 'no podman'
120 return
121 fi
122
123 echo 'IMAGES'
124 podman images --all
125 echo
126
127 if false; then
128 # This causes an interactive prompt
129 echo 'PRUNE'
130 podman system prune || true
131 echo
132
133 show-disk-info
134
135 echo 'PRUNE AS ROOT'
136 sudo podman system prune || true
137 echo
138
139 show-disk-info
140 fi
141}
142
143mount-perms() {
144 ### Ensure that the guest can write to bind mount
145
146 local repo_root=$1
147
148 #show-disk-info
149
150 log-context 'mount-perms'
151
152 # We have to chmod all dirs because 'build/py.sh all' creates
153 # build/temp.linux-*, for example. Also can't exclude .git/ because
154 # submodules need it.
155 time find "$repo_root" -type d -a -print \
156 | xargs -d $'\n' -- chmod --changes 777 \
157 | wc -l
158 echo
159}
160
161job-reset() {
162 ### Called between jobs
163
164 #show-disk-info
165
166 log-context 'job-reset'
167
168 # The VM runs as the 'build' user on sourcehut. The podman container runs as
169 # 'uke' user, which apparently gets UID 100999.
170 #
171 # Running as 'build', we can't remove files created by the guest, so use
172 # 'sudo'.
173 #
174 # It's really these three dirs.
175 # ls -l -d _tmp/soil _tmp/soil/logs _devbuild/bin || true
176
177 sudo $0 mount-perms $PWD
178 echo
179
180 git status .
181 echo
182
183 # Similar to functions in 'build/clean.sh'
184 local -a dirs=(_tmp _bin _build _devbuild _test)
185 #local -a dirs=(_tmp)
186
187 log 'Removing temp dirs'
188 log ''
189
190 du --si -s "${dirs[@]}" || true
191 rm -r -f "${dirs[@]}"
192 echo
193
194 show-disk-info
195}
196
197save-image-stats() {
198 local soil_dir=${1:-_tmp/soil}
199 local docker=${2:-docker}
200 local image=${3:-oilshell/soil-dummy}
201 local tag=${4:-latest}
202
203 # TODO: write image.json with the name and tag?
204
205 mkdir -p $soil_dir
206
207 # NOTE: Works on my dev machine, but produces an empty table on CI?
208 $docker images "$image:v-*" > $soil_dir/images-tagged.txt
209 log "Wrote $soil_dir/images-tagged.txt"
210
211 $docker history $image:$tag > $soil_dir/image-layers.txt
212 log "Wrote $soil_dir/image-layers.txt"
213
214 # NOTE: Works with docker but not podman! podman doesn't support --format ?
215 {
216 # --human=0 gives us raw bytes and ISO timestamps
217 # --no-trunc shows the full command line
218 echo $'num_bytes\tcreated_at\tcreated_by'
219 $docker history --no-trunc --human=0 --format '{{.Size}}\t{{.CreatedAt}}\t{{.CreatedBy}}' $image:$tag
220 } > $soil_dir/image-layers.tsv
221 log "Wrote $soil_dir/image-layers.tsv"
222
223 # TODO: sum into image-layers.json
224 # - total size
225 # - earliest and layer date?
226
227 here-schema-tsv >$soil_dir/image-layers.schema.tsv <<EOF
228column_name type
229num_bytes integer
230created_at string
231created_by string
232EOF
233
234 log "Wrote $soil_dir/image-layers.schema.tsv"
235}
236
237run-job-uke() {
238 local docker=$1 # docker or podman
239 local repo_root=$2
240 local job_name=$3 # e.g. dev-minimal
241 local debug_shell=${4:-}
242
243 log-context 'run-job-uke'
244
245 # Do this on the HOST because we write the pull time into it as well. It's
246 # shared between guest and host.
247 make-soil-dir
248 local soil_dir=$repo_root/_tmp/soil
249
250 local -a flags=()
251
252 local image_id=$job_name
253
254 # Some jobs don't have their own image, and some need docker -t
255 case $job_name in
256 app-tests)
257 # to run ble.sh tests
258 flags=( -t )
259 ;;
260 cpp-coverage)
261 image_id='clang'
262 ;;
263 cpp-tarball)
264 image_id='cpp-small'
265 ;;
266 interactive)
267 # to run 'interactive-osh' with job control enabled
268 flags=( -t )
269
270 # Reuse for now
271 image_id='benchmarks'
272 ;;
273 esac
274
275 local image="docker.io/oilshell/soil-$image_id"
276
277 local tag=$(live-image-tag $image_id)
278
279 local pull_status
280 # Use external time command in POSIX format, so it's consistent between hosts
281 set -o errexit
282 command time -p -o $soil_dir/image-pull-time.txt \
283 $docker pull $image:$tag
284 pull_status=$?
285 set +o errexit
286
287 if test $pull_status -ne 0; then
288 log "$docker pull failed with status $pull_status"
289
290 # Save status for a check later
291 mkdir -p _soil-jobs
292 echo "$pull_status" > _soil-jobs/$job_name.status.txt
293
294 # Return success
295 return
296 fi
297
298 save-image-stats $soil_dir $docker $image $tag
299
300 show-disk-info
301
302 podman-prune
303
304 local -a args
305 if test -n "$debug_shell"; then
306 # launch interactive shell
307 flags+=( -i -t )
308
309 # So we can run GDB
310 # https://stackoverflow.com/questions/35860527/warning-error-disabling-address-space-randomization-operation-not-permitted
311 flags+=( --cap-add SYS_PTRACE --security-opt seccomp=unconfined )
312
313 # can mount other tools for debugging, like clang
314 #local clang_dir=~/git/oilshell/oil_DEPS/clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04
315 #flags+=( --mount "type=bind,source=$clang_dir,target=/home/uke/oil_DEPS/$(basename $clang_dir)" )
316
317 args=(bash)
318 else
319 args=(sh -c "cd /home/uke/oil; soil/worker.sh JOB-$job_name")
320 fi
321
322 $docker run "${flags[@]}" \
323 --mount "type=bind,source=$repo_root,target=/home/uke/oil" \
324 $image:$tag \
325 "${args[@]}"
326}
327
328did-all-succeed() {
329 ### Check if the given jobs succeeded
330
331 local max_status=0
332 for job_name in "$@"; do
333 local status
334 read status unused_job_id < "_soil-jobs/$job_name.status.txt"
335
336 echo "$job_name status: $status"
337 if test $status -gt $max_status; then
338 max_status=$status
339 fi
340 done
341
342 log ''
343 log "Exiting with max job status $max_status"
344
345 return "$max_status"
346}
347
348local-test-uke() {
349 ### Something I can run locally. This is fast.
350
351 # Simulate sourcehut with 'local-test-uke dummy dummy'
352 local job_name=${1:-dummy}
353 local job2=${2:-}
354 local debug_shell=${3:-} # add 'bash' to change it to a debug shell
355 local docker=${4:-docker}
356
357 local branch=$(git rev-parse --abbrev-ref HEAD)
358
359 local fresh_clone=/tmp/soil-$job_name
360 rm -r -f -v $fresh_clone
361
362 local this_repo=$PWD
363 git clone $this_repo $fresh_clone
364 cd $fresh_clone
365 git checkout $branch
366
367 sudo $0 mount-perms $fresh_clone
368 sudo $0 run-job-uke "$docker" $fresh_clone $job_name "$debug_shell"
369
370 # Run another job in the same container, to test interactions
371
372 if test -n "$job2"; then
373 $0 job-reset
374 sudo $0 run-job-uke "$docker" $fresh_clone $job2
375 fi
376}
377
378local-shell() {
379 local job_name=${1:-cpp}
380
381 # no job 2
382 local-test-uke $job_name '' bash
383}
384
385cleanup() {
386 sudo rm -r -f -v _tmp/soil /tmp/soil-*
387}
388
389"$@"