OILS / deps / image.sh View on Github | oils.pub

588 lines, 222 significant
1#!/usr/bin/env bash
2#
3# Manage container images for Soil
4#
5# Usage:
6# deps/image.sh <function name>
7#
8# Dirs maybe to clear:
9#
10# $0 clean-all # Start from scratch
11#
12# Example:
13#
14# (1) Update LATEST_TAG
15#
16# (2) Bootstrap Wedges
17#
18# $0 build wedge-bootstrap-debian-12 # runs init-deb-cache
19#
20# (3) Build wedges
21#
22# build/deps.sh fetch
23# build/deps.sh boxed-wedges-2025
24#
25# (4) Rebuild an image
26#
27# $0 build soil-debian-12 # runs init-deb-cache and populate the cache
28# $0 build soil-test-image T # reuse package cache from apt-get
29# $0 smoke soil-test-image # smoke test
30#
31# (5) Update live version in 'soil/host-shim.sh live-image-tag'
32#
33# (6) Push Everything you Built
34#
35# # pushes both $LATEST_TAG and latest
36# $0 push-many wedge-bootstrap-debian-12 soil-debian-12 soil-test-image
37#
38# More
39# ----
40#
41# Images: https://hub.docker.com/u/oilshell
42#
43# $0 list-tagged # Show versions of images
44#
45# $0 show-cachemount # show files in apt cache
46#
47# $0 prune # seems to clear the cache
48
49set -o nounset
50set -o pipefail
51set -o errexit
52
53source deps/podman.sh # do we need this?
54
55DOCKER=${DOCKER:-podman}
56
57declare -a docker_prefix
58case $DOCKER in
59 docker)
60 # Do we still need -E to preserve CONTAINERS_REGISTRIES_CONF?
61 docker_prefix=( sudo -E DOCKER_BUILDKIT=1 $DOCKER )
62 ;;
63 podman)
64 # this can be rootless! Unlike deps/wedge.sh
65 docker_prefix=( podman )
66 ;;
67 *)
68 die "Invalid docker $DOCKER"
69 ;;
70esac
71
72readonly LATEST_TAG='v-2025-11-02' # podman
73
74clean-all() {
75 dirs='_build/wedge/tmp _build/wedge/binary _build/deps-source'
76 #rm -r -f $dirs
77 sudo rm -r -f $dirs
78}
79
80list() {
81 local which=${1:-all} # all | soil | prep
82
83 local accept=''
84 local reject=''
85 case $which in
86 all)
87 reject='^$'
88 ;;
89 soil) # 13 soil images
90 reject='^(wedge-bootstrap-.*|soil-debian-.*)'
91 ;;
92 prep)
93 # images to prepare
94 # 2025-10: *-debian-10 is Debian Buster from 2019, which was retired in
95 # 2024. You can't do sudo apt-get update
96 # https://wiki.debian.org/DebianReleases
97 accept='^(wedge-bootstrap-debian-12|soil-debian-12)'
98 ;;
99 esac
100
101 if test -n "$accept"; then
102 for name in deps/Dockerfile.*; do
103 local image_id=${name//'deps/Dockerfile.'/}
104 if [[ "$image_id" =~ $accept ]]; then
105 echo $image_id
106 fi
107 done
108 else
109 for name in deps/Dockerfile.*; do
110 local image_id=${name//'deps/Dockerfile.'/}
111 if [[ "$image_id" =~ $reject ]]; then
112 continue
113 fi
114 echo $image_id
115 done
116 fi
117}
118
119list-tagged() {
120 "${docker_prefix[@]}" \
121 images 'oilshell/*' #:v-*'
122}
123
124_latest-one() {
125 local name=$1
126 "${docker_prefix[@]}" \
127 images "oilshell/$name" | head -n 3
128}
129
130_list-latest() {
131 # Should rebuild all these
132 # Except I also want to change the Dockerfile to use Debian 12
133 list-images | xargs -n 1 -- $0 _latest-one
134}
135
136list-latest() {
137 sudo $0 _list-latest
138}
139
140# BUGS in Docker.
141#
142# https://stackoverflow.com/questions/69173822/docker-build-uses-wrong-dockerfile-content-bug
143
144# NOTE: This also clears the exec.cachemount
145prune() {
146 sudo $DOCKER builder prune -f
147}
148
149# https://stackoverflow.com/questions/62834806/docker-buildkit-cache-location-size-and-id
150#
151# It lives somewhere in /var/lib/docker/overlay2
152
153show-cachemount() {
154 sudo $DOCKER system df -v --format '{{ .BuildCache | json }}' \
155 | jq '.[] | select(.CacheType == "exec.cachemount")' | tee _tmp/cachemount.txt
156
157 cat _tmp/cachemount.txt | jq -r '.ID' | while read id; do
158 sudo tree /var/lib/docker/overlay2/$id
159 sudo du --si -s /var/lib/docker/overlay2/$id
160 echo
161 done
162}
163
164tag-latest() {
165 local name=${1:-wedge-bootstrap-debian-12}
166 local tag_built_with=${2:-$LATEST_TAG}
167
168 set -x # 'docker tag' is annoyingly silent
169 "${docker_prefix[@]}" \
170 tag oilshell/$name:{$tag_built_with,latest}
171}
172
173build() {
174 local name=${1:-soil-dummy}
175
176 # OFF by default. TODO: use_cache setting should be automatic
177 local use_cache=${2:-}
178
179 # set -x
180 local -a flags
181 if test -n "$use_cache"; then
182 flags=()
183 else
184 flags=('--no-cache=true')
185 fi
186 #flags+=('--progress=plain')
187
188 # Uh BuildKit is not the default on Linux!
189 # http://jpetazzo.github.io/2021/11/30/docker-build-container-images-antipatterns/
190 #
191 # It is more parallel and has colored output.
192
193 # TODO: use --authfile and more
194 #export-podman
195
196 # can't preserve the entire env: https://github.com/containers/buildah/issues/3887
197 #sudo --preserve-env=CONTAINERS_REGISTRIES_CONF --preserve-env=REGISTRY_AUTH_FILE \
198 "${docker_prefix[@]}" \
199 build "${flags[@]}" \
200 --tag "oilshell/$name:$LATEST_TAG" \
201 --file deps/Dockerfile.$name .
202
203 # Avoid hassle by also tagging it
204 tag-latest $name
205}
206
207push() {
208 local name=${1:-soil-dummy}
209 local tag=${2:-$LATEST_TAG}
210
211 # TODO: replace with flags
212 #export-podman
213
214 local image="oilshell/$name:$tag"
215
216 set -x
217
218 # -E for export-podman vars
219 "${docker_prefix[@]}" \
220 push $image
221 #sudo -E $DOCKER --log-level=debug push $image
222
223 # Also push the 'latest' tag, to avoid getting out of sync
224 "${docker_prefix[@]}" \
225 push oilshell/$name:latest
226}
227
228push-many() {
229 for name in "$@"; do
230 push $name
231 done
232}
233
234smoke-script-1() {
235 echo '
236for file in /etc/debian_version /etc/lsb-release; do
237 if test -f $file; then
238 # spec/ble-idioms tests this
239 #grep -E "foo|^10" $file; echo grep=$?
240
241 echo $file
242 echo
243 cat $file
244 echo
245 else
246 echo "($file does not exist)"
247 fi
248done
249
250echo "bash $BASH_VERSION"
251
252git --version
253
254for name in python python2 python3; do
255 if which $name; then
256 $name -V
257 else
258 echo "$name not found"
259 fi
260done
261
262echo PATH=$PATH
263
264#curl https://op.oilshell.org/
265
266if true; then
267 python2 -c "import readline; print(readline)"
268 echo
269
270 python3 -c "import ssl; print(ssl)"
271 echo
272
273 find /usr/lib | grep -i readline
274 echo
275
276 ls /wedge/oils-for-unix.org/pkg/python2/
277 ls /wedge/oils-for-unix.org/pkg/python2/2.7.18/lib/python2.7/lib-dynload
278
279 ldd /wedge/oils-for-unix.org/pkg/python2/2.7.18/lib/python2.7/lib-dynload/readline.so
280 echo
281
282 exit
283
284 dpkg -S /usr/lib/x86_64-linux-gnu/libssl.so.3
285 echo
286
287 #ls -l /usr/lib/x86_64-linux-gnu/libssl.so.1.1
288
289 apt-cache show libssl-dev
290
291 # find /lib | grep -i libssl
292 # echo
293 # find /usr/local | grep -i libssl
294 # echo
295 # python3-config --libs
296
297 # Useful command
298 # ldconfig -v #| grep ssl
299 # echo
300
301 #find / -name 'libssl.so*'
302fi
303'
304}
305
306smoke-script-2() {
307 echo '
308 cd ~/oil
309 . build/dev-shell.sh
310
311 test/ltrace.sh soil-run
312 exit
313
314 # Bug with python2
315 #devtools/types.sh soil-run
316 #test/lossless.sh soil-run
317 #exit
318
319 #test/spec-version.sh osh-version-text
320
321 echo PATH=$PATH
322
323 #which mksh
324 #mksh -c "echo hi from mksh"
325
326 #test/spec.sh smoke
327 test/spec.sh zsh-assoc
328
329 which python2
330 python2 -V
331 echo
332
333 which python3
334 python3 -V
335 echo
336
337 exit
338
339 # Bug with python2
340 test/lossless.sh soil-run
341
342 python3 -m mypy core/util.py
343 echo
344
345 # test pyflakes
346 test/lint.sh py2-lint core/util.py
347 echo
348
349 #pea/TEST.sh parse-all
350 #pea/TEST.sh run-tests
351
352 re2c --version
353 echo
354
355 # cmark.py
356 doctools/cmark.sh demo-ours
357
358 bloaty --help
359 echo
360
361 exit
362
363 # hm this shows Python
364 uftrace --version
365
366 which uftrace
367 uftrace=$(which uftrace)
368
369 ls -l ~/oils.DEPS/wedge/uftrace/0.13/bin/uftrace
370 uftrace=~/oils.DEPS/wedge/uftrace/0.13/bin/uftrace
371
372 devtools/R-test.sh soil-run
373
374 exit
375
376 cc -pg -o hello deps/source.medo/uftrace/hello.c
377
378 # libmcount-fast is in the uftrace lib/ dir
379 ldd $(which uftrace)
380 echo
381
382 set -x
383 #head /tmp/cache-bust.txt
384
385 $uftrace record hello
386 #uftrace replay hello
387 echo
388
389 #find /usr -name "libm*.so"
390 '
391}
392
393asan-smoke-script() {
394 echo '
395 cd ~/oil
396 pwd
397 whoami
398
399 # this failed with ASAN
400 yaks/TEST.sh soil-run
401 exit
402
403 build/py.sh all
404 ./NINJA-config.sh
405 bin=_bin/cxx-asan/mycpp/examples/varargs.mycpp
406 ninja $bin
407 $bin
408
409 exit
410
411 # ls -l
412 #mkdir -p _devbuild/bin
413 out=_devbuild/bin/th.asan
414
415 build/py.sh time-helper $out -fsanitize=address
416 export ASAN_OPTIONS=detect_leaks=1
417 ldd $out
418 ls -l $out
419 $out
420 exit
421 '
422}
423
424_smoke() {
425 ### Smoke test of container
426 local name=${1:-soil-dummy}
427 local tag=${2:-$LATEST_TAG}
428 local docker=${3:-$DOCKER}
429 local script_func=${4:-} # if empty, then start a debug shell
430
431 local repo_root=$PWD
432
433 local -a flags argv
434 if test -n "$script_func"; then
435 flags=()
436 argv=( bash -c "$("$script_func")" )
437 else
438 flags=( -i -t )
439 argv=( bash )
440 fi
441
442 # Stupid podman!
443 case $docker in
444 podman)
445 export-podman
446 # this fixes mount permissions!
447 flags+=( --userns=keep-id )
448 ;;
449 esac
450
451 $docker run "${flags[@]}" \
452 --mount "type=bind,source=$repo_root,target=/home/uke/oil" \
453 oilshell/$name:$tag "${argv[@]}"
454}
455
456smoke() {
457 sudo $0 _smoke "$@"
458}
459
460smoke-podman-asan() {
461 _smoke soil-cpp-small latest podman asan-smoke-script
462}
463
464smoke-podman() {
465 local name=${1:-dummy}
466
467 # 2025-04: I need to do 'podman login docker.io'
468 #
469 # Running without root
470
471 # need explicit docker.io prefix with podman
472 # smoke $name latest podman docker.io/
473
474 local tag='latest'
475 local prefix='docker.io/'
476 smoke-test-script | podman run -i ${prefix}oilshell/soil-$name:$tag bash
477}
478
479cmd() {
480 ### Run an arbitrary command
481 local name=${1:-soil-dummy}
482 local tag=${2:-$LATEST_TAG}
483
484 shift 2
485
486 sudo $DOCKER run oilshell/$name:$tag "$@"
487}
488
489utf8() {
490 # needed for a spec test, not the default on Debian
491 cmd ovm-tarball bash -c 'LC_ALL=en_US.UTF-8; echo $LC_ALL'
492}
493
494mount-test() {
495 local name=${1:-soil-dummy}
496
497 local -a argv
498 if test $# -le 1; then
499 argv=(sh -c 'ls -l /home/uke/oil')
500 else
501 argv=( "${@:2}" ) # index 2 not 1, weird shell behavior
502 fi
503
504 # mount 'oil' directory as /app. TODO: Oils
505 sudo $DOCKER run \
506 --mount "type=bind,source=$PWD,target=/home/uke/oil" \
507 oilshell/$name "${argv[@]}"
508}
509
510image-history() {
511 local image_id=${1:-soil-dummy}
512 local tag=${2:-latest}
513
514 local image="oilshell/$image_id"
515
516 sudo $DOCKER history $image:$tag
517}
518
519save() {
520 local image_id=${1:-soil-dummy}
521 local tag=${2:-latest}
522
523 local image="oilshell/$image_id"
524
525 mkdir -p _tmp/images
526 local out=_tmp/images/$image_id.tar
527
528 # Use > instead of -o so it doesn'th have root permissions
529 time sudo $DOCKER save $image:$tag > $out
530 ls -l -h $out
531}
532
533# This shows CREATED, command CREATED BY, size
534# It's a human readable size though
535#
536# This doesn't really have anything better
537# https://gist.github.com/MichaelSimons/fb588539dcefd9b5fdf45ba04c302db6
538#
539# It's annoying that the remote registry API is different than the local API.
540
541layers() {
542 local name=${1:-soil-dummy}
543 local tag=${2:-$LATEST_TAG}
544
545 local image="oilshell/$name:$tag"
546
547 # Gah this still prints 237M, not the exact number of bytes!
548 # --format ' {{ .Size }} '
549 sudo $DOCKER history --no-trunc $image
550
551 echo $'Size\tVirtual Size'
552 sudo $DOCKER inspect $image \
553 | jq --raw-output '.[0] | [.Size, .VirtualSize] | @tsv' \
554 | commas
555}
556
557todo-debian-12() {
558 # 7 images
559 grep soil-common deps/Dockerfile.*
560}
561
562todo-purity() {
563 # TODO: we should pass --network none in $0 build
564 #
565 # Hm 7 images need pip download, should reduce them
566 #
567 # There are other sources of impurity, like:
568 # building the R-libs wedge
569 # soil-wild - we can't download the testdata, etc.
570
571 grep -l install-py3-libs deps/Dockerfile.*
572}
573
574todo-tree-shake() {
575 # We should invoke OSH to generate parts of the dockerfile? Or use podman
576 # probably?
577 #
578 # Or maybe it's a default layer in soil-debian-12?
579
580 grep task-five deps/Dockerfile.*
581}
582
583todo-relative() {
584 grep TODO # _build/wedge/relative, not _build/wedge/binary
585}
586
587"$@"
588