OILS / deps / image.sh View on Github | oils.pub

623 lines, 242 significant
1#!/usr/bin/env bash
2#
3# Manage container images for Soil
4#
5# Usage:
6# deps/image.sh <function name>
7#
8# Dirs maybe to clear:
9#
10# $0 clean-all # Start from scratch
11#
12# Example:
13#
14# (1) Update LATEST_TAG
15#
16# (2) Bootstrap Wedges
17#
18# $0 build wedge-bootstrap-debian-12 # runs init-deb-cache
19#
20# (3) Build wedges
21#
22# build/deps.sh fetch # tarballs
23# deps/from-binary.sh download-clang # needed for soil-clang image
24# deps/from-tar.sh download-wild # needed for soil-wild image
25# build/deps.sh boxed-wedges-2025 soil # soil wedges, not just contrib
26#
27# (4) Rebuild an image
28#
29# $0 build soil-debian-12 # runs init-deb-cache and populate the cache
30# $0 build soil-test-image T # reuse package cache from apt-get
31# $0 smoke soil-test-image # smoke test
32#
33# (5) Update live version in 'soil/host-shim.sh live-image-tag'
34#
35# (6) Push Everything you Built
36#
37# # pushes both $LATEST_TAG and latest
38# $0 push-many wedge-bootstrap-debian-12 soil-debian-12 soil-test-image
39#
40# More
41# ----
42#
43# Images: https://github.com/orgs/oils-for-unix/packages
44# formerly: https://hub.docker.com/u/oilshell
45#
46# $0 list-tagged # Show versions of images
47#
48# $0 show-cachemount # show files in apt cache
49#
50# $0 prune # seems to clear the cache
51
52set -o nounset
53set -o pipefail
54set -o errexit
55
56source deps/podman.sh # do we need this?
57
58DOCKER=${DOCKER:-podman}
59
60declare -a docker_prefix
61case $DOCKER in
62 docker)
63 # Do we still need -E to preserve CONTAINERS_REGISTRIES_CONF?
64 docker_prefix=( sudo -E DOCKER_BUILDKIT=1 $DOCKER )
65 ;;
66 podman)
67 # this can be rootless! Unlike deps/wedge.sh
68 docker_prefix=( podman )
69 ;;
70 *)
71 die "Invalid docker $DOCKER"
72 ;;
73esac
74
75readonly LATEST_TAG='v-2026-01-13' # rebuild with re2c 4.3.1 and ghcr.io
76
77clean-all() {
78 dirs='_build/wedge/tmp _build/wedge/binary _build/deps-source'
79 #rm -r -f $dirs
80 sudo rm -r -f $dirs
81}
82
83list() {
84 local which=${1:-all} # all | soil | prep
85
86 local accept=''
87 local reject=''
88 case $which in
89 all)
90 reject='^$'
91 ;;
92 soil) # 13 soil images
93 reject='^(wedge-bootstrap-.*|soil-debian-.*)'
94 ;;
95 prep)
96 # images to prepare
97 # 2025-10: *-debian-10 is Debian Buster from 2019, which was retired in
98 # 2024. You can't do sudo apt-get update
99 # https://wiki.debian.org/DebianReleases
100 accept='^(wedge-bootstrap-debian-12|soil-debian-12)'
101 ;;
102 esac
103
104 if test -n "$accept"; then
105 for name in deps/Dockerfile.*; do
106 local image_id=${name//'deps/Dockerfile.'/}
107 if [[ "$image_id" =~ $accept ]]; then
108 echo $image_id
109 fi
110 done
111 else
112 for name in deps/Dockerfile.*; do
113 local image_id=${name//'deps/Dockerfile.'/}
114 if [[ "$image_id" =~ $reject ]]; then
115 continue
116 fi
117 echo $image_id
118 done
119 fi
120}
121
122list-tagged() {
123 "${docker_prefix[@]}" \
124 images 'oils-for-unix/*' #:v-*'
125}
126
127_latest-one() {
128 local name=$1
129 "${docker_prefix[@]}" \
130 images "oils-for-unix/$name" | head -n 3
131}
132
133#
134# 2025-12: Migration to ghcr.io
135#
136
137migrate-one() {
138 local name=${1:-soil-wild}
139
140 # Good instructions from Claude!
141
142 # Pull from Docker Hub
143 podman pull docker.io/oilshell/$name:latest
144
145 # Tag for GitHub Container Registry with new name
146 podman tag docker.io/oilshell/$name:latest ghcr.io/oils-for-unix/$name:latest
147
148 # Push to GitHub Container Registry
149 podman push ghcr.io/oils-for-unix/$name:latest
150}
151
152migrate-all() {
153 list | xargs --verbose -n 1 -- $0 migrate-one
154}
155
156m-tag-one() {
157 ### Add $LATEST_TAG
158 local image=${1:-soil-wild}
159 podman tag ghcr.io/oils-for-unix/$image:latest ghcr.io/oils-for-unix/$image:$LATEST_TAG
160}
161
162m-tag-all() {
163 list | xargs --verbose -n 1 -- $0 m-tag-one
164}
165
166m-push-one() {
167 ### Push $LATEST_TAG
168 local name=${1:-soil-wild}
169 podman push ghcr.io/oils-for-unix/$name:$LATEST_TAG
170}
171
172m-push-all() {
173 list | xargs --verbose -n 1 -- $0 m-push-one
174}
175
176# BUGS in Docker.
177#
178# https://stackoverflow.com/questions/69173822/docker-build-uses-wrong-dockerfile-content-bug
179
180# NOTE: This also clears the exec.cachemount
181prune() {
182 sudo $DOCKER builder prune -f
183}
184
185# https://stackoverflow.com/questions/62834806/docker-buildkit-cache-location-size-and-id
186#
187# It lives somewhere in /var/lib/docker/overlay2
188
189show-cachemount() {
190 sudo $DOCKER system df -v --format '{{ .BuildCache | json }}' \
191 | jq '.[] | select(.CacheType == "exec.cachemount")' | tee _tmp/cachemount.txt
192
193 cat _tmp/cachemount.txt | jq -r '.ID' | while read id; do
194 sudo tree /var/lib/docker/overlay2/$id
195 sudo du --si -s /var/lib/docker/overlay2/$id
196 echo
197 done
198}
199
200tag-latest() {
201 local name=${1:-wedge-bootstrap-debian-12}
202 local tag_built_with=${2:-$LATEST_TAG}
203
204 set -x # 'docker tag' is annoyingly silent
205 "${docker_prefix[@]}" \
206 tag ghcr.io/oils-for-unix/$name:{$tag_built_with,latest}
207}
208
209build() {
210 local name=${1:-soil-dummy}
211
212 # OFF by default. TODO: use_cache setting should be automatic
213 local use_cache=${2:-}
214
215 # set -x
216 local -a flags
217 if test -n "$use_cache"; then
218 flags=()
219 else
220 flags=('--no-cache=true')
221 fi
222 #flags+=('--progress=plain')
223
224 # Uh BuildKit is not the default on Linux!
225 # http://jpetazzo.github.io/2021/11/30/docker-build-container-images-antipatterns/
226 #
227 # It is more parallel and has colored output.
228
229 # TODO: use --authfile and more
230 #export-podman
231
232 # can't preserve the entire env: https://github.com/containers/buildah/issues/3887
233 #sudo --preserve-env=CONTAINERS_REGISTRIES_CONF --preserve-env=REGISTRY_AUTH_FILE \
234 "${docker_prefix[@]}" \
235 build "${flags[@]}" \
236 --tag "ghcr.io/oils-for-unix/$name:$LATEST_TAG" \
237 --file deps/Dockerfile.$name .
238
239 # Avoid hassle by also tagging it
240 tag-latest $name
241}
242
243build-many() {
244 for name in "$@"; do
245 build $name T # use caching
246 done
247}
248
249push() {
250 local name=${1:-soil-dummy}
251 local tag=${2:-$LATEST_TAG}
252
253 local image="ghcr.io/oils-for-unix/$name"
254
255 set -x
256
257 "${docker_prefix[@]}" push "$image:$tag"
258
259 # Also push the 'latest' tag, to avoid getting out of sync
260 "${docker_prefix[@]}" push "$image:latest"
261}
262
263push-many() {
264 for name in "$@"; do
265 push $name
266 done
267}
268
269smoke-script-1() {
270 echo '
271for file in /etc/debian_version /etc/lsb-release; do
272 if test -f $file; then
273 # spec/ble-idioms tests this
274 #grep -E "foo|^10" $file; echo grep=$?
275
276 echo $file
277 echo
278 cat $file
279 echo
280 else
281 echo "($file does not exist)"
282 fi
283done
284
285echo "bash $BASH_VERSION"
286
287git --version
288
289for name in python python2 python3; do
290 if which $name; then
291 $name -V
292 else
293 echo "$name not found"
294 fi
295done
296
297echo PATH=$PATH
298
299#curl https://op.oilshell.org/
300
301if true; then
302 python2 -c "import readline; print(readline)"
303 echo
304
305 python3 -c "import ssl; print(ssl)"
306 echo
307
308 find /usr/lib | grep -i readline
309 echo
310
311 ls /wedge/oils-for-unix.org/pkg/python2/
312 ls /wedge/oils-for-unix.org/pkg/python2/2.7.18/lib/python2.7/lib-dynload
313
314 ldd /wedge/oils-for-unix.org/pkg/python2/2.7.18/lib/python2.7/lib-dynload/readline.so
315 echo
316
317 exit
318
319 dpkg -S /usr/lib/x86_64-linux-gnu/libssl.so.3
320 echo
321
322 #ls -l /usr/lib/x86_64-linux-gnu/libssl.so.1.1
323
324 apt-cache show libssl-dev
325
326 # find /lib | grep -i libssl
327 # echo
328 # find /usr/local | grep -i libssl
329 # echo
330 # python3-config --libs
331
332 # Useful command
333 # ldconfig -v #| grep ssl
334 # echo
335
336 #find / -name 'libssl.so*'
337fi
338'
339}
340
341smoke-script-2() {
342 echo '
343 cd ~/oil
344 . build/dev-shell.sh
345
346 test/ltrace.sh soil-run
347 exit
348
349 # Bug with python2
350 #devtools/types.sh soil-run
351 #test/lossless.sh soil-run
352 #exit
353
354 #test/spec-version.sh osh-version-text
355
356 echo PATH=$PATH
357
358 #which mksh
359 #mksh -c "echo hi from mksh"
360
361 #test/spec.sh smoke
362 test/spec.sh zsh-assoc
363
364 which python2
365 python2 -V
366 echo
367
368 which python3
369 python3 -V
370 echo
371
372 exit
373
374 # Bug with python2
375 test/lossless.sh soil-run
376
377 python3 -m mypy core/util.py
378 echo
379
380 # test pyflakes
381 test/lint.sh py2-lint core/util.py
382 echo
383
384 #pea/TEST.sh parse-all
385 #pea/TEST.sh run-tests
386
387 re2c --version
388 echo
389
390 # cmark.py
391 doctools/cmark.sh demo-ours
392
393 bloaty --help
394 echo
395
396 exit
397
398 # hm this shows Python
399 uftrace --version
400
401 which uftrace
402 uftrace=$(which uftrace)
403
404 ls -l ~/oils.DEPS/wedge/uftrace/0.13/bin/uftrace
405 uftrace=~/oils.DEPS/wedge/uftrace/0.13/bin/uftrace
406
407 devtools/R-test.sh soil-run
408
409 exit
410
411 cc -pg -o hello deps/source.medo/uftrace/hello.c
412
413 # libmcount-fast is in the uftrace lib/ dir
414 ldd $(which uftrace)
415 echo
416
417 set -x
418 #head /tmp/cache-bust.txt
419
420 $uftrace record hello
421 #uftrace replay hello
422 echo
423
424 #find /usr -name "libm*.so"
425 '
426}
427
428asan-smoke-script() {
429 echo '
430 cd ~/oil
431 pwd
432 whoami
433
434 # this failed with ASAN
435 yaks/TEST.sh soil-run
436 exit
437
438 build/py.sh all
439 ./NINJA-config.sh
440 bin=_bin/cxx-asan/mycpp/examples/varargs.mycpp
441 ninja $bin
442 $bin
443
444 exit
445
446 # ls -l
447 #mkdir -p _devbuild/bin
448 out=_devbuild/bin/th.asan
449
450 build/py.sh time-helper $out -fsanitize=address
451 export ASAN_OPTIONS=detect_leaks=1
452 ldd $out
453 ls -l $out
454 $out
455 exit
456 '
457}
458
459_smoke() {
460 ### Smoke test of container
461 local name=${1:-soil-dummy}
462 local tag=${2:-$LATEST_TAG}
463 local docker=${3:-$DOCKER}
464 local script_func=${4:-} # if empty, then start a debug shell
465
466 local repo_root=$PWD
467
468 local -a flags argv
469 if test -n "$script_func"; then
470 flags=()
471 argv=( bash -c "$("$script_func")" )
472 else
473 flags=( -i -t )
474 argv=( bash )
475 fi
476
477 # Stupid podman!
478 case $docker in
479 podman)
480 export-podman
481 # this fixes mount permissions!
482 flags+=( --userns=keep-id )
483 ;;
484 esac
485
486 $docker run "${flags[@]}" \
487 --mount "type=bind,source=$repo_root,target=/home/uke/oil" \
488 oils-for-unix/$name:$tag "${argv[@]}"
489}
490
491smoke() {
492 sudo $0 _smoke "$@"
493}
494
495smoke-podman-asan() {
496 _smoke soil-cpp-small latest podman asan-smoke-script
497}
498
499smoke-podman() {
500 local name=${1:-dummy}
501
502 # 2025-04: I need to do 'podman login docker.io'
503 #
504 # Running without root
505
506 # need explicit docker.io prefix with podman
507 # smoke $name latest podman docker.io/
508
509 local tag='latest'
510 local prefix='docker.io/'
511 smoke-test-script | podman run -i ${prefix}oils-for-unix/soil-$name:$tag bash
512}
513
514cmd() {
515 ### Run an arbitrary command
516 local name=${1:-soil-dummy}
517 local tag=${2:-$LATEST_TAG}
518
519 shift 2
520
521 sudo $DOCKER run oils-for-unix/$name:$tag "$@"
522}
523
524utf8() {
525 # needed for a spec test, not the default on Debian
526 cmd ovm-tarball bash -c 'LC_ALL=en_US.UTF-8; echo $LC_ALL'
527}
528
529mount-test() {
530 local name=${1:-soil-dummy}
531
532 local -a argv
533 if test $# -le 1; then
534 argv=(sh -c 'ls -l /home/uke/oil')
535 else
536 argv=( "${@:2}" ) # index 2 not 1, weird shell behavior
537 fi
538
539 # mount 'oil' directory as /app. TODO: Oils
540 sudo $DOCKER run \
541 --mount "type=bind,source=$PWD,target=/home/uke/oil" \
542 oils-for-unix/$name "${argv[@]}"
543}
544
545image-history() {
546 local image_id=${1:-soil-dummy}
547 local tag=${2:-latest}
548
549 local image="oils-for-unix/$image_id"
550
551 sudo $DOCKER history $image:$tag
552}
553
554save() {
555 local image_id=${1:-soil-dummy}
556 local tag=${2:-latest}
557
558 local image="oils-for-unix/$image_id"
559
560 mkdir -p _tmp/images
561 local out=_tmp/images/$image_id.tar
562
563 # Use > instead of -o so it doesn'th have root permissions
564 time sudo $DOCKER save $image:$tag > $out
565 ls -l -h $out
566}
567
568# This shows CREATED, command CREATED BY, size
569# It's a human readable size though
570#
571# This doesn't really have anything better
572# https://gist.github.com/MichaelSimons/fb588539dcefd9b5fdf45ba04c302db6
573#
574# It's annoying that the remote registry API is different than the local API.
575
576layers() {
577 local name=${1:-soil-dummy}
578 local tag=${2:-$LATEST_TAG}
579
580 local image="oils-for-unix/$name:$tag"
581
582 # Gah this still prints 237M, not the exact number of bytes!
583 # --format ' {{ .Size }} '
584 sudo $DOCKER history --no-trunc $image
585
586 echo $'Size\tVirtual Size'
587 sudo $DOCKER inspect $image \
588 | jq --raw-output '.[0] | [.Size, .VirtualSize] | @tsv' \
589 | commas
590}
591
592todo-debian-12() {
593 # 7 images
594 grep soil-common deps/Dockerfile.*
595}
596
597todo-purity() {
598 # TODO: we should pass --network none in $0 build
599 #
600 # Hm 7 images need pip download, should reduce them
601 #
602 # There are other sources of impurity, like:
603 # building the R-libs wedge
604 # soil-wild - we can't download the testdata, etc.
605
606 grep -l install-py3-libs deps/Dockerfile.*
607}
608
609todo-tree-shake() {
610 # We should invoke OSH to generate parts of the dockerfile? Or use podman
611 # probably?
612 #
613 # Or maybe it's a default layer in soil-debian-12?
614
615 grep task-five deps/Dockerfile.*
616}
617
618todo-relative() {
619 grep TODO # _build/wedge/relative, not _build/wedge/binary
620}
621
622"$@"
623