OILS / deps / image.sh View on Github | oils.pub

622 lines, 239 significant
1#!/usr/bin/env bash
2#
3# Manage container images for Soil
4#
5# Usage:
6# deps/image.sh <function name>
7#
8# Dirs maybe to clear:
9#
10# $0 clean-all # Start from scratch
11#
12# Example:
13#
14# (1) Update LATEST_TAG
15#
16# (2) Bootstrap Wedges
17#
18# $0 build wedge-bootstrap-debian-12 # runs init-deb-cache
19#
20# (3) Build wedges
21#
22# build/deps.sh fetch
23# build/deps.sh boxed-wedges-2025
24#
25# (4) Rebuild an image
26#
27# $0 build soil-debian-12 # runs init-deb-cache and populate the cache
28# $0 build soil-test-image T # reuse package cache from apt-get
29# $0 smoke soil-test-image # smoke test
30#
31# (5) Update live version in 'soil/host-shim.sh live-image-tag'
32#
33# (6) Push Everything you Built
34#
35# # pushes both $LATEST_TAG and latest
36# $0 push-many wedge-bootstrap-debian-12 soil-debian-12 soil-test-image
37#
38# More
39# ----
40#
41# Images: https://github.com/orgs/oils-for-unix/packages
42# formerly: https://hub.docker.com/u/oilshell
43#
44# $0 list-tagged # Show versions of images
45#
46# $0 show-cachemount # show files in apt cache
47#
48# $0 prune # seems to clear the cache
49
50set -o nounset
51set -o pipefail
52set -o errexit
53
54source deps/podman.sh # do we need this?
55
56DOCKER=${DOCKER:-podman}
57
58declare -a docker_prefix
59case $DOCKER in
60 docker)
61 # Do we still need -E to preserve CONTAINERS_REGISTRIES_CONF?
62 docker_prefix=( sudo -E DOCKER_BUILDKIT=1 $DOCKER )
63 ;;
64 podman)
65 # this can be rootless! Unlike deps/wedge.sh
66 docker_prefix=( podman )
67 ;;
68 *)
69 die "Invalid docker $DOCKER"
70 ;;
71esac
72
73readonly LATEST_TAG='v-2025-12-21' # docker.io -> ghcr.io migraiton
74
75clean-all() {
76 dirs='_build/wedge/tmp _build/wedge/binary _build/deps-source'
77 #rm -r -f $dirs
78 sudo rm -r -f $dirs
79}
80
81list() {
82 local which=${1:-all} # all | soil | prep
83
84 local accept=''
85 local reject=''
86 case $which in
87 all)
88 reject='^$'
89 ;;
90 soil) # 13 soil images
91 reject='^(wedge-bootstrap-.*|soil-debian-.*)'
92 ;;
93 prep)
94 # images to prepare
95 # 2025-10: *-debian-10 is Debian Buster from 2019, which was retired in
96 # 2024. You can't do sudo apt-get update
97 # https://wiki.debian.org/DebianReleases
98 accept='^(wedge-bootstrap-debian-12|soil-debian-12)'
99 ;;
100 esac
101
102 if test -n "$accept"; then
103 for name in deps/Dockerfile.*; do
104 local image_id=${name//'deps/Dockerfile.'/}
105 if [[ "$image_id" =~ $accept ]]; then
106 echo $image_id
107 fi
108 done
109 else
110 for name in deps/Dockerfile.*; do
111 local image_id=${name//'deps/Dockerfile.'/}
112 if [[ "$image_id" =~ $reject ]]; then
113 continue
114 fi
115 echo $image_id
116 done
117 fi
118}
119
120list-tagged() {
121 "${docker_prefix[@]}" \
122 images 'oilshell/*' #:v-*'
123}
124
125_latest-one() {
126 local name=$1
127 "${docker_prefix[@]}" \
128 images "oilshell/$name" | head -n 3
129}
130
131#
132# 2025-12: Migration to ghcr.io
133#
134
135migrate-one() {
136 local name=${1:-soil-wild}
137
138 # Good instructions from Claude!
139
140 # Pull from Docker Hub
141 podman pull docker.io/oilshell/$name:latest
142
143 # Tag for GitHub Container Registry with new name
144 podman tag docker.io/oilshell/$name:latest ghcr.io/oils-for-unix/$name:latest
145
146 # Push to GitHub Container Registry
147 podman push ghcr.io/oils-for-unix/$name:latest
148}
149
150migrate-all() {
151 list | xargs --verbose -n 1 -- $0 migrate-one
152}
153
154m-tag-one() {
155 ### Add $LATEST_TAG
156 local image=${1:-soil-wild}
157 podman tag ghcr.io/oils-for-unix/$image:latest ghcr.io/oils-for-unix/$image:$LATEST_TAG
158}
159
160m-tag-all() {
161 list | xargs --verbose -n 1 -- $0 m-tag-one
162}
163
164m-push-one() {
165 ### Push $LATEST_TAG
166 local name=${1:-soil-wild}
167 podman push ghcr.io/oils-for-unix/$name:$LATEST_TAG
168}
169
170m-push-all() {
171 list | xargs --verbose -n 1 -- $0 m-push-one
172}
173
174# BUGS in Docker.
175#
176# https://stackoverflow.com/questions/69173822/docker-build-uses-wrong-dockerfile-content-bug
177
178# NOTE: This also clears the exec.cachemount
179prune() {
180 sudo $DOCKER builder prune -f
181}
182
183# https://stackoverflow.com/questions/62834806/docker-buildkit-cache-location-size-and-id
184#
185# It lives somewhere in /var/lib/docker/overlay2
186
187show-cachemount() {
188 sudo $DOCKER system df -v --format '{{ .BuildCache | json }}' \
189 | jq '.[] | select(.CacheType == "exec.cachemount")' | tee _tmp/cachemount.txt
190
191 cat _tmp/cachemount.txt | jq -r '.ID' | while read id; do
192 sudo tree /var/lib/docker/overlay2/$id
193 sudo du --si -s /var/lib/docker/overlay2/$id
194 echo
195 done
196}
197
198tag-latest() {
199 local name=${1:-wedge-bootstrap-debian-12}
200 local tag_built_with=${2:-$LATEST_TAG}
201
202 set -x # 'docker tag' is annoyingly silent
203 "${docker_prefix[@]}" \
204 tag oilshell/$name:{$tag_built_with,latest}
205}
206
207build() {
208 local name=${1:-soil-dummy}
209
210 # OFF by default. TODO: use_cache setting should be automatic
211 local use_cache=${2:-}
212
213 # set -x
214 local -a flags
215 if test -n "$use_cache"; then
216 flags=()
217 else
218 flags=('--no-cache=true')
219 fi
220 #flags+=('--progress=plain')
221
222 # Uh BuildKit is not the default on Linux!
223 # http://jpetazzo.github.io/2021/11/30/docker-build-container-images-antipatterns/
224 #
225 # It is more parallel and has colored output.
226
227 # TODO: use --authfile and more
228 #export-podman
229
230 # can't preserve the entire env: https://github.com/containers/buildah/issues/3887
231 #sudo --preserve-env=CONTAINERS_REGISTRIES_CONF --preserve-env=REGISTRY_AUTH_FILE \
232 "${docker_prefix[@]}" \
233 build "${flags[@]}" \
234 --tag "oilshell/$name:$LATEST_TAG" \
235 --file deps/Dockerfile.$name .
236
237 # Avoid hassle by also tagging it
238 tag-latest $name
239}
240
241push() {
242 local name=${1:-soil-dummy}
243 local tag=${2:-$LATEST_TAG}
244
245 # TODO: replace with flags
246 #export-podman
247
248 local image="oilshell/$name:$tag"
249
250 set -x
251
252 # -E for export-podman vars
253 "${docker_prefix[@]}" \
254 push $image
255 #sudo -E $DOCKER --log-level=debug push $image
256
257 # Also push the 'latest' tag, to avoid getting out of sync
258 "${docker_prefix[@]}" \
259 push oilshell/$name:latest
260}
261
262push-many() {
263 for name in "$@"; do
264 push $name
265 done
266}
267
268smoke-script-1() {
269 echo '
270for file in /etc/debian_version /etc/lsb-release; do
271 if test -f $file; then
272 # spec/ble-idioms tests this
273 #grep -E "foo|^10" $file; echo grep=$?
274
275 echo $file
276 echo
277 cat $file
278 echo
279 else
280 echo "($file does not exist)"
281 fi
282done
283
284echo "bash $BASH_VERSION"
285
286git --version
287
288for name in python python2 python3; do
289 if which $name; then
290 $name -V
291 else
292 echo "$name not found"
293 fi
294done
295
296echo PATH=$PATH
297
298#curl https://op.oilshell.org/
299
300if true; then
301 python2 -c "import readline; print(readline)"
302 echo
303
304 python3 -c "import ssl; print(ssl)"
305 echo
306
307 find /usr/lib | grep -i readline
308 echo
309
310 ls /wedge/oils-for-unix.org/pkg/python2/
311 ls /wedge/oils-for-unix.org/pkg/python2/2.7.18/lib/python2.7/lib-dynload
312
313 ldd /wedge/oils-for-unix.org/pkg/python2/2.7.18/lib/python2.7/lib-dynload/readline.so
314 echo
315
316 exit
317
318 dpkg -S /usr/lib/x86_64-linux-gnu/libssl.so.3
319 echo
320
321 #ls -l /usr/lib/x86_64-linux-gnu/libssl.so.1.1
322
323 apt-cache show libssl-dev
324
325 # find /lib | grep -i libssl
326 # echo
327 # find /usr/local | grep -i libssl
328 # echo
329 # python3-config --libs
330
331 # Useful command
332 # ldconfig -v #| grep ssl
333 # echo
334
335 #find / -name 'libssl.so*'
336fi
337'
338}
339
340smoke-script-2() {
341 echo '
342 cd ~/oil
343 . build/dev-shell.sh
344
345 test/ltrace.sh soil-run
346 exit
347
348 # Bug with python2
349 #devtools/types.sh soil-run
350 #test/lossless.sh soil-run
351 #exit
352
353 #test/spec-version.sh osh-version-text
354
355 echo PATH=$PATH
356
357 #which mksh
358 #mksh -c "echo hi from mksh"
359
360 #test/spec.sh smoke
361 test/spec.sh zsh-assoc
362
363 which python2
364 python2 -V
365 echo
366
367 which python3
368 python3 -V
369 echo
370
371 exit
372
373 # Bug with python2
374 test/lossless.sh soil-run
375
376 python3 -m mypy core/util.py
377 echo
378
379 # test pyflakes
380 test/lint.sh py2-lint core/util.py
381 echo
382
383 #pea/TEST.sh parse-all
384 #pea/TEST.sh run-tests
385
386 re2c --version
387 echo
388
389 # cmark.py
390 doctools/cmark.sh demo-ours
391
392 bloaty --help
393 echo
394
395 exit
396
397 # hm this shows Python
398 uftrace --version
399
400 which uftrace
401 uftrace=$(which uftrace)
402
403 ls -l ~/oils.DEPS/wedge/uftrace/0.13/bin/uftrace
404 uftrace=~/oils.DEPS/wedge/uftrace/0.13/bin/uftrace
405
406 devtools/R-test.sh soil-run
407
408 exit
409
410 cc -pg -o hello deps/source.medo/uftrace/hello.c
411
412 # libmcount-fast is in the uftrace lib/ dir
413 ldd $(which uftrace)
414 echo
415
416 set -x
417 #head /tmp/cache-bust.txt
418
419 $uftrace record hello
420 #uftrace replay hello
421 echo
422
423 #find /usr -name "libm*.so"
424 '
425}
426
427asan-smoke-script() {
428 echo '
429 cd ~/oil
430 pwd
431 whoami
432
433 # this failed with ASAN
434 yaks/TEST.sh soil-run
435 exit
436
437 build/py.sh all
438 ./NINJA-config.sh
439 bin=_bin/cxx-asan/mycpp/examples/varargs.mycpp
440 ninja $bin
441 $bin
442
443 exit
444
445 # ls -l
446 #mkdir -p _devbuild/bin
447 out=_devbuild/bin/th.asan
448
449 build/py.sh time-helper $out -fsanitize=address
450 export ASAN_OPTIONS=detect_leaks=1
451 ldd $out
452 ls -l $out
453 $out
454 exit
455 '
456}
457
458_smoke() {
459 ### Smoke test of container
460 local name=${1:-soil-dummy}
461 local tag=${2:-$LATEST_TAG}
462 local docker=${3:-$DOCKER}
463 local script_func=${4:-} # if empty, then start a debug shell
464
465 local repo_root=$PWD
466
467 local -a flags argv
468 if test -n "$script_func"; then
469 flags=()
470 argv=( bash -c "$("$script_func")" )
471 else
472 flags=( -i -t )
473 argv=( bash )
474 fi
475
476 # Stupid podman!
477 case $docker in
478 podman)
479 export-podman
480 # this fixes mount permissions!
481 flags+=( --userns=keep-id )
482 ;;
483 esac
484
485 $docker run "${flags[@]}" \
486 --mount "type=bind,source=$repo_root,target=/home/uke/oil" \
487 oilshell/$name:$tag "${argv[@]}"
488}
489
490smoke() {
491 sudo $0 _smoke "$@"
492}
493
494smoke-podman-asan() {
495 _smoke soil-cpp-small latest podman asan-smoke-script
496}
497
498smoke-podman() {
499 local name=${1:-dummy}
500
501 # 2025-04: I need to do 'podman login docker.io'
502 #
503 # Running without root
504
505 # need explicit docker.io prefix with podman
506 # smoke $name latest podman docker.io/
507
508 local tag='latest'
509 local prefix='docker.io/'
510 smoke-test-script | podman run -i ${prefix}oilshell/soil-$name:$tag bash
511}
512
513cmd() {
514 ### Run an arbitrary command
515 local name=${1:-soil-dummy}
516 local tag=${2:-$LATEST_TAG}
517
518 shift 2
519
520 sudo $DOCKER run oilshell/$name:$tag "$@"
521}
522
523utf8() {
524 # needed for a spec test, not the default on Debian
525 cmd ovm-tarball bash -c 'LC_ALL=en_US.UTF-8; echo $LC_ALL'
526}
527
528mount-test() {
529 local name=${1:-soil-dummy}
530
531 local -a argv
532 if test $# -le 1; then
533 argv=(sh -c 'ls -l /home/uke/oil')
534 else
535 argv=( "${@:2}" ) # index 2 not 1, weird shell behavior
536 fi
537
538 # mount 'oil' directory as /app. TODO: Oils
539 sudo $DOCKER run \
540 --mount "type=bind,source=$PWD,target=/home/uke/oil" \
541 oilshell/$name "${argv[@]}"
542}
543
544image-history() {
545 local image_id=${1:-soil-dummy}
546 local tag=${2:-latest}
547
548 local image="oilshell/$image_id"
549
550 sudo $DOCKER history $image:$tag
551}
552
553save() {
554 local image_id=${1:-soil-dummy}
555 local tag=${2:-latest}
556
557 local image="oilshell/$image_id"
558
559 mkdir -p _tmp/images
560 local out=_tmp/images/$image_id.tar
561
562 # Use > instead of -o so it doesn'th have root permissions
563 time sudo $DOCKER save $image:$tag > $out
564 ls -l -h $out
565}
566
567# This shows CREATED, command CREATED BY, size
568# It's a human readable size though
569#
570# This doesn't really have anything better
571# https://gist.github.com/MichaelSimons/fb588539dcefd9b5fdf45ba04c302db6
572#
573# It's annoying that the remote registry API is different than the local API.
574
575layers() {
576 local name=${1:-soil-dummy}
577 local tag=${2:-$LATEST_TAG}
578
579 local image="oilshell/$name:$tag"
580
581 # Gah this still prints 237M, not the exact number of bytes!
582 # --format ' {{ .Size }} '
583 sudo $DOCKER history --no-trunc $image
584
585 echo $'Size\tVirtual Size'
586 sudo $DOCKER inspect $image \
587 | jq --raw-output '.[0] | [.Size, .VirtualSize] | @tsv' \
588 | commas
589}
590
591todo-debian-12() {
592 # 7 images
593 grep soil-common deps/Dockerfile.*
594}
595
596todo-purity() {
597 # TODO: we should pass --network none in $0 build
598 #
599 # Hm 7 images need pip download, should reduce them
600 #
601 # There are other sources of impurity, like:
602 # building the R-libs wedge
603 # soil-wild - we can't download the testdata, etc.
604
605 grep -l install-py3-libs deps/Dockerfile.*
606}
607
608todo-tree-shake() {
609 # We should invoke OSH to generate parts of the dockerfile? Or use podman
610 # probably?
611 #
612 # Or maybe it's a default layer in soil-debian-12?
613
614 grep task-five deps/Dockerfile.*
615}
616
617todo-relative() {
618 grep TODO # _build/wedge/relative, not _build/wedge/binary
619}
620
621"$@"
622