OILS / deps / image.sh View on Github | oils.pub

615 lines, 237 significant
1#!/usr/bin/env bash
2#
3# Manage container images for Soil
4#
5# Usage:
6# deps/image.sh <function name>
7#
8# Dirs maybe to clear:
9#
10# $0 clean-all # Start from scratch
11#
12# Example:
13#
14# (1) Update LATEST_TAG
15#
16# (2) Bootstrap Wedges
17#
18# $0 build wedge-bootstrap-debian-12 # runs init-deb-cache
19#
20# (3) Build wedges
21#
22# build/deps.sh fetch
23# build/deps.sh boxed-wedges-2025
24#
25# (4) Rebuild an image
26#
27# $0 build soil-debian-12 # runs init-deb-cache and populate the cache
28# $0 build soil-test-image T # reuse package cache from apt-get
29# $0 smoke soil-test-image # smoke test
30#
31# (5) Update live version in 'soil/host-shim.sh live-image-tag'
32#
33# (6) Push Everything you Built
34#
35# # pushes both $LATEST_TAG and latest
36# $0 push-many wedge-bootstrap-debian-12 soil-debian-12 soil-test-image
37#
38# More
39# ----
40#
41# Images: https://github.com/orgs/oils-for-unix/packages
42# formerly: https://hub.docker.com/u/oilshell
43#
44# $0 list-tagged # Show versions of images
45#
46# $0 show-cachemount # show files in apt cache
47#
48# $0 prune # seems to clear the cache
49
50set -o nounset
51set -o pipefail
52set -o errexit
53
54source deps/podman.sh # do we need this?
55
56DOCKER=${DOCKER:-podman}
57
58declare -a docker_prefix
59case $DOCKER in
60 docker)
61 # Do we still need -E to preserve CONTAINERS_REGISTRIES_CONF?
62 docker_prefix=( sudo -E DOCKER_BUILDKIT=1 $DOCKER )
63 ;;
64 podman)
65 # this can be rootless! Unlike deps/wedge.sh
66 docker_prefix=( podman )
67 ;;
68 *)
69 die "Invalid docker $DOCKER"
70 ;;
71esac
72
73readonly LATEST_TAG='v-2026-01-13' # rebuild with re2c 4.3.1 and ghcr.io
74
75clean-all() {
76 dirs='_build/wedge/tmp _build/wedge/binary _build/deps-source'
77 #rm -r -f $dirs
78 sudo rm -r -f $dirs
79}
80
81list() {
82 local which=${1:-all} # all | soil | prep
83
84 local accept=''
85 local reject=''
86 case $which in
87 all)
88 reject='^$'
89 ;;
90 soil) # 13 soil images
91 reject='^(wedge-bootstrap-.*|soil-debian-.*)'
92 ;;
93 prep)
94 # images to prepare
95 # 2025-10: *-debian-10 is Debian Buster from 2019, which was retired in
96 # 2024. You can't do sudo apt-get update
97 # https://wiki.debian.org/DebianReleases
98 accept='^(wedge-bootstrap-debian-12|soil-debian-12)'
99 ;;
100 esac
101
102 if test -n "$accept"; then
103 for name in deps/Dockerfile.*; do
104 local image_id=${name//'deps/Dockerfile.'/}
105 if [[ "$image_id" =~ $accept ]]; then
106 echo $image_id
107 fi
108 done
109 else
110 for name in deps/Dockerfile.*; do
111 local image_id=${name//'deps/Dockerfile.'/}
112 if [[ "$image_id" =~ $reject ]]; then
113 continue
114 fi
115 echo $image_id
116 done
117 fi
118}
119
120list-tagged() {
121 "${docker_prefix[@]}" \
122 images 'oils-for-unix/*' #:v-*'
123}
124
125_latest-one() {
126 local name=$1
127 "${docker_prefix[@]}" \
128 images "oils-for-unix/$name" | head -n 3
129}
130
131#
132# 2025-12: Migration to ghcr.io
133#
134
135migrate-one() {
136 local name=${1:-soil-wild}
137
138 # Good instructions from Claude!
139
140 # Pull from Docker Hub
141 podman pull docker.io/oilshell/$name:latest
142
143 # Tag for GitHub Container Registry with new name
144 podman tag docker.io/oilshell/$name:latest ghcr.io/oils-for-unix/$name:latest
145
146 # Push to GitHub Container Registry
147 podman push ghcr.io/oils-for-unix/$name:latest
148}
149
150migrate-all() {
151 list | xargs --verbose -n 1 -- $0 migrate-one
152}
153
154m-tag-one() {
155 ### Add $LATEST_TAG
156 local image=${1:-soil-wild}
157 podman tag ghcr.io/oils-for-unix/$image:latest ghcr.io/oils-for-unix/$image:$LATEST_TAG
158}
159
160m-tag-all() {
161 list | xargs --verbose -n 1 -- $0 m-tag-one
162}
163
164m-push-one() {
165 ### Push $LATEST_TAG
166 local name=${1:-soil-wild}
167 podman push ghcr.io/oils-for-unix/$name:$LATEST_TAG
168}
169
170m-push-all() {
171 list | xargs --verbose -n 1 -- $0 m-push-one
172}
173
174# BUGS in Docker.
175#
176# https://stackoverflow.com/questions/69173822/docker-build-uses-wrong-dockerfile-content-bug
177
178# NOTE: This also clears the exec.cachemount
179prune() {
180 sudo $DOCKER builder prune -f
181}
182
183# https://stackoverflow.com/questions/62834806/docker-buildkit-cache-location-size-and-id
184#
185# It lives somewhere in /var/lib/docker/overlay2
186
187show-cachemount() {
188 sudo $DOCKER system df -v --format '{{ .BuildCache | json }}' \
189 | jq '.[] | select(.CacheType == "exec.cachemount")' | tee _tmp/cachemount.txt
190
191 cat _tmp/cachemount.txt | jq -r '.ID' | while read id; do
192 sudo tree /var/lib/docker/overlay2/$id
193 sudo du --si -s /var/lib/docker/overlay2/$id
194 echo
195 done
196}
197
198tag-latest() {
199 local name=${1:-wedge-bootstrap-debian-12}
200 local tag_built_with=${2:-$LATEST_TAG}
201
202 set -x # 'docker tag' is annoyingly silent
203 "${docker_prefix[@]}" \
204 tag ghcr.io/oils-for-unix/$name:{$tag_built_with,latest}
205}
206
207build() {
208 local name=${1:-soil-dummy}
209
210 # OFF by default. TODO: use_cache setting should be automatic
211 local use_cache=${2:-}
212
213 # set -x
214 local -a flags
215 if test -n "$use_cache"; then
216 flags=()
217 else
218 flags=('--no-cache=true')
219 fi
220 #flags+=('--progress=plain')
221
222 # Uh BuildKit is not the default on Linux!
223 # http://jpetazzo.github.io/2021/11/30/docker-build-container-images-antipatterns/
224 #
225 # It is more parallel and has colored output.
226
227 # TODO: use --authfile and more
228 #export-podman
229
230 # can't preserve the entire env: https://github.com/containers/buildah/issues/3887
231 #sudo --preserve-env=CONTAINERS_REGISTRIES_CONF --preserve-env=REGISTRY_AUTH_FILE \
232 "${docker_prefix[@]}" \
233 build "${flags[@]}" \
234 --tag "ghcr.io/oils-for-unix/$name:$LATEST_TAG" \
235 --file deps/Dockerfile.$name .
236
237 # Avoid hassle by also tagging it
238 tag-latest $name
239}
240
241push() {
242 local name=${1:-soil-dummy}
243 local tag=${2:-$LATEST_TAG}
244
245 local image="ghcr.io/oils-for-unix/$name"
246
247 set -x
248
249 "${docker_prefix[@]}" push "$image:$tag"
250
251 # Also push the 'latest' tag, to avoid getting out of sync
252 "${docker_prefix[@]}" push "$image:latest"
253}
254
255push-many() {
256 for name in "$@"; do
257 push $name
258 done
259}
260
261smoke-script-1() {
262 echo '
263for file in /etc/debian_version /etc/lsb-release; do
264 if test -f $file; then
265 # spec/ble-idioms tests this
266 #grep -E "foo|^10" $file; echo grep=$?
267
268 echo $file
269 echo
270 cat $file
271 echo
272 else
273 echo "($file does not exist)"
274 fi
275done
276
277echo "bash $BASH_VERSION"
278
279git --version
280
281for name in python python2 python3; do
282 if which $name; then
283 $name -V
284 else
285 echo "$name not found"
286 fi
287done
288
289echo PATH=$PATH
290
291#curl https://op.oilshell.org/
292
293if true; then
294 python2 -c "import readline; print(readline)"
295 echo
296
297 python3 -c "import ssl; print(ssl)"
298 echo
299
300 find /usr/lib | grep -i readline
301 echo
302
303 ls /wedge/oils-for-unix.org/pkg/python2/
304 ls /wedge/oils-for-unix.org/pkg/python2/2.7.18/lib/python2.7/lib-dynload
305
306 ldd /wedge/oils-for-unix.org/pkg/python2/2.7.18/lib/python2.7/lib-dynload/readline.so
307 echo
308
309 exit
310
311 dpkg -S /usr/lib/x86_64-linux-gnu/libssl.so.3
312 echo
313
314 #ls -l /usr/lib/x86_64-linux-gnu/libssl.so.1.1
315
316 apt-cache show libssl-dev
317
318 # find /lib | grep -i libssl
319 # echo
320 # find /usr/local | grep -i libssl
321 # echo
322 # python3-config --libs
323
324 # Useful command
325 # ldconfig -v #| grep ssl
326 # echo
327
328 #find / -name 'libssl.so*'
329fi
330'
331}
332
333smoke-script-2() {
334 echo '
335 cd ~/oil
336 . build/dev-shell.sh
337
338 test/ltrace.sh soil-run
339 exit
340
341 # Bug with python2
342 #devtools/types.sh soil-run
343 #test/lossless.sh soil-run
344 #exit
345
346 #test/spec-version.sh osh-version-text
347
348 echo PATH=$PATH
349
350 #which mksh
351 #mksh -c "echo hi from mksh"
352
353 #test/spec.sh smoke
354 test/spec.sh zsh-assoc
355
356 which python2
357 python2 -V
358 echo
359
360 which python3
361 python3 -V
362 echo
363
364 exit
365
366 # Bug with python2
367 test/lossless.sh soil-run
368
369 python3 -m mypy core/util.py
370 echo
371
372 # test pyflakes
373 test/lint.sh py2-lint core/util.py
374 echo
375
376 #pea/TEST.sh parse-all
377 #pea/TEST.sh run-tests
378
379 re2c --version
380 echo
381
382 # cmark.py
383 doctools/cmark.sh demo-ours
384
385 bloaty --help
386 echo
387
388 exit
389
390 # hm this shows Python
391 uftrace --version
392
393 which uftrace
394 uftrace=$(which uftrace)
395
396 ls -l ~/oils.DEPS/wedge/uftrace/0.13/bin/uftrace
397 uftrace=~/oils.DEPS/wedge/uftrace/0.13/bin/uftrace
398
399 devtools/R-test.sh soil-run
400
401 exit
402
403 cc -pg -o hello deps/source.medo/uftrace/hello.c
404
405 # libmcount-fast is in the uftrace lib/ dir
406 ldd $(which uftrace)
407 echo
408
409 set -x
410 #head /tmp/cache-bust.txt
411
412 $uftrace record hello
413 #uftrace replay hello
414 echo
415
416 #find /usr -name "libm*.so"
417 '
418}
419
420asan-smoke-script() {
421 echo '
422 cd ~/oil
423 pwd
424 whoami
425
426 # this failed with ASAN
427 yaks/TEST.sh soil-run
428 exit
429
430 build/py.sh all
431 ./NINJA-config.sh
432 bin=_bin/cxx-asan/mycpp/examples/varargs.mycpp
433 ninja $bin
434 $bin
435
436 exit
437
438 # ls -l
439 #mkdir -p _devbuild/bin
440 out=_devbuild/bin/th.asan
441
442 build/py.sh time-helper $out -fsanitize=address
443 export ASAN_OPTIONS=detect_leaks=1
444 ldd $out
445 ls -l $out
446 $out
447 exit
448 '
449}
450
451_smoke() {
452 ### Smoke test of container
453 local name=${1:-soil-dummy}
454 local tag=${2:-$LATEST_TAG}
455 local docker=${3:-$DOCKER}
456 local script_func=${4:-} # if empty, then start a debug shell
457
458 local repo_root=$PWD
459
460 local -a flags argv
461 if test -n "$script_func"; then
462 flags=()
463 argv=( bash -c "$("$script_func")" )
464 else
465 flags=( -i -t )
466 argv=( bash )
467 fi
468
469 # Stupid podman!
470 case $docker in
471 podman)
472 export-podman
473 # this fixes mount permissions!
474 flags+=( --userns=keep-id )
475 ;;
476 esac
477
478 $docker run "${flags[@]}" \
479 --mount "type=bind,source=$repo_root,target=/home/uke/oil" \
480 oils-for-unix/$name:$tag "${argv[@]}"
481}
482
483smoke() {
484 sudo $0 _smoke "$@"
485}
486
487smoke-podman-asan() {
488 _smoke soil-cpp-small latest podman asan-smoke-script
489}
490
491smoke-podman() {
492 local name=${1:-dummy}
493
494 # 2025-04: I need to do 'podman login docker.io'
495 #
496 # Running without root
497
498 # need explicit docker.io prefix with podman
499 # smoke $name latest podman docker.io/
500
501 local tag='latest'
502 local prefix='docker.io/'
503 smoke-test-script | podman run -i ${prefix}oils-for-unix/soil-$name:$tag bash
504}
505
506cmd() {
507 ### Run an arbitrary command
508 local name=${1:-soil-dummy}
509 local tag=${2:-$LATEST_TAG}
510
511 shift 2
512
513 sudo $DOCKER run oils-for-unix/$name:$tag "$@"
514}
515
516utf8() {
517 # needed for a spec test, not the default on Debian
518 cmd ovm-tarball bash -c 'LC_ALL=en_US.UTF-8; echo $LC_ALL'
519}
520
521mount-test() {
522 local name=${1:-soil-dummy}
523
524 local -a argv
525 if test $# -le 1; then
526 argv=(sh -c 'ls -l /home/uke/oil')
527 else
528 argv=( "${@:2}" ) # index 2 not 1, weird shell behavior
529 fi
530
531 # mount 'oil' directory as /app. TODO: Oils
532 sudo $DOCKER run \
533 --mount "type=bind,source=$PWD,target=/home/uke/oil" \
534 oils-for-unix/$name "${argv[@]}"
535}
536
537image-history() {
538 local image_id=${1:-soil-dummy}
539 local tag=${2:-latest}
540
541 local image="oils-for-unix/$image_id"
542
543 sudo $DOCKER history $image:$tag
544}
545
546save() {
547 local image_id=${1:-soil-dummy}
548 local tag=${2:-latest}
549
550 local image="oils-for-unix/$image_id"
551
552 mkdir -p _tmp/images
553 local out=_tmp/images/$image_id.tar
554
555 # Use > instead of -o so it doesn'th have root permissions
556 time sudo $DOCKER save $image:$tag > $out
557 ls -l -h $out
558}
559
560# This shows CREATED, command CREATED BY, size
561# It's a human readable size though
562#
563# This doesn't really have anything better
564# https://gist.github.com/MichaelSimons/fb588539dcefd9b5fdf45ba04c302db6
565#
566# It's annoying that the remote registry API is different than the local API.
567
568layers() {
569 local name=${1:-soil-dummy}
570 local tag=${2:-$LATEST_TAG}
571
572 local image="oils-for-unix/$name:$tag"
573
574 # Gah this still prints 237M, not the exact number of bytes!
575 # --format ' {{ .Size }} '
576 sudo $DOCKER history --no-trunc $image
577
578 echo $'Size\tVirtual Size'
579 sudo $DOCKER inspect $image \
580 | jq --raw-output '.[0] | [.Size, .VirtualSize] | @tsv' \
581 | commas
582}
583
584todo-debian-12() {
585 # 7 images
586 grep soil-common deps/Dockerfile.*
587}
588
589todo-purity() {
590 # TODO: we should pass --network none in $0 build
591 #
592 # Hm 7 images need pip download, should reduce them
593 #
594 # There are other sources of impurity, like:
595 # building the R-libs wedge
596 # soil-wild - we can't download the testdata, etc.
597
598 grep -l install-py3-libs deps/Dockerfile.*
599}
600
601todo-tree-shake() {
602 # We should invoke OSH to generate parts of the dockerfile? Or use podman
603 # probably?
604 #
605 # Or maybe it's a default layer in soil-debian-12?
606
607 grep task-five deps/Dockerfile.*
608}
609
610todo-relative() {
611 grep TODO # _build/wedge/relative, not _build/wedge/binary
612}
613
614"$@"
615