OILS / deps / images.sh View on Github | oilshell.org

283 lines, 125 significant
1#!/usr/bin/env bash
2#
3# Manage container images for Soil
4#
5# Usage:
6# deps/images.sh <function name>
7#
8# Example: Rebuild an image:
9#
10# (1) Update LATEST_TAG
11#
12# (2) Rebuild
13#
14# deps/images.sh build soil-common # populates apt cache. WHY DO I NEED THIS?
15# deps/images.sh build soil-debian-12
16# deps/images.sh build soil-pea T # reuse package cache from apt-get
17# deps/images.sh smoke soil-pea
18#
19# (3) Push image and common, including latest
20#
21# deps/images.sh push soil-cpp-small v-2024-06-08
22#
23# deps/images.sh push soil-common v-2024-06-08
24# sudo docker tag oilshell/soil-common:{v-2024-06-08,latest}
25# deps/images.sh push soil-common latest
26#
27# (4) Update live version in 'soil/host-shim.sh live-image-tag'
28#
29# Our images:
30#
31# https://hub.docker.com/u/oilshell
32#
33# deps/images.sh list-tagged
34#
35# Bootstrapping Wedges
36# --------------------
37#
38# deps/images.sh build wedge-bootstrap-debian-10 T
39# deps/images.sh push wedge-bootstrap-debian-10 v-2024-06-08
40
41set -o nounset
42set -o pipefail
43set -o errexit
44
45source deps/podman.sh
46
47DOCKER=${DOCKER:-docker}
48
49# Build with this tag
50readonly LATEST_TAG='v-2024-08-26'
51
52# BUGS in Docker.
53#
54# https://stackoverflow.com/questions/69173822/docker-build-uses-wrong-dockerfile-content-bug
55
56# NOTE: This also clears the exec.cachemount
57prune() {
58 sudo $DOCKER builder prune -f
59}
60
61# https://stackoverflow.com/questions/62834806/docker-buildkit-cache-location-size-and-id
62#
63# It lives somewhere in /var/lib/docker/overlay2
64
65show-cachemount() {
66 sudo $DOCKER system df -v --format '{{ .BuildCache | json }}' \
67 | jq '.[] | select(.CacheType == "exec.cachemount")' | tee _tmp/cachemount.txt
68
69 cat _tmp/cachemount.txt | jq -r '.ID' | while read id; do
70 sudo tree /var/lib/docker/overlay2/$id
71 sudo du --si -s /var/lib/docker/overlay2/$id
72 echo
73 done
74}
75
76tag-common() {
77 local hash=$1 # get hash from $0 list-tagged
78 sudo $DOCKER tag $hash oilshell/soil-common:latest
79}
80
81build() {
82 local name=${1:-soil-dummy}
83 local use_cache=${2:-} # OFF by default
84
85 # set -x
86 local -a flags
87 if test -n "$use_cache"; then
88 flags=()
89 else
90 flags=('--no-cache=true')
91 fi
92 #flags+=('--progress=plain')
93
94 # Uh BuildKit is not the default on Linux!
95 # http://jpetazzo.github.io/2021/11/30/docker-build-container-images-antipatterns/
96 #
97 # It is more parallel and has colored output.
98
99 # TODO: use --authfile and more
100 #export-podman
101
102 # can't preserve the entire env: https://github.com/containers/buildah/issues/3887
103 #sudo --preserve-env=CONTAINERS_REGISTRIES_CONF --preserve-env=REGISTRY_AUTH_FILE \
104 sudo -E DOCKER_BUILDKIT=1 \
105 $DOCKER build "${flags[@]}" \
106 --tag "oilshell/$name:$LATEST_TAG" \
107 --file deps/Dockerfile.$name .
108}
109
110list-images() {
111 for name in deps/Dockerfile.*; do
112 local image_id=${name//'deps/Dockerfile.'/}
113 if test "$image_id" = 'test-image'; then
114 continue
115 fi
116 echo $image_id
117 done
118}
119
120tag-all-latest() {
121 list-images | egrep -v 'wedge-builder|bootstrap' | while read image; do
122 local tag
123 tag=$(soil/host-shim.sh live-image-tag $image)
124
125 echo "$tag $image"
126
127 # syntax: source -> target
128 sudo $DOCKER tag oilshell/soil-$image:$tag oilshell/soil-$image:latest
129 done
130}
131
132push-all-latest() {
133 ### 'latest' can lag behind the tagged version, so push to catch up
134
135 # because our 'my-sizes' script fetches the latest manifest
136
137 list-images | grep -v 'wedge-builder|bootstrap' | while read image_id; do
138 echo "___ $image_id"
139 push $image_id latest
140 done
141}
142
143list-tagged() {
144 sudo $DOCKER images 'oilshell/*' #:v-*'
145}
146
147push() {
148 local name=${1:-soil-dummy}
149 local tag=${2:-$LATEST_TAG}
150
151 # TODO: replace with flags
152 #export-podman
153
154 local image="oilshell/$name:$tag"
155
156 # -E for export-podman vars
157 sudo -E $DOCKER push $image
158 #sudo -E $DOCKER --log-level=debug push $image
159}
160
161smoke() {
162 ### Smoke test of container
163 local name=${1:-soil-dummy}
164 local tag=${2:-$LATEST_TAG}
165 local docker=${3:-$DOCKER}
166 local prefix=${4:-}
167
168 #sudo docker run oilshell/$name
169 #sudo docker run oilshell/$name python2 -c 'print("python2")'
170
171 # Need to point at registries.conf ?
172 #export-podman
173
174 sudo $docker run ${prefix}oilshell/$name:$tag bash -c '
175echo "bash $BASH_VERSION"
176
177git --version
178
179for name in python python2 python3; do
180 if which $name; then
181 $name -V
182 else
183 echo "$name not found"
184 fi
185done
186
187echo PATH=$PATH
188
189curl https://ci.oilshell.org/
190'
191
192 # Python 2.7 build/prepare.sh requires this
193 #sudo docker run oilshell/$name python -V
194
195 #sudo docker run oilshell/$name python3 -c 'import pexpect; print(pexpect)'
196}
197
198smoke-podman() {
199 local name=${1:-dummy}
200
201 # need explicit docker.io prefix with podman
202 smoke $name latest podman docker.io/
203}
204
205cmd() {
206 ### Run an arbitrary command
207 local name=${1:-soil-dummy}
208 local tag=${2:-$LATEST_TAG}
209
210 shift 2
211
212 sudo $DOCKER run oilshell/$name:$tag "$@"
213}
214
215utf8() {
216 # needed for a spec test, not the default on Debian
217 cmd ovm-tarball bash -c 'LC_ALL=en_US.UTF-8; echo $LC_ALL'
218}
219
220mount-test() {
221 local name=${1:-soil-dummy}
222
223 local -a argv
224 if test $# -le 1; then
225 argv=(sh -c 'ls -l /home/uke/oil')
226 else
227 argv=( "${@:2}" ) # index 2 not 1, weird shell behavior
228 fi
229
230 # mount Oil directory as /app
231 sudo $DOCKER run \
232 --mount "type=bind,source=$PWD,target=/home/uke/oil" \
233 oilshell/$name "${argv[@]}"
234}
235
236image-history() {
237 local image_id=${1:-soil-dummy}
238 local tag=${2:-latest}
239
240 local image="oilshell/$image_id"
241
242 sudo $DOCKER history $image:$tag
243}
244
245save() {
246 local image_id=${1:-soil-dummy}
247 local tag=${2:-latest}
248
249 local image="oilshell/$image_id"
250
251 mkdir -p _tmp/images
252 local out=_tmp/images/$image_id.tar
253
254 # Use > instead of -o so it doesn'th have root permissions
255 time sudo $DOCKER save $image:$tag > $out
256 ls -l -h $out
257}
258
259# This shows CREATED, command CREATED BY, size
260# It's a human readable size though
261#
262# This doesn't really have anything better
263# https://gist.github.com/MichaelSimons/fb588539dcefd9b5fdf45ba04c302db6
264#
265# It's annoying that the remote registry API is different than the local API.
266
267layers() {
268 local name=${1:-soil-dummy}
269 local tag=${2:-$LATEST_TAG}
270
271 local image="oilshell/$name:$tag"
272
273 # Gah this still prints 237M, not the exact number of bytes!
274 # --format ' {{ .Size }} '
275 sudo $DOCKER history --no-trunc $image
276
277 echo $'Size\tVirtual Size'
278 sudo $DOCKER inspect $image \
279 | jq --raw-output '.[0] | [.Size, .VirtualSize] | @tsv' \
280 | commas
281}
282
283"$@"