OILS / deps / wedge.sh View on Github | oils.pub

648 lines, 354 significant
1#!/usr/bin/env bash
2#
3# Build a wedge.
4#
5# Usage:
6# deps/wedge.sh <function name>
7#
8# Containerized build:
9#
10# $0 build deps/source.medo/re2c/
11#
12# Host build, without containers:
13#
14# $0 unboxed deps/source.medo/re2c/
15#
16# Individual steps:
17#
18# $0 unboxed-make deps/source.medo/re2c/
19# $0 unboxed-install deps/source.medo/re2c/
20# $0 unboxed-smoke-test deps/source.medo/re2c/
21#
22# Host dir structure:
23#
24# ~/git/oils-for-unix/oils/
25# deps/
26# source.medo/ # Source Files
27# MEDO # points to silo
28# re2c.wedge.sh # later it will be re2c.wedge.hay
29# re2c-3.0.blob # .tar.gz file that you can 'medo sync'
30# re2c-3.1.blob
31# opaque.medo/ # Binary files, e.g. Clang
32# derived.medo/ # Saved output of 'wedge build'
33#
34# _build/ # Temp dirs and output
35# obj/ # for C++ / Ninja
36# deps-source/ # sync'd from deps/source.medo - should it be
37# # _build/wedge/source?
38# wedge/ # for containerized builds
39# tmp/ # build directory
40# boxed/ # output of containerized build
41# # TODO: rename from /binary/
42# logs/
43# smoke-test/ # current dir for smoke test
44
45# Every package ("wedge") has these dirs associated with it:
46#
47# 1. Dir with additional tests / files, near tarball and *.wedge.sh ($wedge_dir)
48# 2. Where it's extracted ($src_dir)
49# 3. The temp dir where you run ./configure --prefix; make; make install ($build_dir)
50# 4. The dir to install to ($install_dir)
51# 5. The temp dir where the smoke test is run
52
53# For Debian/Ubuntu
54
55# Note: xz-utils needed to extract, but medo should make that transparent?
56#
57# Container dir structure
58#
59# /home/uke/
60# tmp-mount/
61# _build/ # Build into this temp dir
62# deps-source/
63# re2c/
64# re2c-3.0.tar.xz
65# re2c-3.0/ # Extract it here
66# wedge/
67# re2c
68# /wedge/ # Output is mounted to oil/_mount/wedge-out
69# oilshell.org/
70# pkg/
71# re2c/
72# 3.0/
73# debug-info/ # Probably needs to be at an absolute path because of
74# # --debug-link
75# re2c/
76# 3.0/
77#
78# Then Dockerfile.wild does:
79#
80# COPY _build/wedge/binary/oils-for-unix.org/pkg/re2c/3.0 \
81# /wedge/oils-for-unix.org/pkg/re2c/3.0
82
83set -o nounset
84set -o pipefail
85set -o errexit
86
87REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
88readonly REPO_ROOT
89
90log() {
91 echo "$@" >&2
92}
93
94die() {
95 log "$0: fatal: $@"
96 exit 1
97}
98
99if test -n "${WEDGE_2025:-}"; then
100 log "*** WEDGE running in 2025 mode with ../oils.DEPS"
101
102 # We don't have any absolute/relative distinction
103 # SHOULD match $WEDGE_2025_DIR in build/deps.sh
104 OILS_ABSOLUTE_ROOT="$HOME/oils.DEPS/wedge"
105 OILS_RELATIVE_ROOT="$HOME/oils.DEPS/wedge"
106 OILS_GUEST_DIR=/home/uke0/oils
107else
108 OILS_ABSOLUTE_ROOT='/wedge/oils-for-unix.org'
109
110 # The user may build a wedge outside a container here
111 OILS_RELATIVE_ROOT="$HOME/wedge/oils-for-unix.org"
112
113 OILS_GUEST_DIR=/home/uke0/oil # old dir
114fi
115
116#
117# Dirs
118#
119
120source-dir() {
121 if test -n "${WEDGE_TARBALL_NAME:-}"; then
122 # for Python-3.10.4 to override 'python3' package name
123 echo "$REPO_ROOT/_build/deps-source/$WEDGE_NAME/$WEDGE_TARBALL_NAME-$WEDGE_VERSION"
124
125 else
126 echo "$REPO_ROOT/_build/deps-source/$WEDGE_NAME/$WEDGE_NAME-$WEDGE_VERSION"
127 fi
128}
129
130build-dir() {
131 # call it tmp-build?
132 echo "$REPO_ROOT/_build/wedge/tmp/$WEDGE_NAME-$WEDGE_VERSION"
133}
134
135install-dir() {
136 local prefix
137 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
138 prefix=$OILS_ABSOLUTE_ROOT
139 else
140 prefix=$OILS_RELATIVE_ROOT
141 fi
142
143 # TODO: We want to support multiple versions of the same wedge
144 # So maybe we can provide
145 #
146 # WEDGE_VERSION_LIST='4.4 5.2'
147 #
148 # And then provide a flag to select them?
149
150 if test -n "${WEDGE_2025:-}"; then
151 echo "$prefix/$WEDGE_NAME/$WEDGE_VERSION"
152 else
153 echo "$prefix/pkg/$WEDGE_NAME/$WEDGE_VERSION"
154 fi
155}
156
157smoke-test-dir() {
158 echo "$REPO_ROOT/_build/wedge/smoke-test/$WEDGE_NAME-$WEDGE_VERSION"
159}
160
161load-wedge() {
162 ### source .wedge.sh file and ensure it conforms to protocol
163
164 local wedge_dir=$1
165 local version_requested=${2:-}
166
167 echo "Loading $wedge_dir"
168 echo
169
170 source $wedge_dir/WEDGE
171
172 echo " OK name: ${WEDGE_NAME?"$wedge_dir: WEDGE_NAME required"}"
173
174 # This WEDGE supports a single version.
175 if test -n "${WEDGE_VERSION:-}"; then
176 echo " -- single version: $WEDGE_VERSION"
177 fi
178
179 # Can validate version against this
180 if test -n "${WEDGE_VERSION_LIST:-}"; then
181 echo " -- version list: $WEDGE_VERSION_LIST"
182
183 if test -z "$version_requested"; then
184 die "FAIL Expected explicit version, one of: $WEDGE_VERSION_LIST"
185 fi
186
187 case "$WEDGE_VERSION_LIST" in
188 *"$version_requested"*)
189 echo " OK Setting WEDGE_VERSION to $version_requested"
190 WEDGE_VERSION=$version_requested
191 ;;
192 *)
193 die "FAIL Requested version $version_requested should be one of: $WEDGE_VERSION_LIST"
194 ;;
195 esac
196 fi
197
198 if test -n "${WEDGE_TARBALL_NAME:-}"; then
199 echo " -- tarball name: $WEDGE_TARBALL_NAME"
200 fi
201 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
202 echo ' -- WEDGE_IS_ABSOLUTE'
203 fi
204
205 # Python and R installation use the network
206 if test -n "${WEDGE_LEAKY_BUILD:-}"; then
207 echo ' -- WEDGE_LEAKY_BUILD'
208 fi
209
210 if declare -f wedge-make; then
211 echo " OK wedge-make"
212 elif declare -f wedge-make-from-source-dir; then
213 echo " OK wedge-make-from-source-dir"
214 else
215 die "$wedge_dir: wedge-make(-from-source-dir) not declared"
216 fi
217
218 if declare -f wedge-install; then
219 echo " OK wedge-install"
220 elif declare -f wedge-make-from-source-dir; then
221 echo " OK wedge-install-from-source-dir"
222 else
223 die "$wedge_dir: wedge-install(-from-source-dir) not declared"
224 fi
225
226 # Just one function for now
227 for func in wedge-smoke-test; do
228 if declare -f $func > /dev/null; then
229 echo " OK $func"
230 else
231 die "$wedge_dir: $func not declared"
232 fi
233 done
234 echo
235
236 echo "Loaded $wedge_dir"
237 echo
238}
239
240_run-sourced-func() {
241 "$@"
242}
243
244#
245# Actions
246#
247
248validate() {
249 local wedge=$1
250 local version_requested=${2:-}
251
252 load-wedge $wedge "$version_requested"
253}
254
255abs-wedge-dir() {
256 local wedge_dir=$1
257 case $wedge_dir in
258 /*) # it's already absolute
259 echo $wedge_dir
260 ;;
261 *)
262 echo $PWD/$wedge_dir
263 ;;
264 esac
265}
266
267unboxed-make() {
268 ### Build on the host
269 local wedge_dir=$1 # e.g. re2c.wedge.sh
270 local version_requested=${2:-} # e.g. 5.2
271 local install_dir=${3:-}
272 # NOT created because it might require root permissions!
273 if test -z "$install_dir"; then
274 install_dir=$(install-dir)
275 fi
276
277 local source_dir
278 source_dir=$(source-dir)
279 echo " SRC $source_dir"
280
281 local build_dir
282 build_dir=$(build-dir)
283
284 local abs_wedge_dir
285 abs_wedge_dir=$(abs-wedge-dir $wedge_dir)
286
287 rm -r -f -v $build_dir
288 mkdir -p $build_dir
289
290 if declare -f wedge-make-from-source-dir; then
291 # e.g. for yash, which can't build outside the source tree
292 pushd $source_dir
293 wedge-make-from-source-dir $source_dir $install_dir $abs_wedge_dir
294 popd
295 else
296 pushd $build_dir
297 wedge-make $source_dir $build_dir $install_dir $abs_wedge_dir
298 popd
299 fi
300}
301
302
303# https://www.gnu.org/prep/standards/html_node/Standard-Targets.html
304
305# Do not strip executables when installing them. This helps eventual
306# debugging that may be needed later, and nowadays disk space is cheap and
307# dynamic loaders typically ensure debug sections are not loaded during
308# normal execution. Users that need stripped binaries may invoke the
309# install-strip target to do that.
310
311_unboxed-install() {
312 local wedge_dir=$1 # e.g. re2c.wedge.sh
313 local version_requested=${2:-} # e.g. 5.2
314 local install_dir=${3:-}
315 if test -z "$install_dir"; then
316 install_dir=$(install-dir)
317 fi
318 mkdir -p $install_dir
319
320 load-wedge $wedge_dir "$version_requested"
321
322 local source_dir
323 source_dir=$(source-dir)
324
325 local build_dir
326 build_dir=$(build-dir)
327
328 if declare -f wedge-make-from-source-dir; then
329 pushd $source_dir
330 wedge-install-from-source-dir $source_dir $install_dir
331 popd
332 else
333 # Note: install-dir needed for time-helper, but not others
334 #
335 # I think it would nicer to pushd $build_dir in most cases
336
337 wedge-install $build_dir $install_dir
338 fi
339}
340
341unboxed-install() {
342 local wedge=$1 # e.g. re2.wedge.sh
343
344 if test -n "${WEDGE_2025:-}"; then
345 _unboxed-install "$@"
346 else
347 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
348 sudo $0 _unboxed-install "$@"
349 else
350 _unboxed-install "$@"
351 fi
352 fi
353}
354
355unboxed-smoke-test() {
356 local wedge_dir=$1 # e.g. re2c/ with WEDGE
357 local version_requested=${2:-} # e.g. 5.2
358 local install_dir=${3:-}
359 if test -z "$install_dir"; then
360 install_dir=$(install-dir)
361 fi
362
363 load-wedge $wedge_dir "$version_requested"
364
365 local smoke_test_dir
366 smoke_test_dir=$(smoke-test-dir)
367
368 echo ' SMOKE TEST'
369
370 local abs_wedge_dir
371 abs_wedge_dir=$(abs-wedge-dir $wedge_dir)
372
373 # TODO: To ensure a clean dir, it might be better to test that it does NOT
374 # exist first, and just make it. If it exists, then remove everything.
375
376 rm -r -f -v $smoke_test_dir
377 mkdir -p $smoke_test_dir
378
379 pushd $smoke_test_dir
380 set -x
381 wedge-smoke-test $install_dir $abs_wedge_dir
382 set +x
383 popd
384
385 echo ' OK'
386}
387
388unboxed-stats() {
389 local wedge=$1
390
391 load-wedge $wedge "$version_requested"
392
393 du --si -s $(source-dir)
394 echo
395
396 du --si -s $(build-dir)
397 echo
398
399 du --si -s $(install-dir)
400 echo
401}
402
403unboxed() {
404 local wedge_src_dir=$1
405
406 # Can override default version. Could be a flag since it's optional? But
407 # right now we always pass it.
408 local version_requested=$2
409
410 # Generally passed to ./configure
411 local wedge_out_base_dir=$3
412
413 case $wedge_out_base_dir in
414 # Turn it from ../oils.DEPS/wedge into EITHER:
415 # /home/uke/oils.DEPS/wedge
416 # /home/andy/git/oils-for-unix/oils.DEPS/wedge
417 ../*)
418 mkdir -p $wedge_out_base_dir
419 wedge_out_base_dir=$(cd $wedge_out_base_dir; pwd)
420 ;;
421 esac
422
423 log "*** unboxed $wedge_src_dir $version_requested wedge_out_base_dir=$wedge_out_base_dir"
424
425 load-wedge $wedge_src_dir "$version_requested"
426
427 local install_dir=$wedge_out_base_dir/$WEDGE_NAME/$WEDGE_VERSION
428
429 unboxed-make $wedge_src_dir "$version_requested" "$install_dir"
430
431 unboxed-install $wedge_src_dir "$version_requested" "$install_dir"
432
433 unboxed-smoke-test $wedge_src_dir "$version_requested" "$install_dir"
434}
435
436readonly DEFAULT_DISTRO=debian-10 # Debian Buster
437
438DOCKER=${DOCKER:-podman}
439
440declare -a docker_prefix
441case $DOCKER in
442 docker)
443 # -E to preserve CONTAINERS_REGISTRIES_CONF
444 docker_prefix=( sudo -E $DOCKER )
445 ;;
446 podman)
447 # Hm rootless podman gives mkdir permission error, but sudo podman works
448 docker_prefix=( sudo -E podman )
449 ;;
450 *)
451 die "Invalid docker $DOCKER"
452 ;;
453esac
454
455
456boxed() {
457 ### Build inside a container, and put output in a specific place.
458
459 # TODO: Specify the container OS, CPU like x86-64, etc.
460
461 local wedge=$1
462 local version_requested=${2:-}
463 # /wedge/oils-for-unix.org/pkg or ~/wedge/oils-for-unix.org/pkg
464 local wedge_out_base_dir=${3:-}
465 local distro=${4:-$DEFAULT_DISTRO}
466
467 local bootstrap_image=oilshell/wedge-bootstrap-$distro
468
469 log "*** boxed $wedge $version_requested out=$wedge_out_base_dir distro=$distro"
470
471 load-wedge $wedge "$version_requested"
472
473 # Permissions will be different, so we separate the two
474
475 local wedge_host_dir
476 local wedge_guest_dir
477 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
478 wedge_host_dir=_build/wedge/binary # TODO: rename to /absolute/
479 wedge_guest_dir=/wedge
480 else
481 wedge_host_dir=_build/wedge/relative
482 wedge_guest_dir=/home/uke0/wedge
483 fi
484 local wedge_guest_pkg_dir=$wedge_guest_dir/oils-for-unix.org/pkg
485
486 mkdir -v -p $wedge_host_dir
487
488 # TODO:
489 #
490 # Mount
491 # INPUTS: the PKG.wedge.sh, and the tarball
492 # CODE: this script: deps/wedge.sh
493 # OUTPUT: /wedge/oils-for-unix.org
494 # TODO: Also put logs and symbols somewhere
495
496 # Run unboxed-{build,install,smoke-test} INSIDE the container
497 local -a args=(
498 sh -c 'cd ~/oil; deps/wedge.sh unboxed "$1" "$2" "$3"'
499 dummy "$wedge" "$version_requested" "$wedge_guest_pkg_dir"
500 )
501
502 local -a docker_flags=()
503 if test -n "${WEDGE_LEAKY_BUILD:-}"; then
504 :
505 else
506 # Disable network for hermetic builds. TODO: Add automated test
507 docker_flags=( --network none )
508 fi
509
510 # TODO:
511 # - Don't mount the whole REPO_ROOT
512 # - We want the bare minimum of files, for cache invalidation
513 # - Maybe make it read only
514 # - Bind mount WEDGE_DEPS='', space separated list of paths
515 # - py3-libs depends on python3 and mypy wedges!
516
517 "${docker_prefix[@]}" run "${docker_flags[@]}" \
518 --mount "type=bind,source=$REPO_ROOT,target=/home/uke0/oil" \
519 --mount "type=bind,source=$PWD/$wedge_host_dir,target=$wedge_guest_dir" \
520 $bootstrap_image \
521 "${args[@]}"
522}
523
524boxed-2025() {
525 ### Build inside a container, and put output in a specific place.
526
527 # TODO: Specify the container OS, CPU like x86-64, etc.
528
529 local wedge=$1
530 local version_requested=${2:-}
531 # NOT USED. Because 2025 wedges are not absolute/relative. We use the same dir
532 local wedge_out_base_dir=${3:-}
533 local distro=${4:-$DEFAULT_DISTRO}
534
535 local bootstrap_image=oilshell/wedge-bootstrap-$distro
536
537 echo "*** boxed-2025 $wedge $version_requested out=$wedge_out_base_dir distro=$distro"
538 echo
539
540 load-wedge $wedge "$version_requested"
541
542 # Permissions will be different, so we separate the two
543
544 # Boxed wedges are put in this HOST dir, as opposed to as opposed to
545 # ../oils.DEPS for unboxed wedges
546 local wedge_host_dir=_build/boxed/wedge
547
548 local guest_repo_root=/home/uke0/oils
549 local guest_wedge_out_dir=/home/uke0/oils.DEPS/wedge
550
551 mkdir -v -p $wedge_host_dir
552
553 # TODO:
554 #
555 # Mount
556 # INPUTS: the PKG.wedge.sh, and the tarball
557 # CODE: this script: deps/wedge.sh
558 # OUTPUT: /wedge/oils-for-unix.org
559 # TODO: Also put logs and symbols somewhere
560
561 # Run unboxed-{build,install,smoke-test} INSIDE the container
562 local -a args=(
563 sh -c 'cd ~/oils; deps/wedge.sh unboxed "$1" "$2" "$3"'
564 dummy "$wedge" "$version_requested" "$guest_wedge_out_dir"
565 )
566
567 local -a docker_flags=()
568 if test -n "${WEDGE_LEAKY_BUILD:-}"; then
569 :
570 else
571 # Disable network for hermetic builds. TODO: Add automated test
572 docker_flags=( --network none )
573 fi
574
575 # TODO:
576 # - Don't mount the whole REPO_ROOT
577 # - We want the bare minimum of files, for cache invalidation
578 # - Maybe make it read only
579 # - Bind mount WEDGE_DEPS='', space separated list of paths
580 # - py3-libs depends on python3 and mypy wedges!
581
582 # -E to preserve CONTAINERS_REGISTRIES_CONF
583 "${docker_prefix[@]}" run "${docker_flags[@]}" \
584 --env WEDGE_2025=1 \
585 --mount "type=bind,source=$REPO_ROOT,target=$guest_repo_root" \
586 --mount "type=bind,source=$PWD/$wedge_host_dir,target=$guest_wedge_out_dir" \
587 $bootstrap_image \
588 "${args[@]}"
589}
590
591smoke-test() {
592 local wedge_dir=$1
593 local wedge_out_dir=${2:-_build/wedge/binary} # TODO: rename to /boxed
594 local version_requested=${3:-}
595 local distro=${4:-$DEFAULT_DISTRO}
596 local debug_shell=${5:-}
597
598 load-wedge $wedge_dir "$version_requested"
599
600 local bootstrap_image=oilshell/wedge-bootstrap-$distro
601
602 local -a args=(
603 sh -c 'cd "$1"; deps/wedge.sh unboxed-smoke-test $2' dummy "$OILS_GUEST_DIR" "$wedge_dir"
604 )
605 local -a docker_flags=()
606 if test -n "$debug_shell"; then
607 docker_flags=( -i -t )
608 args=( "$debug_shell" )
609 fi
610
611 local wedge_mount_dir
612 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
613 wedge_mount_dir=/wedge
614 else
615 wedge_mount_dir=/home/uke0/wedge
616 fi
617 "${docker_prefix[@]}" run "${docker_flags[@]}" \
618 --network none \
619 --mount "type=bind,source=$REPO_ROOT,target=$OILS_GUEST_DIR" \
620 --mount "type=bind,source=$PWD/$wedge_out_dir,target=$wedge_mount_dir" \
621 $bootstrap_image \
622 "${args[@]}"
623}
624
625if [[ $# -eq 0 || $1 =~ ^(--help|-h)$ ]]; then
626 # A trick for help. TODO: Move this to a common file, and combine with help
627 # in test/spec.sh.
628
629 awk '
630 $0 ~ /^#/ { print $0 }
631 $0 !~ /^#/ { print ""; exit }
632 ' $0
633 exit
634fi
635
636case $1 in
637 validate|\
638 unboxed|\
639 unboxed-make|unboxed-install|_unboxed-install|\
640 unboxed-smoke-test|unboxed-stats|\
641 boxed|boxed-2025|smoke-test)
642 "$@"
643 ;;
644
645 *)
646 die "Invalid action '$1'"
647 ;;
648esac