OILS / build / deps.sh View on Github | oils.pub

1376 lines, 839 significant
1#!/usr/bin/env bash
2#
3# Script for contributors to build dev dependencies -- packaged as cross-distro
4# "wedges". Tested in the Soil CI.
5#
6# Usage:
7# build/deps.sh <function name>
8#
9# Examples:
10# build/deps.sh fetch
11# build/deps.sh install-wedges # contributor wedges, to build both Python and C++
12# build/deps.sh install-wedges soil # more wedges for Soil CI container, on host
13# build/deps.sh boxed-wedges-2025 # build for copying inside OCI container
14#
15# Note: could be build/deps.sh unboxed-wedges versus boxed-wedges, and then
16# install-wedges as an ALIAS
17#
18# build/deps.sh rm-oils-crap # rm -r -f /wedge ~/wedge to start over
19#
20
21# Contributed sanity check, for getting started
22if ! test -d stdlib/osh; then
23 echo 'Run this script from the Oils repo root'
24 exit 1
25fi
26
27: ${LIB_OSH=stdlib/osh}
28source $LIB_OSH/bash-strict.sh
29source $LIB_OSH/task-five.sh
30
31REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
32
33source build/dev-shell.sh # We generally assume this, but let's be safe
34source deps/from-apt.sh # PY3_BUILD_DEPS
35source test/tsv-lib.sh # tsv-concat
36source web/table/html.sh # table-sort-{begin,end}
37
38# Also in build/dev-shell.sh
39USER_WEDGE_DIR=~/wedge/oils-for-unix.org/pkg
40ROOT_WEDGE_DIR=/wedge/oils-for-unix.org/pkg
41
42readonly WEDGE_2025_DIR=../oils.DEPS/wedge
43mkdir -p $WEDGE_2025_DIR
44
45readonly DEPS_SOURCE_DIR=_build/deps-source
46
47readonly RE2C_VERSION=3.0
48readonly RE2C_URL="https://github.com/skvadrik/re2c/releases/download/$RE2C_VERSION/re2c-$RE2C_VERSION.tar.xz"
49
50readonly CMARK_VERSION=0.29.0
51readonly CMARK_URL="https://github.com/commonmark/cmark/archive/$CMARK_VERSION.tar.gz"
52
53readonly PY_FTP_MIRROR="${PY_FTP_MIRROR:-https://www.python.org/ftp}"
54
55readonly PY2_VERSION=2.7.18
56readonly PY2_URL="$PY_FTP_MIRROR/python/$PY2_VERSION/Python-$PY2_VERSION.tar.xz"
57
58readonly PY3_VERSION=3.10.4
59readonly PY3_URL="$PY_FTP_MIRROR/python/$PY3_VERSION/Python-$PY3_VERSION.tar.xz"
60
61readonly BASH_VER=4.4 # don't clobber BASH_VERSION
62readonly BASH_URL="https://www.oilshell.org/blob/spec-bin/bash-$BASH_VER.tar.gz"
63
64# Another version of bash to test
65readonly BASH5_VER=5.2.21
66readonly BASH5_URL="https://www.oilshell.org/blob/spec-bin/bash-$BASH5_VER.tar.gz"
67
68readonly DASH_VERSION=0.5.10.2
69readonly DASH_URL="https://www.oilshell.org/blob/spec-bin/dash-$DASH_VERSION.tar.gz"
70
71readonly ZSH_OLD_VER=5.1.1
72readonly ZSH_OLD_URL="https://www.oilshell.org/blob/spec-bin/zsh-$ZSH_OLD_VER.tar.xz"
73
74readonly ZSH_NEW_VER=5.9
75readonly ZSH_NEW_URL="https://www.oilshell.org/blob/spec-bin/zsh-$ZSH_NEW_VER.tar.xz"
76
77readonly MKSH_VERSION=R52c
78readonly MKSH_URL="https://www.oilshell.org/blob/spec-bin/mksh-$MKSH_VERSION.tgz"
79
80readonly BUSYBOX_VERSION='1.35.0'
81readonly BUSYBOX_URL="https://www.oilshell.org/blob/spec-bin/busybox-$BUSYBOX_VERSION.tar.bz2"
82
83readonly YASH_VERSION=2.49
84readonly YASH_URL="https://www.oilshell.org/blob/spec-bin/yash-$YASH_VERSION.tar.xz"
85
86readonly MYPY_GIT_URL=https://github.com/python/mypy
87readonly MYPY_VERSION=0.780
88
89readonly PY3_LIBS=~/wedge/oils-for-unix.org/pkg/py3-libs/$MYPY_VERSION
90
91# Version 2.4.0 from 2021-10-06 was the last version that supported Python 2
92# https://github.com/PyCQA/pyflakes/blob/main/NEWS.rst
93readonly PYFLAKES_VERSION=2.4.0
94#readonly PYFLAKES_URL='https://files.pythonhosted.org/packages/15/60/c577e54518086e98470e9088278247f4af1d39cb43bcbd731e2c307acd6a/pyflakes-2.4.0.tar.gz'
95# 2023-07: Mirrored to avoid network problem on broome during release
96readonly PYFLAKES_URL='https://www.oilshell.org/blob/pyflakes-2.4.0.tar.gz'
97
98readonly BLOATY_VERSION=1.1
99readonly BLOATY_URL='https://github.com/google/bloaty/releases/download/v1.1/bloaty-1.1.tar.bz2'
100
101readonly UFTRACE_VERSION=0.13
102readonly UFTRACE_URL='https://github.com/namhyung/uftrace/archive/refs/tags/v0.13.tar.gz'
103
104readonly SOUFFLE_VERSION=2.4.1
105readonly SOUFFLE_URL=https://github.com/souffle-lang/souffle/archive/refs/tags/2.4.1.tar.gz
106
107readonly R_LIBS_VERSION='2023-04-18'
108readonly TIME_HELPER_VERSION='2023-02-28'
109readonly PY3_LIBS_VERSION=2023-03-04
110
111readonly WEDGE_LOG_DIR=_build/wedge/logs
112
113readonly BOXED_WEDGE_DIR=_build/boxed/wedge
114readonly BOXED_LOG_DIR=_build/boxed/logs
115
116log() {
117 echo "$@" >& 2
118}
119
120die() {
121 log "$0: fatal: $@"
122 exit 1
123}
124
125rm-oils-crap() {
126 ### Remove OLD wedges (prior to 2025)
127
128 rm -r -f -v ~/wedge
129 sudo rm -r -f -v /wedge
130}
131
132# Note: git is an implicit dependency -- that's how we got the repo in the
133# first place!
134
135# python2-dev is no longer available on Debian 12
136# python-dev also seems gone
137#
138# wget: for fetching wedges (not on Debian by default!)
139# tree: tiny package that's useful for showing what we installed
140# g++: essential
141# libreadline-dev: needed for the build/prepare.sh Python build.
142# gawk: used by spec-runner.sh for the special match() function.
143# cmake: for cmark
144# PY3_BUILD_DEPS - I think these will be used for building the Python 2 wedge
145# as well
146readonly -a WEDGE_DEPS_DEBIAN=(
147 bzip2
148 wget
149 tree
150 gawk
151 g++
152 ninja-build
153 cmake
154 libreadline-dev
155 systemtap-sdt-dev
156
157 # for Souffle, flex and bison
158 #flex bison
159
160 "${PY3_BUILD_DEPS[@]}"
161 "${SPEC_TEST_DEPS[@]}"
162)
163
164readonly -a WEDGE_DEPS_ALPINE=(
165 coreutils findutils
166
167 bzip2
168 xz
169
170 wget tree gawk
171
172 gcc g++
173 ninja-build
174 # https://pkgs.alpinelinux.org/packages?name=ninja-is-really-ninja&branch=v3.19&repo=&arch=&maintainer=
175 ninja-is-really-ninja
176 cmake
177
178 readline-dev
179 zlib-dev
180 libffi-dev
181 openssl-dev
182
183 ncurses-dev
184
185 # for Souffle, flex and bison
186 #flex bison
187)
188
189readonly -a WEDGE_DEPS_FEDORA=(
190
191 # Weird, Fedora doesn't have these by default!
192 hostname
193 tar
194 bzip2
195
196 # https://packages.fedoraproject.org/pkgs/wget/wget/
197 wget
198 # https://packages.fedoraproject.org/pkgs/tree-pkg/tree/
199 tree
200 gawk
201
202 # https://packages.fedoraproject.org/pkgs/gcc/gcc/
203 gcc gcc-c++
204
205 ninja-build
206 cmake
207
208 readline-devel
209
210 # Like PY3_BUILD_DEPS
211 # https://packages.fedoraproject.org/pkgs/zlib/zlib-devel/
212 zlib-devel
213 # https://packages.fedoraproject.org/pkgs/libffi/libffi-devel/
214 libffi-devel
215 # https://packages.fedoraproject.org/pkgs/openssl/openssl-devel/
216 openssl-devel
217
218 # For building zsh from source?
219 # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
220 ncurses-devel
221 #libcap-devel
222
223 # still have a job control error compiling bash
224 # https://packages.fedoraproject.org/pkgs/glibc/glibc-devel/
225 # glibc-devel
226
227 libasan
228)
229
230readonly -a WEDGE_DEPS_ARCH=(
231 # https://archlinux.org/packages/core/x86_64/bzip2/
232 bzip2
233
234 # https://archlinux.org/packages/extra/x86_64/wget/
235 wget
236
237 # https://archlinux.org/packages/extra/x86_64/tree/
238 tree
239
240 # https://archlinux.org/packages/core/x86_64/gawk/
241 gawk
242
243 # https://archlinux.org/packages/core/x86_64/gcc/
244 gcc
245
246 # https://archlinux.org/packages/community/x86_64/ninja/
247 ninja
248
249 # https://archlinux.org/packages/extra/x86_64/cmake/
250 cmake
251
252 # https://archlinux.org/packages/core/x86_64/readline/
253 readline
254
255 # https://archlinux.org/packages/core/x86_64/zlib/
256 zlib
257
258 # https://archlinux.org/packages/core/x86_64/libffi/
259 libffi
260
261 # https://archlinux.org/packages/core/x86_64/openssl/
262 openssl
263
264 # https://archlinux.org/packages/core/x86_64/ncurses/
265 ncurses
266
267 # Development headers are included in the main packages on Arch,
268 # unlike other distros that separate them into -dev/-devel packages
269
270 # Python 2 from the AUR
271 # https://aur.archlinux.org/packages/python2
272 base-devel # needed for building packages from the AUR
273
274)
275
276install-debian-packages() {
277 ### Packages for build/py.sh all, building wedges, etc.
278
279 set -x # show what needs sudo
280
281 # pass -y for say gitpod
282 sudo apt "$@" install "${WEDGE_DEPS_DEBIAN[@]}"
283 set +x
284
285 # maybe pass -y through
286 test/spec-bin.sh install-shells-with-apt "$@"
287}
288
289install-ubuntu-packages() {
290 ### Debian and Ubuntu packages are the same; this function is suggested on the wiki
291 install-debian-packages "$@"
292}
293
294wedge-deps-debian() {
295 # Install packages without prompt
296
297 # 2024-02 - there was an Ubuntu update, and we started needing this
298 sudo apt-get -y update
299
300 install-debian-packages -y
301}
302
303wedge-deps-fedora() {
304 # https://linuxconfig.org/install-development-tools-on-redhat-8
305 # Trying to get past compile errors
306 # sudo dnf group install --assumeyes 'Development Tools'
307
308 sudo dnf install --assumeyes "${WEDGE_DEPS_FEDORA[@]}"
309}
310
311wedge-deps-alpine() {
312 # https://linuxconfig.org/install-development-tools-on-redhat-8
313 # Trying to get past compile errors
314 # sudo dnf group install --assumeyes 'Development Tools'
315
316 sudo apk add "${WEDGE_DEPS_ALPINE[@]}"
317}
318
319wedge-deps-arch() {
320 # Install packages without prompt
321
322 # First sync the package database
323 sudo pacman -Sy
324
325 # Then install packages
326 for pkg in "${WEDGE_DEPS_ARCH[@]}"; do
327 # Only install if not already installed
328 if ! pacman -Qi "$pkg" >/dev/null 2>&1; then
329 sudo pacman --noconfirm -S "$pkg"
330 fi
331 done
332}
333
334#
335# Fetch
336#
337
338download-to() {
339 local dir=$1
340 local url=$2
341 wget --no-clobber --directory-prefix "$dir" "$url"
342}
343
344maybe-extract() {
345 local wedge_dir=$1
346 local tar_name=$2
347 local out_dir=$3
348
349 if test -d "$wedge_dir/$out_dir"; then
350 log "Not extracting because $wedge_dir/$out_dir exists"
351 return
352 fi
353
354 local tar=$wedge_dir/$tar_name
355 case $tar_name in
356 *.gz|*.tgz) # mksh ends with .tgz
357 flag='--gzip'
358 ;;
359 *.bz2)
360 flag='--bzip2'
361 ;;
362 *.xz)
363 flag='--xz'
364 ;;
365 *)
366 die "tar with unknown extension: $tar_name"
367 ;;
368 esac
369
370 tar --extract $flag --file $tar --directory $wedge_dir
371}
372
373clone-mypy() {
374 ### replaces deps/from-git
375 local dest_dir=$1
376 local version=${2:-$MYPY_VERSION}
377
378 local dest=$dest_dir/mypy-$version
379 if test -d $dest; then
380 log "Not cloning because $dest exists"
381 return
382 fi
383
384 # v$VERSION is a tag, not a branch
385
386 # size optimization: --depth=1 --shallow-submodules
387 # https://git-scm.com/docs/git-clone
388
389 git clone --recursive --branch v$version \
390 --depth=1 --shallow-submodules \
391 $MYPY_GIT_URL $dest
392
393 # TODO: verify commit checksum
394}
395
396copy-source-medo() {
397 mkdir -p $DEPS_SOURCE_DIR
398
399 # Copy the whole tree, including the .treeptr files
400 cp --verbose --recursive --no-target-directory \
401 deps/source.medo/ $DEPS_SOURCE_DIR/
402}
403
404fetch-spec-bin() {
405 download-to $DEPS_SOURCE_DIR/bash "$BASH_URL"
406 maybe-extract $DEPS_SOURCE_DIR/bash "$(basename $BASH_URL)" bash-$BASH_VER
407
408 download-to $DEPS_SOURCE_DIR/bash "$BASH5_URL"
409 maybe-extract $DEPS_SOURCE_DIR/bash "$(basename $BASH5_URL)" bash-$BASH5_VER
410
411 download-to $DEPS_SOURCE_DIR/dash "$DASH_URL"
412 maybe-extract $DEPS_SOURCE_DIR/dash "$(basename $DASH_URL)" dash-$DASH_VERSION
413
414 download-to $DEPS_SOURCE_DIR/zsh "$ZSH_OLD_URL"
415 maybe-extract $DEPS_SOURCE_DIR/zsh "$(basename $ZSH_OLD_URL)" zsh-$ZSH_OLD_VER
416
417 download-to $DEPS_SOURCE_DIR/zsh "$ZSH_NEW_URL"
418 maybe-extract $DEPS_SOURCE_DIR/zsh "$(basename $ZSH_NEW_URL)" zsh-$ZSH_NEW_VER
419
420 download-to $DEPS_SOURCE_DIR/mksh "$MKSH_URL"
421 maybe-extract $DEPS_SOURCE_DIR/mksh "$(basename $MKSH_URL)" mksh-$MKSH_VERSION
422
423 download-to $DEPS_SOURCE_DIR/busybox "$BUSYBOX_URL"
424 maybe-extract $DEPS_SOURCE_DIR/busybox "$(basename $BUSYBOX_URL)" busybox-$BUSYBOX_VERSION
425
426 download-to $DEPS_SOURCE_DIR/yash "$YASH_URL"
427 maybe-extract $DEPS_SOURCE_DIR/yash "$(basename $YASH_URL)" yash-$YASH_VERSION
428
429 # Patch: this tarball doesn't follow the convention $name-$version
430 if test -d $DEPS_SOURCE_DIR/mksh/mksh; then
431 pushd $DEPS_SOURCE_DIR/mksh
432 mv -v mksh mksh-$MKSH_VERSION
433 popd
434 fi
435}
436
437fetch() {
438 local py_only=${1:-}
439
440 # For now, simulate what 'medo expand deps/source.medo _build/deps-source'
441 # would do: fetch compressed tarballs designated by .treeptr files, and
442 # expand them.
443
444 # _build/deps-source/
445 # re2c/
446 # WEDGE
447 # re2c-3.0/ # expanded .tar.xz file
448
449 copy-source-medo
450
451 # Hack
452 local dest=$DEPS_SOURCE_DIR/time-helper/time-helper-$TIME_HELPER_VERSION
453 mkdir -p $dest
454 cp -v benchmarks/time-helper.c $dest
455
456 download-to $DEPS_SOURCE_DIR/re2c "$RE2C_URL"
457 download-to $DEPS_SOURCE_DIR/cmark "$CMARK_URL"
458 maybe-extract $DEPS_SOURCE_DIR/re2c "$(basename $RE2C_URL)" re2c-$RE2C_VERSION
459 maybe-extract $DEPS_SOURCE_DIR/cmark "$(basename $CMARK_URL)" cmark-$CMARK_VERSION
460
461 if test -n "$py_only"; then
462 log "Fetched dependencies for 'build/py.sh'"
463 return
464 fi
465
466 download-to $DEPS_SOURCE_DIR/pyflakes "$PYFLAKES_URL"
467 maybe-extract $DEPS_SOURCE_DIR/pyflakes "$(basename $PYFLAKES_URL)" \
468 pyflakes-$PYFLAKES_VERSION
469
470 download-to $DEPS_SOURCE_DIR/python2 "$PY2_URL"
471 download-to $DEPS_SOURCE_DIR/python3 "$PY3_URL"
472 maybe-extract $DEPS_SOURCE_DIR/python2 "$(basename $PY2_URL)" Python-$PY2_VERSION
473 maybe-extract $DEPS_SOURCE_DIR/python3 "$(basename $PY3_URL)" Python-$PY3_VERSION
474
475 fetch-spec-bin
476
477 # bloaty and uftrace are for benchmarks, in containers
478 download-to $DEPS_SOURCE_DIR/bloaty "$BLOATY_URL"
479 download-to $DEPS_SOURCE_DIR/uftrace "$UFTRACE_URL"
480 maybe-extract $DEPS_SOURCE_DIR/bloaty "$(basename $BLOATY_URL)" bloaty-$BLOATY_VERSION
481 maybe-extract $DEPS_SOURCE_DIR/uftrace "$(basename $UFTRACE_URL)" uftrace-$UFTRACE_VERSION
482
483 # This is in $DEPS_SOURCE_DIR to COPY into containers, which mycpp will directly import.
484
485 # It's also copied into a wedge in install-wedges.
486 clone-mypy $DEPS_SOURCE_DIR/mypy
487
488 if false; then
489 download-to $DEPS_SOURCE_DIR/souffle "$SOUFFLE_URL"
490 maybe-extract $DEPS_SOURCE_DIR/souffle "$(basename $SOUFFLE_URL)" souffle-$SOUFFLE_VERSION
491 fi
492
493 if command -v tree > /dev/null; then
494 tree -L 2 $DEPS_SOURCE_DIR
495 fi
496}
497
498fetch-py() {
499 fetch py_only
500}
501
502mirror-pyflakes() {
503 ### Workaround for network error during release
504 scp \
505 $DEPS_SOURCE_DIR/pyflakes/"$(basename $PYFLAKES_URL)" \
506 oilshell.org:oilshell.org/blob/
507}
508
509mirror-python() {
510 ### Can't reach python.org from some machines
511 scp \
512 $DEPS_SOURCE_DIR/python2/"$(basename $PY2_URL)" \
513 oilshell.org:oilshell.org/blob/
514
515 scp \
516 $DEPS_SOURCE_DIR/python3/"$(basename $PY3_URL)" \
517 oilshell.org:oilshell.org/blob/
518}
519
520mirror-zsh() {
521 scp \
522 _tmp/zsh-5.9.tar.xz \
523 oilshell.org:oilshell.org/blob/
524}
525
526wedge-exists() {
527 ### Does an installed wedge already exist?
528 local name=$1
529 local version=$2
530 local wedge_base_dir=${3:-/wedge/oils-for-unix.org} # e.g. ../oils.DEPS/wedge
531
532 local installed=$wedge_base_dir/$name/$version
533
534 if test -d $installed; then
535 log "$installed already exists"
536 return 0
537 else
538 return 1
539 fi
540}
541
542boxed-wedge-exists() {
543 ### Does an installed wedge already exist?
544
545 local name=$1
546 local version=$2
547
548 # NOT USED for now
549 local distro=${3:-debian-12}
550
551 local installed=$BOXED_WEDGE_DIR/$name/$version
552
553 if test -d $installed; then
554 log "$installed already exists"
555 return 0
556 else
557 return 1
558 fi
559}
560
561#
562# Special case of py3-libs
563#
564# TODO: it should a WEDGE
565# It needs a BUILD DEPENDENCY on:
566# - the python3 wedge, so you can do python3 -m pip install.
567# - the mypy repo, which has test-requirements.txt
568#
569
570upgrade-typed-ast() {
571 local file=$1
572 sed -i 's/typed_ast.*/typed_ast==1.5.0/' $file
573}
574
575test-typed-ast() {
576 local dir=~/wedge/oils-for-unix.org/pkg/mypy/0.780
577
578 cp -v $dir/mypy-requirements.txt _tmp
579
580 local file=_tmp/mypy-requirements.txt
581 cat $file
582 #echo
583
584 # 1.5.0 fixed this bug
585 # https://github.com/python/typed_ast/issues/169
586
587 upgrade-typed-ast $file
588 echo
589 cat $file
590}
591
592download-py3-libs() {
593 ### Download source/binary packages, AFTER python3 is installed
594
595 # Note that this is NOT source code; there is binary code, e.g. in
596 # lxml-*.whl
597
598 local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
599 local py_package_dir=_cache/py3-libs
600 mkdir -p $py_package_dir
601
602 python3 -m pip download -d $py_package_dir -r $mypy_dir/test-requirements.txt
603 python3 -m pip download -d $py_package_dir pexpect pyte
604}
605
606install-py3-libs-in-venv() {
607 local venv_dir=$1
608 local mypy_dir=$2 # This is a param for host build vs. container build
609 local package_dir=_cache/py3-libs
610
611 source $venv_dir/bin/activate # enter virtualenv
612
613 # 2023-07 note: we're installing yapf in a DIFFERENT venv, because it
614 # conflicts with MyPy deps!
615 # "ERROR: pip's dependency resolver does not currently take into account all
616 # the packages that are installed."
617
618 # --find-links uses a "cache dir" for packages (weird flag name!)
619
620 # Avoids a warning, but doesn't fix typed_ast
621 #time python3 -m pip install --find-links $package_dir wheel
622
623 upgrade-typed-ast $mypy_dir/mypy-requirements.txt
624
625 # for mycpp/
626 time python3 -m pip install --find-links $package_dir -r $mypy_dir/test-requirements.txt
627
628 # pexpect: for spec/stateful/*.py
629 time python3 -m pip install --find-links $package_dir pexpect pyte
630}
631
632install-py3-libs-from-cache() {
633 # As well as end users
634 local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
635 local wedge_out_dir=${2:-$USER_WEDGE_DIR}
636
637 log "Ensuring pip is installed (interpreter $(command -v python3)"
638 python3 -m ensurepip
639
640 local venv_dir=$wedge_out_dir/py3-libs/$PY3_LIBS_VERSION
641 log "Creating venv in $venv_dir"
642
643 # Note: the bin/python3 in this venv is a symlink to python3 in $PATH, i.e.
644 # the /wedge we just built
645 python3 -m venv $venv_dir
646
647 log "Installing MyPy deps in venv"
648
649 # Run in a subshell because it mutates shell state
650 $0 install-py3-libs-in-venv $venv_dir $mypy_dir
651}
652
653install-py3-libs() {
654 ### Invoked by Dockerfile.cpp-small, etc. and by fake-py3-libs-wedge
655 local mypy_dir=${1:-}
656 local wedge_out_dir=${2:-}
657
658 local py3
659 py3=$(command -v python3)
660 case $py3 in
661 *wedge/oils-for-unix.org/*)
662 ;;
663 *oils.DEPS/*)
664 ;;
665 *)
666 die "python3 is '$py3', but expected it to be in a wedge"
667 ;;
668 esac
669
670 log "python3 is $py3"
671
672 download-py3-libs $mypy_dir
673 install-py3-libs-from-cache "$mypy_dir" "$wedge_out_dir"
674}
675
676#
677# Wedge manifests
678#
679
680py-wedges() {
681 ### for build/py.sh all
682 local how=${1:-legacy}
683
684 if test $how = 'boxed'; then
685 local where='debian-12'
686 else
687 local where='HOST'
688 fi
689 case $how in
690 boxed|unboxed)
691 echo time-helper $TIME_HELPER_VERSION $WEDGE_2025_DIR $where
692 echo cmark $CMARK_VERSION $WEDGE_2025_DIR $where
693 echo re2c $RE2C_VERSION $WEDGE_2025_DIR $where
694 echo python2 $PY2_VERSION $WEDGE_2025_DIR $where
695 echo pyflakes $PYFLAKES_VERSION $WEDGE_2025_DIR $where
696 ;;
697 legacy)
698 echo time-helper $TIME_HELPER_VERSION $ROOT_WEDGE_DIR $where
699 echo cmark $CMARK_VERSION $ROOT_WEDGE_DIR $where
700 echo re2c $RE2C_VERSION $ROOT_WEDGE_DIR $where
701 echo python2 $PY2_VERSION $ROOT_WEDGE_DIR $where
702 echo pyflakes $PYFLAKES_VERSION $USER_WEDGE_DIR $where
703 ;;
704 *)
705 die "Invalid how $how"
706 esac
707}
708
709cpp-wedges() {
710 ### for ninja / mycpp translation
711 local how=${1:-legacy}
712
713 if test $how = 'boxed'; then
714 local where='debian-12'
715 else
716 local where='HOST'
717 fi
718 case $how in
719 boxed|unboxed)
720 echo python3 $PY3_VERSION $WEDGE_2025_DIR $where
721 echo mypy $MYPY_VERSION $WEDGE_2025_DIR $where
722 ;;
723 legacy)
724 echo python3 $PY3_VERSION $ROOT_WEDGE_DIR $where
725 echo mypy $MYPY_VERSION $USER_WEDGE_DIR $where
726 ;;
727 *)
728 die "Invalid how $how"
729 esac
730
731 # py3-libs has a built time dep on both python3 and MyPy, so we're doing it
732 # separately for now
733 #echo py3-libs $PY3_LIBS_VERSION $USER_WEDGE_DIR
734}
735
736spec-bin-wedges() {
737 ### for test/spec-py.sh osh-all
738 local how=${1:-legacy}
739
740 if test $how = 'boxed'; then
741 local where='debian-12'
742 else
743 local where='HOST'
744 fi
745 case $how in
746 boxed|unboxed)
747 echo dash $DASH_VERSION $WEDGE_2025_DIR $where
748 echo bash $BASH_VER $WEDGE_2025_DIR $where
749 echo bash $BASH5_VER $WEDGE_2025_DIR $where
750 echo mksh $MKSH_VERSION $WEDGE_2025_DIR $where
751 echo zsh $ZSH_OLD_VER $WEDGE_2025_DIR $where
752 echo zsh $ZSH_NEW_VER $WEDGE_2025_DIR $where
753 echo busybox $BUSYBOX_VERSION $WEDGE_2025_DIR $where
754 echo yash $YASH_VERSION $WEDGE_2025_DIR $where
755 ;;
756 legacy)
757 echo dash $DASH_VERSION $USER_WEDGE_DIR $where
758 echo bash $BASH_VER $USER_WEDGE_DIR $where
759 echo bash $BASH5_VER $USER_WEDGE_DIR $where
760 echo mksh $MKSH_VERSION $USER_WEDGE_DIR $where
761 echo zsh $ZSH_OLD_VER $USER_WEDGE_DIR $where
762 echo zsh $ZSH_NEW_VER $USER_WEDGE_DIR $where
763 echo busybox $BUSYBOX_VERSION $USER_WEDGE_DIR $where
764 echo yash $YASH_VERSION $USER_WEDGE_DIR $where
765 ;;
766 *)
767 die "Invalid how $how"
768 esac
769}
770
771zsh-wedges() {
772 local how=${1:-legacy}
773
774 if test $how = 'boxed'; then
775 local where='debian-12'
776 else
777 local where='HOST'
778 fi
779 case $how in
780 boxed|unboxed)
781 echo zsh $ZSH_OLD_VER $WEDGE_2025_DIR $where
782 echo zsh $ZSH_NEW_VER $WEDGE_2025_DIR $where
783 ;;
784 legacy)
785 echo zsh $ZSH_OLD_VER $USER_WEDGE_DIR $where
786 echo zsh $ZSH_NEW_VER $USER_WEDGE_DIR $where
787 ;;
788 *)
789 die "Invalid how $how"
790 esac
791}
792
793smoke-wedges() {
794 local how=${1:-legacy}
795
796 if test $how = 'boxed'; then
797 local where='debian-12'
798 else
799 local where='HOST'
800 fi
801 case $how in
802 boxed|unboxed)
803 echo dash $DASH_VERSION $WEDGE_2025_DIR $where
804 ;;
805 legacy)
806 echo dash $DASH_VERSION $USER_WEDGE_DIR $where
807 ;;
808 *)
809 die "Invalid how $how"
810 esac
811}
812
813cmark-wedges() {
814 local how=${1:-legacy}
815
816 if test $how = 'boxed'; then
817 local where='debian-12'
818 else
819 local where='HOST'
820 fi
821 case $how in
822 boxed|unboxed)
823 echo cmark $CMARK_VERSION $WEDGE_2025_DIR $where
824 ;;
825 legacy)
826 echo cmark $CMARK_VERSION $ROOT_WEDGE_DIR $where
827 ;;
828 *)
829 die "Invalid how $how"
830 esac
831}
832
833extra-wedges() {
834 # Contributors don't need uftrace, bloaty, and probably R-libs
835 # Although R-libs could be useful for benchmarks
836
837 local how=${1:-legacy}
838
839 case $how in
840 boxed)
841 echo R-libs $R_LIBS_VERSION $WEDGE_2025_DIR debian-12
842 echo uftrace $UFTRACE_VERSION $WEDGE_2025_DIR debian-12
843 echo bloaty $BLOATY_VERSION $WEDGE_2025_DIR debian-10 # It works on Debian 10, not 12
844 ;;
845 unboxed)
846 echo R-libs $R_LIBS_VERSION $WEDGE_2025_DIR HOST
847 echo uftrace $UFTRACE_VERSION $WEDGE_2025_DIR HOST
848 echo bloaty $BLOATY_VERSION $WEDGE_2025_DIR HOST
849 ;;
850 legacy)
851 echo R-libs $R_LIBS_VERSION $USER_WEDGE_DIR HOST
852 echo uftrace $UFTRACE_VERSION $ROOT_WEDGE_DIR HOST
853 echo bloaty $BLOATY_VERSION $ROOT_WEDGE_DIR HOST
854 ;;
855 *)
856 die "Invalid how $how"
857 esac
858}
859
860contributor-wedges() {
861 local how=${1:-}
862
863 py-wedges "$how"
864 cpp-wedges "$how"
865 spec-bin-wedges "$how"
866}
867
868#
869# More
870#
871
872timestamp() {
873 date '+%H:%M:%S'
874}
875
876my-time-tsv() {
877 python3 benchmarks/time_.py \
878 --tsv \
879 --time-span --rusage \
880 "$@"
881}
882
883maybe-install-wedge() {
884 local how=${1:-legacy}
885 local name=$2
886 local version=$3
887 local wedge_base_dir=$4
888 local where=$5
889
890 if test $where != HOST; then
891 die 'Expected $where to be "HOST"'
892 fi
893
894 local task_file=$WEDGE_LOG_DIR/$name-$version.task.tsv
895 local log_file=$WEDGE_LOG_DIR/$name-$version.log.txt
896
897 echo " TASK $(timestamp) $name $version > $log_file"
898
899 # python3 because it's OUTSIDE the container
900 # Separate columns that could be joined: number of files, total size
901 my-time-tsv --print-header \
902 --field xargs_slot \
903 --field wedge \
904 --field wedge_HREF \
905 --field version \
906 --output $task_file
907
908 if wedge-exists "$name" "$version" "$wedge_base_dir"; then
909 echo "CACHED $(timestamp) $name $version"
910 return
911 fi
912
913 case $how in
914 unboxed)
915 # TODO: I think all the builds should set the install dir, which is the LOCAL output
916 # - wedge unboxed
917 # - wedge unboxed-2025
918 # - wedge boxed-2025
919 # instead of relying on this flag
920 #
921 # So we have (NAME, VERSION, INSTALL_OUT) as our params. That makes sense
922 export WEDGE_2025=1
923 ;;
924 esac
925
926 local -a cmd=( deps/wedge.sh unboxed _build/deps-source/$name/ $version $wedge_base_dir)
927
928 set +o errexit
929 my-time-tsv \
930 --field "$XARGS_SLOT" \
931 --field "$name" \
932 --field "$name-$version.log.txt" \
933 --field "$version" \
934 --append \
935 --output $task_file \
936 -- \
937 "${cmd[@]}" "$@" >$log_file 2>&1
938 local status=$?
939 set -o errexit
940
941 if test "$status" -eq 0; then
942 echo " OK $(timestamp) $name $version"
943 else
944 echo " FAIL $(timestamp) $name $version"
945 fi
946}
947
948dummy-task() {
949 ### For testing log capture
950 local name=$1
951 local version=$2
952
953 echo "Building $name $version"
954
955 # random float between 0 and 3
956 # weirdly we need a seed from bash
957 # https://stackoverflow.com/questions/4048378/random-numbers-generation-with-awk-in-bash-shell
958 local secs
959 secs=$(awk -v seed=$RANDOM 'END { srand(seed); print rand() * 3 }' < /dev/null)
960
961 echo "sleep $secs"
962 sleep $secs
963
964 echo 'stdout'
965 log 'stderr'
966
967 if test $name = 'mksh'; then
968 echo "simulate failure for $name"
969 exit 2
970 fi
971}
972
973dummy-task-wrapper() {
974 # Similar to test/common.sh run-task-with-status, used by
975 # test/{spec,wild}-runner.sh
976 local name=$1
977 local version=$2
978
979 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
980 local log_file=$WEDGE_LOG_DIR/$name.log.txt
981
982 echo " TASK $(timestamp) $name $version > $log_file"
983
984 # python3 because it's OUTSIDE the container
985 # Separate columns that could be joined: number of files, total size
986 my-time-tsv --print-header \
987 --field xargs_slot \
988 --field wedge \
989 --field wedge_HREF \
990 --field version \
991 --output $task_file
992
993 my-time-tsv \
994 --field "$XARGS_SLOT" \
995 --field "$name" \
996 --field "$name.log.txt" \
997 --field "$version" \
998 --append \
999 --output $task_file \
1000 $0 dummy-task "$@" >$log_file 2>&1 || true
1001
1002 echo " DONE $(timestamp) $name $version"
1003}
1004
1005html-head() {
1006 # python3 because we're outside containers
1007 PYTHONPATH=. python3 doctools/html_head.py "$@"
1008}
1009
1010index-html() {
1011 local tasks_tsv=$1
1012
1013 local base_url='../../../web'
1014 html-head --title 'Wedge Builds' \
1015 "$base_url/ajax.js" \
1016 "$base_url/table/table-sort.js" \
1017 "$base_url/table/table-sort.css" \
1018 "$base_url/base.css"
1019
1020 table-sort-begin 'width60'
1021
1022 cat <<EOF
1023 <p id="home-link">
1024 <a href="/">oils.pub</a>
1025 </p>
1026
1027 <h1>Wedge Builds</h1>
1028EOF
1029
1030 tsv2html3 $tasks_tsv
1031
1032 cat <<EOF
1033 <p>
1034 <a href="tasks.tsv">tasks.tsv</a>
1035 </p>
1036EOF
1037
1038 table-sort-end 'tasks' # ID for sorting
1039}
1040
1041NPROC=$(nproc)
1042#NPROC=1
1043
1044install-wedge-list() {
1045 ### Reads task rows from stdin
1046 local how=${1:-legacy} # unboxed | legacy
1047 local parallel=${2:-}
1048
1049 mkdir -p $WEDGE_LOG_DIR
1050
1051 local -a flags
1052 if test -n "$parallel"; then
1053 log ""
1054 log "=== Installing wedges with $NPROC jobs in parallel"
1055 log ""
1056 flags=( -P $NPROC )
1057 else
1058 log ""
1059 log "=== Installing wedges serially"
1060 log ""
1061 fi
1062
1063 # Reads from stdin
1064 # Note: --process-slot-var requires GNU xargs! busybox args doesn't have it.
1065 #
1066 # $name $version $wedge_dir
1067 xargs "${flags[@]}" -n 4 --process-slot-var=XARGS_SLOT -- $0 maybe-install-wedge "$how"
1068}
1069
1070write-task-report() {
1071 local log_dir=${1:-$WEDGE_LOG_DIR}
1072
1073 local tasks_tsv=$log_dir/tasks.tsv
1074
1075 python3 devtools/tsv_concat.py $log_dir/*.task.tsv > $tasks_tsv
1076 log "Wrote $tasks_tsv"
1077
1078 # TODO: version can be right-justified?
1079 here-schema-tsv-4col >$log_dir/tasks.schema.tsv <<EOF
1080column_name type precision strftime
1081status integer 0 -
1082elapsed_secs float 1 -
1083start_time float 1 %H:%M:%S
1084end_time float 1 %H:%M:%S
1085user_secs float 1 -
1086sys_secs float 1 -
1087max_rss_KiB integer 0 -
1088xargs_slot integer 0 -
1089wedge string 0 -
1090wedge_HREF string 0 -
1091version string 0 -
1092EOF
1093
1094 index-html $tasks_tsv > $log_dir/index.html
1095 log "Wrote $log_dir/index.html"
1096}
1097
1098fake-py3-libs-wedge() {
1099 local wedge_out_dir=${1:-}
1100
1101 local name=py3-libs
1102 local version=$PY3_LIBS_VERSION
1103
1104 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
1105 local log_file=$WEDGE_LOG_DIR/$name.log.txt
1106
1107 my-time-tsv --print-header \
1108 --field xargs_slot \
1109 --field wedge \
1110 --field wedge_HREF \
1111 --field version \
1112 --output $task_file
1113
1114 # There is no xargs slot!
1115 my-time-tsv \
1116 --field "-1" \
1117 --field "$name" \
1118 --field "$name.log.txt" \
1119 --field "$version" \
1120 --append \
1121 --output $task_file \
1122 -- \
1123 $0 install-py3-libs '' "$wedge_out_dir" >$log_file 2>&1 || true
1124
1125 echo " FAKE $(timestamp) $name $version"
1126}
1127
1128print-wedge-list() {
1129 local which_wedges=${1:-contrib} # contrib | soil | smoke
1130 local how=${2:-legacy} # boxed | unboxed | legacy
1131
1132 case $which_wedges in
1133 contrib)
1134 contributor-wedges "$how"
1135 ;;
1136 soil)
1137 contributor-wedges "$how"
1138 extra-wedges "$how"
1139 ;;
1140 extra-only)
1141 extra-wedges "$how"
1142 ;;
1143 smoke)
1144 #zsh-wedges "$how"
1145 smoke-wedges "$how"
1146 ;;
1147 cmark) # for testing mkdir /wedge BUG
1148 cmark-wedges "$how"
1149 ;;
1150 *)
1151 die "Invalid which_wedges $which_wedges"
1152 ;;
1153 esac
1154}
1155
1156install-wedges-parallel() {
1157 local which_wedges=${1:-contrib} # contrib | soil | smoke
1158 local how=${2:-legacy} # boxed | unboxed | legacy
1159
1160 # For contributor setup: we need to use this BEFORE running build/py.sh all
1161 build/py.sh time-helper
1162
1163 echo " START $(timestamp)"
1164
1165 # Do all of them in parallel
1166 print-wedge-list "$which_wedges" "$how" | install-wedge-list "$how" T
1167
1168 # python3 interpreter is needed to download and install python packages
1169 mkdir -p ../oils.DEPS/bin # must create it
1170 deps/make-bin.sh python3 ../oils.DEPS
1171
1172 local wedge_out_dir
1173 case $how in
1174 unboxed)
1175 wedge_out_dir=$WEDGE_2025_DIR
1176 ;;
1177 legacy)
1178 wedge_out_dir=$USER_WEDGE_DIR
1179 ;;
1180 *)
1181 die "Invalid how $how"
1182 ;;
1183 esac
1184 fake-py3-libs-wedge "$wedge_out_dir"
1185
1186 echo " END $(timestamp)"
1187
1188 write-task-report
1189}
1190
1191install-wedges-OLD() {
1192 # the /wedge and ~/wedge stuff we got rid of
1193 install-wedges-parallel contrib legacy
1194}
1195
1196install-wedges-contrib-2025() {
1197 ### Install in ../oils.DEPS (aka unboxed)
1198
1199 # ../oils.DEPS/wedge
1200 install-wedges-parallel contrib unboxed
1201
1202 # ../oils.DEPS/bin
1203 deps/make-bin.sh contrib
1204}
1205
1206install-wedges() {
1207 ### What we tell users to run
1208
1209 # Migrated as of 2025-10
1210 install-wedges-contrib-2025 "$@"
1211}
1212
1213install-wedges-soil() {
1214 ### dev-setup-{debian,fedora,alpine} jobs call this
1215
1216 # Old wedges
1217 # install-wedges-parallel soil legacy
1218
1219 # ../oils.DEPS/wedge
1220 install-wedges-parallel soil unboxed
1221
1222 # ../oils.DEPS/bin
1223 deps/make-bin.sh contrib
1224}
1225
1226#
1227# Wedges built inside a container, for copying into a container
1228#
1229
1230boxed-clean() {
1231 # Source dir is _build/deps-source
1232 time sudo rm -r -f _build/boxed
1233}
1234
1235boxed-uftrace-OLD() {
1236 ### uftrace /wedge build: until we can move it to ../oils.DEPS/wedge
1237
1238 deps/wedge.sh boxed deps/source.medo/uftrace/ '' $ROOT_WEDGE_DIR debian-12
1239}
1240
1241maybe-boxed-wedge() {
1242 local name=$1
1243 local version=$2
1244 local wedge_base_dir=$3 # e.g. oils.DEPS or _build/boxed/wedge?
1245 local distro=$4 # e.g. debian-12 or empty
1246
1247 local task_file=$BOXED_LOG_DIR/$name-$version.task.tsv
1248 local log_file=$BOXED_LOG_DIR/$name-$version.log.txt
1249
1250 echo " TASK $(timestamp) $name $version > $log_file"
1251
1252 # python3 because it's OUTSIDE the container
1253 # Separate columns that could be joined: number of files, total size
1254 my-time-tsv --print-header \
1255 --field xargs_slot \
1256 --field wedge \
1257 --field wedge_HREF \
1258 --field version \
1259 --output $task_file
1260
1261 if boxed-wedge-exists "$name" "$version" "$distro"; then
1262 echo "CACHED $(timestamp) $name $version"
1263 return
1264 fi
1265
1266 #local -a cmd=( deps/wedge.sh boxed-2025 _build/deps-source/$name/ $version)
1267 local -a cmd=(
1268 deps/wedge.sh boxed-2025 deps/source.medo/$name/ "$version" "$wedge_base_dir" "$distro"
1269 )
1270
1271 set +o errexit
1272 my-time-tsv \
1273 --field "$XARGS_SLOT" \
1274 --field "$name" \
1275 --field "$name-$version.log.txt" \
1276 --field "$version" \
1277 --append \
1278 --output $task_file \
1279 -- \
1280 "${cmd[@]}" "$@" >$log_file 2>&1
1281 local status=$?
1282 set -o errexit
1283
1284 if test "$status" -eq 0; then
1285 echo " OK $(timestamp) $name $version"
1286 else
1287 echo " FAIL $(timestamp) $name $version"
1288 fi
1289}
1290
1291do-boxed-wedge-list() {
1292 ### Reads task rows from stdin
1293 local parallel=${1:-}
1294
1295 mkdir -p $BOXED_LOG_DIR
1296
1297 local -a flags
1298 if test -n "$parallel"; then
1299 log ""
1300 log "=== Building boxed wedges with $NPROC jobs in parallel"
1301 log ""
1302 flags=( -P $NPROC )
1303 else
1304 log ""
1305 log "=== Building boxed wedges serially"
1306 log ""
1307 fi
1308
1309 # Reads from stdin
1310 # Note: --process-slot-var requires GNU xargs! busybox args doesn't have it.
1311 #
1312 # $name $version $wedge_dir
1313 xargs "${flags[@]}" -n 4 --process-slot-var=XARGS_SLOT -- $0 maybe-boxed-wedge
1314}
1315
1316_boxed-wedges-2025() {
1317 local which_wedges=${1:-contrib} # contrib | soil | smoke
1318
1319 # deps/wedge.sh reads this
1320 export DOCKER=${2:-docker}
1321
1322 # For contributor setup: we need to use this BEFORE running build/py.sh all
1323 #build/py.sh time-helper
1324
1325 echo " START $(timestamp)"
1326
1327 # Do all of them in parallel
1328 print-wedge-list "$which_wedges" boxed | do-boxed-wedge-list T
1329
1330 # Dockerfile.* calls install-py3-libs from within the container, so we don't need this
1331 # It depends on MyPy
1332 #fake-py3-libs-wedge ../oils.DEPS/wedge
1333
1334 echo " END $(timestamp)"
1335
1336 write-task-report $BOXED_LOG_DIR
1337}
1338
1339boxed-wedges-2025() {
1340 time $0 _boxed-wedges-2025 "$@"
1341}
1342
1343libssl-bug() {
1344 set -x
1345 file=_build/wedge/binary/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/lib-dynload/_ssl.cpython-310-x86_64-linux-gnu.so
1346
1347 # TODO: figure out how to get this to find libssl.so.3, not .so.1.1
1348 ldd $file
1349 echo
1350 ls -l $file
1351}
1352
1353libssl-smoke() {
1354 deps/wedge.sh smoke-test deps/source.medo/python3/ '' '' debian-12
1355}
1356
1357smoke-unboxed-boxed() {
1358 install-wedges smoke legacy
1359 echo
1360 ls -l ~/wedge/oils-for-unix.org/pkg/dash
1361
1362 install-wedges smoke unboxed
1363 echo
1364 ls -l ../oils.DEPS/wedge/dash
1365
1366 boxed-wedges-2025 smoke
1367 echo
1368 ls -l _build/boxed/wedge/dash
1369
1370 # old boxed wedges: they never had xargs automation
1371 #boxed-wedges
1372
1373 # TODO: now invalidate cache, and build again
1374}
1375
1376task-five "$@"