OILS / build / deps.sh View on Github | oils.pub

1425 lines, 858 significant
1#!/usr/bin/env bash
2#
3# Script for contributors to build dev dependencies -- packaged as cross-distro
4# "wedges". Tested in the Soil CI.
5#
6# Usage:
7# build/deps.sh <function name>
8#
9# Examples:
10# build/deps.sh fetch
11# build/deps.sh install-wedges # contributor wedges, to build both Python and C++
12# build/deps.sh install-wedges-soil # more wedges for Soil CI container, on host
13# build/deps.sh boxed-wedges-2025 # build for copying inside OCI container
14#
15# More:
16# build/deps.sh print-wedge-list # print manifest
17#
18# Note: could be build/deps.sh unboxed-wedges versus boxed-wedges, and then
19# install-wedges as an ALIAS
20
21# Contributed sanity check, for getting started
22if ! test -d stdlib/osh; then
23 echo 'Run this script from the Oils repo root'
24 exit 1
25fi
26
27: ${LIB_OSH=stdlib/osh}
28source $LIB_OSH/bash-strict.sh
29source $LIB_OSH/task-five.sh
30
31REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
32
33source build/dev-shell.sh # We generally assume this, but let's be safe
34source deps/from-apt.sh # PY3_BUILD_DEPS
35source test/tsv-lib.sh # tsv-concat
36source web/table/html.sh # table-sort-{begin,end}
37
38# Also in build/dev-shell.sh
39USER_WEDGE_DIR=~/wedge/oils-for-unix.org/pkg
40ROOT_WEDGE_DIR=/wedge/oils-for-unix.org/pkg
41
42readonly WEDGE_2025_DIR=../oils.DEPS/wedge
43mkdir -p $WEDGE_2025_DIR
44
45readonly DEPS_SOURCE_DIR=_build/deps-source
46
47readonly RE2C_VERSION=4.3.1
48readonly RE2C_URL="https://github.com/skvadrik/re2c/releases/download/$RE2C_VERSION/re2c-$RE2C_VERSION.tar.xz"
49
50readonly CMARK_VERSION=0.29.0
51readonly CMARK_URL="https://github.com/commonmark/cmark/archive/$CMARK_VERSION.tar.gz"
52
53readonly PY_FTP_MIRROR="${PY_FTP_MIRROR:-https://www.python.org/ftp}"
54
55readonly PY2_VERSION=2.7.18
56readonly PY2_URL="$PY_FTP_MIRROR/python/$PY2_VERSION/Python-$PY2_VERSION.tar.xz"
57
58readonly PY3_VERSION=3.10.4
59readonly PY3_URL="$PY_FTP_MIRROR/python/$PY3_VERSION/Python-$PY3_VERSION.tar.xz"
60
61readonly BASH_VER=4.4 # don't clobber BASH_VERSION
62readonly BASH_URL="https://op.oils.pub/blob/spec-bin/bash-$BASH_VER.tar.gz"
63
64# Another version of bash to test
65readonly BASH5_VER=5.2.21
66readonly BASH5_URL="https://op.oils.pub/blob/spec-bin/bash-$BASH5_VER.tar.gz"
67
68readonly DASH_VERSION=0.5.10.2
69readonly DASH_URL="https://op.oils.pub/blob/spec-bin/dash-$DASH_VERSION.tar.gz"
70
71readonly ZSH_OLD_VER=5.1.1
72readonly ZSH_OLD_URL="https://op.oils.pub/blob/spec-bin/zsh-$ZSH_OLD_VER.tar.xz"
73
74readonly ZSH_NEW_VER=5.9
75readonly ZSH_NEW_URL="https://op.oils.pub/blob/spec-bin/zsh-$ZSH_NEW_VER.tar.xz"
76
77readonly MKSH_VERSION=R52c
78readonly MKSH_URL="https://op.oils.pub/blob/spec-bin/mksh-$MKSH_VERSION.tgz"
79
80readonly BUSYBOX_VERSION='1.35.0'
81readonly BUSYBOX_URL="https://op.oils.pub/blob/spec-bin/busybox-$BUSYBOX_VERSION.tar.bz2"
82
83readonly YASH_VERSION=2.49
84readonly YASH_URL="https://op.oils.pub/blob/spec-bin/yash-$YASH_VERSION.tar.xz"
85
86readonly MYPY_GIT_URL=https://github.com/python/mypy
87readonly MYPY_VERSION=0.780
88
89readonly PY3_LIBS=~/wedge/oils-for-unix.org/pkg/py3-libs/$MYPY_VERSION
90
91# Version 2.4.0 from 2021-10-06 was the last version that supported Python 2
92# https://github.com/PyCQA/pyflakes/blob/main/NEWS.rst
93readonly PYFLAKES_VERSION=2.4.0
94#readonly PYFLAKES_URL='https://files.pythonhosted.org/packages/15/60/c577e54518086e98470e9088278247f4af1d39cb43bcbd731e2c307acd6a/pyflakes-2.4.0.tar.gz'
95# 2023-07: Mirrored to avoid network problem on broome during release
96readonly PYFLAKES_URL='https://op.oils.pub/blob/pyflakes-2.4.0.tar.gz'
97
98readonly BLOATY_VERSION=1.1
99readonly BLOATY_URL='https://github.com/google/bloaty/releases/download/v1.1/bloaty-1.1.tar.bz2'
100
101readonly UFTRACE_VERSION=0.13
102readonly UFTRACE_URL='https://github.com/namhyung/uftrace/archive/refs/tags/v0.13.tar.gz'
103
104readonly SOUFFLE_VERSION=2.4.1
105readonly SOUFFLE_URL=https://github.com/souffle-lang/souffle/archive/refs/tags/2.4.1.tar.gz
106
107readonly R_LIBS_VERSION='2023-04-18'
108readonly TIME_HELPER_VERSION='2023-02-28'
109readonly PY3_LIBS_VERSION=2023-03-04
110
111readonly WEDGE_LOG_DIR=_build/wedge/logs
112
113readonly BOXED_WEDGE_DIR=_build/boxed/wedge
114readonly BOXED_LOG_DIR=_build/boxed/logs
115
116log() {
117 echo "$@" >& 2
118}
119
120die() {
121 log "$0: fatal: $@"
122 exit 1
123}
124
125rm-oils-crap() {
126 ### Remove OLD wedges (prior to 2025)
127
128 rm -r -f -v ~/wedge
129 sudo rm -r -f -v /wedge
130}
131
132# Note: git is an implicit dependency -- that's how we got the repo in the
133# first place!
134
135# python2-dev is no longer available on Debian 12
136# python-dev also seems gone
137#
138# wget: for fetching wedges (not on Debian by default!)
139# tree: tiny package that's useful for showing what we installed
140# g++: essential
141# libreadline-dev: needed for the build/prepare.sh Python build.
142# gawk: used by spec-runner.sh for the special match() function.
143# cmake: for cmark
144# PY3_BUILD_DEPS - I think these will be used for building the Python 2 wedge
145# as well
146readonly -a WEDGE_DEPS_DEBIAN=(
147 bzip2
148 wget
149 tree
150 gawk
151 g++
152 ninja-build
153 cmake
154 libreadline-dev
155 systemtap-sdt-dev
156 xz-utils
157
158 # for Souffle, flex and bison
159 #flex bison
160
161 "${PY3_BUILD_DEPS[@]}"
162 "${SPEC_TEST_DEPS[@]}"
163)
164
165readonly -a WEDGE_DEPS_ALPINE=(
166 coreutils findutils
167
168 bzip2
169 xz
170
171 wget tree gawk
172
173 gcc g++
174 ninja-build
175 # https://pkgs.alpinelinux.org/packages?name=ninja-is-really-ninja&branch=v3.19&repo=&arch=&maintainer=
176 ninja-is-really-ninja
177 cmake
178
179 readline-dev
180 zlib-dev
181 libffi-dev
182 openssl-dev
183
184 ncurses-dev
185
186 # for Souffle, flex and bison
187 #flex bison
188)
189
190readonly -a WEDGE_DEPS_FEDORA=(
191
192 # Weird, Fedora doesn't have these by default!
193 hostname
194 tar
195 bzip2
196
197 # https://packages.fedoraproject.org/pkgs/wget/wget/
198 wget
199 # https://packages.fedoraproject.org/pkgs/tree-pkg/tree/
200 tree
201 gawk
202
203 # https://packages.fedoraproject.org/pkgs/gcc/gcc/
204 gcc gcc-c++
205
206 ninja-build
207 cmake
208
209 readline-devel
210
211 # Like PY3_BUILD_DEPS
212 # https://packages.fedoraproject.org/pkgs/zlib/zlib-devel/
213 zlib-devel
214 # https://packages.fedoraproject.org/pkgs/libffi/libffi-devel/
215 libffi-devel
216 # https://packages.fedoraproject.org/pkgs/openssl/openssl-devel/
217 openssl-devel
218
219 # For building zsh from source?
220 # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
221 ncurses-devel
222 #libcap-devel
223
224 # still have a job control error compiling bash
225 # https://packages.fedoraproject.org/pkgs/glibc/glibc-devel/
226 # glibc-devel
227
228 libasan
229)
230
231readonly -a WEDGE_DEPS_ARCH=(
232 # https://archlinux.org/packages/core/x86_64/bzip2/
233 bzip2
234
235 # https://archlinux.org/packages/extra/x86_64/wget/
236 wget
237
238 # https://archlinux.org/packages/extra/x86_64/tree/
239 tree
240
241 # https://archlinux.org/packages/core/x86_64/gawk/
242 gawk
243
244 # https://archlinux.org/packages/core/x86_64/gcc/
245 gcc
246
247 # https://archlinux.org/packages/community/x86_64/ninja/
248 ninja
249
250 # https://archlinux.org/packages/extra/x86_64/cmake/
251 cmake
252
253 # https://archlinux.org/packages/core/x86_64/readline/
254 readline
255
256 # https://archlinux.org/packages/core/x86_64/zlib/
257 zlib
258
259 # https://archlinux.org/packages/core/x86_64/libffi/
260 libffi
261
262 # https://archlinux.org/packages/core/x86_64/openssl/
263 openssl
264
265 # https://archlinux.org/packages/core/x86_64/ncurses/
266 ncurses
267
268 # Development headers are included in the main packages on Arch,
269 # unlike other distros that separate them into -dev/-devel packages
270
271 # Python 2 from the AUR
272 # https://aur.archlinux.org/packages/python2
273 base-devel # needed for building packages from the AUR
274
275)
276
277install-debian-packages() {
278 ### Packages for build/py.sh all, building wedges, etc.
279
280 set -x # show what needs sudo
281
282 # pass -y for say gitpod
283 sudo apt "$@" install "${WEDGE_DEPS_DEBIAN[@]}"
284 set +x
285
286 # maybe pass -y through
287 test/spec-bin.sh install-shells-with-apt "$@"
288}
289
290install-ubuntu-packages() {
291 ### Debian and Ubuntu packages are the same; this function is suggested on the wiki
292 install-debian-packages "$@"
293}
294
295wedge-deps-debian() {
296 # Install packages without prompt
297
298 # 2024-02 - there was an Ubuntu update, and we started needing this
299 sudo apt-get -y update
300
301 install-debian-packages -y
302}
303
304wedge-deps-fedora() {
305 # https://linuxconfig.org/install-development-tools-on-redhat-8
306 # Trying to get past compile errors
307 # sudo dnf group install --assumeyes 'Development Tools'
308
309 sudo dnf install --assumeyes "${WEDGE_DEPS_FEDORA[@]}"
310}
311
312wedge-deps-alpine() {
313 # https://linuxconfig.org/install-development-tools-on-redhat-8
314 # Trying to get past compile errors
315 # sudo dnf group install --assumeyes 'Development Tools'
316
317 sudo apk add "${WEDGE_DEPS_ALPINE[@]}"
318}
319
320wedge-deps-arch() {
321 # Install packages without prompt
322
323 # First sync the package database
324 sudo pacman -Sy
325
326 # Then install packages
327 for pkg in "${WEDGE_DEPS_ARCH[@]}"; do
328 # Only install if not already installed
329 if ! pacman -Qi "$pkg" >/dev/null 2>&1; then
330 sudo pacman --noconfirm -S "$pkg"
331 fi
332 done
333}
334
335#
336# Fetch
337#
338
339download-to() {
340 local dir=$1
341 local url=$2
342 wget --no-clobber --directory-prefix "$dir" "$url"
343}
344
345maybe-extract() {
346 local wedge_dir=$1
347 local tar_name=$2
348 local out_dir=$3
349
350 if test -d "$wedge_dir/$out_dir"; then
351 log "Not extracting because $wedge_dir/$out_dir exists"
352 return
353 fi
354
355 local tar=$wedge_dir/$tar_name
356 case $tar_name in
357 *.gz|*.tgz) # mksh ends with .tgz
358 flag='--gzip'
359 ;;
360 *.bz2)
361 flag='--bzip2'
362 ;;
363 *.xz)
364 flag='--xz'
365 ;;
366 *)
367 die "tar with unknown extension: $tar_name"
368 ;;
369 esac
370
371 tar --extract $flag --file $tar --directory $wedge_dir
372}
373
374clone-mypy() {
375 ### replaces deps/from-git
376 local dest_dir=$1
377 local version=${2:-$MYPY_VERSION}
378
379 local dest=$dest_dir/mypy-$version
380 if test -d $dest; then
381 log "Not cloning because $dest exists"
382 return
383 fi
384
385 # v$VERSION is a tag, not a branch
386
387 # size optimization: --depth=1 --shallow-submodules
388 # https://git-scm.com/docs/git-clone
389
390 git clone --recursive --branch v$version \
391 --depth=1 --shallow-submodules \
392 $MYPY_GIT_URL $dest
393
394 # TODO: verify commit checksum
395}
396
397copy-source-medo() {
398 mkdir -p $DEPS_SOURCE_DIR
399
400 # Copy the whole tree, including the .treeptr files
401 cp --verbose --recursive --no-target-directory \
402 deps/source.medo/ $DEPS_SOURCE_DIR/
403}
404
405fetch-spec-bin() {
406 download-to $DEPS_SOURCE_DIR/bash "$BASH_URL"
407 maybe-extract $DEPS_SOURCE_DIR/bash "$(basename $BASH_URL)" bash-$BASH_VER
408
409 download-to $DEPS_SOURCE_DIR/bash "$BASH5_URL"
410 maybe-extract $DEPS_SOURCE_DIR/bash "$(basename $BASH5_URL)" bash-$BASH5_VER
411
412 download-to $DEPS_SOURCE_DIR/dash "$DASH_URL"
413 maybe-extract $DEPS_SOURCE_DIR/dash "$(basename $DASH_URL)" dash-$DASH_VERSION
414
415 download-to $DEPS_SOURCE_DIR/zsh "$ZSH_OLD_URL"
416 maybe-extract $DEPS_SOURCE_DIR/zsh "$(basename $ZSH_OLD_URL)" zsh-$ZSH_OLD_VER
417
418 download-to $DEPS_SOURCE_DIR/zsh "$ZSH_NEW_URL"
419 maybe-extract $DEPS_SOURCE_DIR/zsh "$(basename $ZSH_NEW_URL)" zsh-$ZSH_NEW_VER
420
421 download-to $DEPS_SOURCE_DIR/mksh "$MKSH_URL"
422 maybe-extract $DEPS_SOURCE_DIR/mksh "$(basename $MKSH_URL)" mksh-$MKSH_VERSION
423
424 download-to $DEPS_SOURCE_DIR/busybox "$BUSYBOX_URL"
425 maybe-extract $DEPS_SOURCE_DIR/busybox "$(basename $BUSYBOX_URL)" busybox-$BUSYBOX_VERSION
426
427 download-to $DEPS_SOURCE_DIR/yash "$YASH_URL"
428 maybe-extract $DEPS_SOURCE_DIR/yash "$(basename $YASH_URL)" yash-$YASH_VERSION
429
430 # Patch: this tarball doesn't follow the convention $name-$version
431 if test -d $DEPS_SOURCE_DIR/mksh/mksh; then
432 pushd $DEPS_SOURCE_DIR/mksh
433 mv -v mksh mksh-$MKSH_VERSION
434 popd
435 fi
436}
437
438fetch() {
439 local py_only=${1:-}
440
441 # For now, simulate what 'medo expand deps/source.medo _build/deps-source'
442 # would do: fetch compressed tarballs designated by .treeptr files, and
443 # expand them.
444
445 # _build/deps-source/
446 # re2c/
447 # WEDGE
448 # re2c-3.0/ # expanded .tar.xz file
449
450 copy-source-medo
451
452 # Hack
453 local dest=$DEPS_SOURCE_DIR/time-helper/time-helper-$TIME_HELPER_VERSION
454 mkdir -p $dest
455 cp -v benchmarks/time-helper.c $dest
456
457 download-to $DEPS_SOURCE_DIR/re2c "$RE2C_URL"
458 download-to $DEPS_SOURCE_DIR/cmark "$CMARK_URL"
459 maybe-extract $DEPS_SOURCE_DIR/re2c "$(basename $RE2C_URL)" re2c-$RE2C_VERSION
460 maybe-extract $DEPS_SOURCE_DIR/cmark "$(basename $CMARK_URL)" cmark-$CMARK_VERSION
461
462 if test -n "$py_only"; then
463 log "Fetched dependencies for 'build/py.sh'"
464 return
465 fi
466
467 download-to $DEPS_SOURCE_DIR/pyflakes "$PYFLAKES_URL"
468 maybe-extract $DEPS_SOURCE_DIR/pyflakes "$(basename $PYFLAKES_URL)" \
469 pyflakes-$PYFLAKES_VERSION
470
471 download-to $DEPS_SOURCE_DIR/python2 "$PY2_URL"
472 download-to $DEPS_SOURCE_DIR/python3 "$PY3_URL"
473 maybe-extract $DEPS_SOURCE_DIR/python2 "$(basename $PY2_URL)" Python-$PY2_VERSION
474 maybe-extract $DEPS_SOURCE_DIR/python3 "$(basename $PY3_URL)" Python-$PY3_VERSION
475
476 fetch-spec-bin
477
478 # bloaty and uftrace are for benchmarks, in containers
479 download-to $DEPS_SOURCE_DIR/bloaty "$BLOATY_URL"
480 download-to $DEPS_SOURCE_DIR/uftrace "$UFTRACE_URL"
481 maybe-extract $DEPS_SOURCE_DIR/bloaty "$(basename $BLOATY_URL)" bloaty-$BLOATY_VERSION
482 maybe-extract $DEPS_SOURCE_DIR/uftrace "$(basename $UFTRACE_URL)" uftrace-$UFTRACE_VERSION
483
484 # This is in $DEPS_SOURCE_DIR to COPY into containers, which mycpp will directly import.
485
486 # It's also copied into a wedge in install-wedges.
487 clone-mypy $DEPS_SOURCE_DIR/mypy
488
489 if false; then
490 download-to $DEPS_SOURCE_DIR/souffle "$SOUFFLE_URL"
491 maybe-extract $DEPS_SOURCE_DIR/souffle "$(basename $SOUFFLE_URL)" souffle-$SOUFFLE_VERSION
492 fi
493
494 if command -v tree > /dev/null; then
495 tree -L 2 $DEPS_SOURCE_DIR
496 fi
497}
498
499fetch-py() {
500 fetch py_only
501}
502
503mirror-pyflakes() {
504 ### Workaround for network error during release
505 scp \
506 $DEPS_SOURCE_DIR/pyflakes/"$(basename $PYFLAKES_URL)" \
507 op.oils.pub:op.oils.pub/blob/
508}
509
510mirror-python() {
511 ### Can't reach python.org from some machines
512 scp \
513 $DEPS_SOURCE_DIR/python2/"$(basename $PY2_URL)" \
514 op.oils.pub:op.oils.pub/blob/
515
516 scp \
517 $DEPS_SOURCE_DIR/python3/"$(basename $PY3_URL)" \
518 op.oils.pub:op.oils.pub/blob/
519}
520
521mirror-zsh() {
522 scp \
523 _tmp/zsh-5.9.tar.xz \
524 op.oils.pub:op.oils.pub/blob/
525}
526
527wedge-exists() {
528 ### Does an installed wedge already exist?
529 local name=$1
530 local version=$2
531 local wedge_base_dir=${3:-/wedge/oils-for-unix.org} # e.g. ../oils.DEPS/wedge
532
533 local installed=$wedge_base_dir/$name/$version
534
535 if test -d $installed; then
536 log "$installed already exists"
537 return 0
538 else
539 return 1
540 fi
541}
542
543boxed-wedge-exists() {
544 ### Does an installed wedge already exist?
545
546 local name=$1
547 local version=$2
548
549 # NOT USED for now
550 local distro=${3:-debian-12}
551
552 local installed=$BOXED_WEDGE_DIR/$name/$version
553
554 if test -d $installed; then
555 log "$installed already exists"
556 return 0
557 else
558 return 1
559 fi
560}
561
562#
563# Special case of py3-libs
564#
565# TODO: it should a WEDGE
566# It needs a BUILD DEPENDENCY on:
567# - the python3 wedge, so you can do python3 -m pip install.
568# - the mypy repo, which has test-requirements.txt
569#
570
571upgrade-typed-ast() {
572 local file=$1
573 sed -i 's/typed_ast.*/typed_ast==1.5.0/' $file
574}
575
576test-typed-ast() {
577 local dir=~/wedge/oils-for-unix.org/pkg/mypy/0.780
578
579 cp -v $dir/mypy-requirements.txt _tmp
580
581 local file=_tmp/mypy-requirements.txt
582 cat $file
583 #echo
584
585 # 1.5.0 fixed this bug
586 # https://github.com/python/typed_ast/issues/169
587
588 upgrade-typed-ast $file
589 echo
590 cat $file
591}
592
593download-py3-libs() {
594 ### Download source/binary packages, AFTER python3 is installed
595
596 # Note that this is NOT source code; there is binary code, e.g. in
597 # lxml-*.whl
598
599 local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
600 local py_package_dir=_cache/py3-libs
601 mkdir -p $py_package_dir
602
603 python3 -m pip download -d $py_package_dir -r $mypy_dir/test-requirements.txt
604 python3 -m pip download -d $py_package_dir pexpect pyte
605}
606
607install-py3-libs-in-venv() {
608 local venv_dir=$1
609 local mypy_dir=$2 # This is a param for host build vs. container build
610 local package_dir=_cache/py3-libs
611
612 source $venv_dir/bin/activate # enter virtualenv
613
614 # 2023-07 note: we're installing yapf in a DIFFERENT venv, because it
615 # conflicts with MyPy deps!
616 # "ERROR: pip's dependency resolver does not currently take into account all
617 # the packages that are installed."
618
619 # --find-links uses a "cache dir" for packages (weird flag name!)
620
621 # Avoids a warning, but doesn't fix typed_ast
622 #time python3 -m pip install --find-links $package_dir wheel
623
624 upgrade-typed-ast $mypy_dir/mypy-requirements.txt
625
626 # for mycpp/
627 time python3 -m pip install --find-links $package_dir -r $mypy_dir/test-requirements.txt
628
629 # pexpect: for spec/stateful/*.py
630 time python3 -m pip install --find-links $package_dir pexpect pyte
631}
632
633install-py3-libs-from-cache() {
634 # As well as end users
635 local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
636 local wedge_out_dir=${2:-$USER_WEDGE_DIR}
637
638 log "Ensuring pip is installed (interpreter $(command -v python3)"
639 python3 -m ensurepip
640
641 local venv_dir=$wedge_out_dir/py3-libs/$PY3_LIBS_VERSION
642 log "Creating venv in $venv_dir"
643
644 # Note: the bin/python3 in this venv is a symlink to python3 in $PATH, i.e.
645 # the /wedge we just built
646 python3 -m venv $venv_dir
647
648 log "Installing MyPy deps in venv"
649
650 # Run in a subshell because it mutates shell state
651 $0 install-py3-libs-in-venv $venv_dir $mypy_dir
652}
653
654install-py3-libs() {
655 ### Invoked by Dockerfile.cpp-small, etc. and by fake-py3-libs-wedge
656 local mypy_dir=${1:-}
657 local wedge_out_dir=${2:-}
658
659 local py3
660 py3=$(command -v python3)
661 case $py3 in
662 *wedge/oils-for-unix.org/*)
663 ;;
664 *oils.DEPS/*)
665 ;;
666 *)
667 die "python3 is '$py3', but expected it to be in a wedge"
668 ;;
669 esac
670
671 log "python3 is $py3"
672
673 download-py3-libs $mypy_dir
674 install-py3-libs-from-cache "$mypy_dir" "$wedge_out_dir"
675}
676
677#
678# Wedge manifests
679#
680
681py-wedges() {
682 ### for build/py.sh all
683 local how=${1:-legacy}
684
685 if test $how = 'boxed'; then
686 local where='debian-12'
687 else
688 local where='HOST'
689 fi
690 case $how in
691 boxed|unboxed)
692 echo time-helper $TIME_HELPER_VERSION $WEDGE_2025_DIR $where
693 echo cmark $CMARK_VERSION $WEDGE_2025_DIR $where
694 echo re2c $RE2C_VERSION $WEDGE_2025_DIR $where
695 echo python2 $PY2_VERSION $WEDGE_2025_DIR $where
696 echo pyflakes $PYFLAKES_VERSION $WEDGE_2025_DIR $where
697 ;;
698 legacy)
699 echo time-helper $TIME_HELPER_VERSION $ROOT_WEDGE_DIR $where
700 echo cmark $CMARK_VERSION $ROOT_WEDGE_DIR $where
701 echo re2c $RE2C_VERSION $ROOT_WEDGE_DIR $where
702 echo python2 $PY2_VERSION $ROOT_WEDGE_DIR $where
703 echo pyflakes $PYFLAKES_VERSION $USER_WEDGE_DIR $where
704 ;;
705 *)
706 die "Invalid how $how"
707 esac
708}
709
710cpp-wedges() {
711 ### for ninja / mycpp translation
712 local how=${1:-legacy}
713
714 if test $how = 'boxed'; then
715 local where='debian-12'
716 else
717 local where='HOST'
718 fi
719 case $how in
720 boxed|unboxed)
721 echo python3 $PY3_VERSION $WEDGE_2025_DIR $where
722 echo mypy $MYPY_VERSION $WEDGE_2025_DIR $where
723 ;;
724 legacy)
725 echo python3 $PY3_VERSION $ROOT_WEDGE_DIR $where
726 echo mypy $MYPY_VERSION $USER_WEDGE_DIR $where
727 ;;
728 *)
729 die "Invalid how $how"
730 esac
731
732 # py3-libs has a built time dep on both python3 and MyPy, so we're doing it
733 # separately for now
734 #echo py3-libs $PY3_LIBS_VERSION $USER_WEDGE_DIR
735}
736
737spec-bin-wedges() {
738 ### for test/spec-py.sh osh-all
739 local how=${1:-legacy}
740
741 if test $how = 'boxed'; then
742 local where='debian-12'
743 else
744 local where='HOST'
745 fi
746 case $how in
747 boxed|unboxed)
748 echo dash $DASH_VERSION $WEDGE_2025_DIR $where
749 echo bash $BASH_VER $WEDGE_2025_DIR $where
750 echo bash $BASH5_VER $WEDGE_2025_DIR $where
751 echo mksh $MKSH_VERSION $WEDGE_2025_DIR $where
752 echo zsh $ZSH_OLD_VER $WEDGE_2025_DIR $where
753 echo zsh $ZSH_NEW_VER $WEDGE_2025_DIR $where
754 echo busybox $BUSYBOX_VERSION $WEDGE_2025_DIR $where
755 echo yash $YASH_VERSION $WEDGE_2025_DIR $where
756 ;;
757 legacy)
758 echo dash $DASH_VERSION $USER_WEDGE_DIR $where
759 echo bash $BASH_VER $USER_WEDGE_DIR $where
760 echo bash $BASH5_VER $USER_WEDGE_DIR $where
761 echo mksh $MKSH_VERSION $USER_WEDGE_DIR $where
762 echo zsh $ZSH_OLD_VER $USER_WEDGE_DIR $where
763 echo zsh $ZSH_NEW_VER $USER_WEDGE_DIR $where
764 echo busybox $BUSYBOX_VERSION $USER_WEDGE_DIR $where
765 echo yash $YASH_VERSION $USER_WEDGE_DIR $where
766 ;;
767 *)
768 die "Invalid how $how"
769 esac
770}
771
772zsh-wedges() {
773 local how=${1:-legacy}
774
775 if test $how = 'boxed'; then
776 local where='debian-12'
777 else
778 local where='HOST'
779 fi
780 case $how in
781 boxed|unboxed)
782 echo zsh $ZSH_OLD_VER $WEDGE_2025_DIR $where
783 echo zsh $ZSH_NEW_VER $WEDGE_2025_DIR $where
784 ;;
785 legacy)
786 echo zsh $ZSH_OLD_VER $USER_WEDGE_DIR $where
787 echo zsh $ZSH_NEW_VER $USER_WEDGE_DIR $where
788 ;;
789 *)
790 die "Invalid how $how"
791 esac
792}
793
794smoke-wedges() {
795 local how=${1:-legacy}
796
797 if test $how = 'boxed'; then
798 local where='debian-12'
799 else
800 local where='HOST'
801 fi
802 case $how in
803 boxed|unboxed)
804 echo dash $DASH_VERSION $WEDGE_2025_DIR $where
805 ;;
806 legacy)
807 echo dash $DASH_VERSION $USER_WEDGE_DIR $where
808 ;;
809 *)
810 die "Invalid how $how"
811 esac
812}
813
814cmark-wedges() {
815 local how=${1:-legacy}
816
817 if test $how = 'boxed'; then
818 local where='debian-12'
819 else
820 local where='HOST'
821 fi
822 case $how in
823 boxed|unboxed)
824 echo cmark $CMARK_VERSION $WEDGE_2025_DIR $where
825 ;;
826 legacy)
827 echo cmark $CMARK_VERSION $ROOT_WEDGE_DIR $where
828 ;;
829 *)
830 die "Invalid how $how"
831 esac
832}
833
834extra-wedges() {
835 # Contributors don't need uftrace, bloaty, and probably R-libs
836 # Although R-libs could be useful for benchmarks
837
838 local how=${1:-legacy}
839
840 case $how in
841 boxed)
842 echo R-libs $R_LIBS_VERSION $WEDGE_2025_DIR debian-12
843 echo uftrace $UFTRACE_VERSION $WEDGE_2025_DIR debian-12
844 echo bloaty $BLOATY_VERSION $WEDGE_2025_DIR debian-10 # It works on Debian 10, not 12
845 ;;
846 unboxed)
847 echo R-libs $R_LIBS_VERSION $WEDGE_2025_DIR HOST
848 echo uftrace $UFTRACE_VERSION $WEDGE_2025_DIR HOST
849 echo bloaty $BLOATY_VERSION $WEDGE_2025_DIR HOST
850 ;;
851 legacy)
852 echo R-libs $R_LIBS_VERSION $USER_WEDGE_DIR HOST
853 echo uftrace $UFTRACE_VERSION $ROOT_WEDGE_DIR HOST
854 echo bloaty $BLOATY_VERSION $ROOT_WEDGE_DIR HOST
855 ;;
856 *)
857 die "Invalid how $how"
858 esac
859}
860
861contributor-wedges() {
862 local how=${1:-}
863
864 py-wedges "$how"
865 cpp-wedges "$how"
866 spec-bin-wedges "$how"
867}
868
869#
870# More
871#
872
873timestamp() {
874 date '+%H:%M:%S'
875}
876
877my-time-tsv() {
878 python3 benchmarks/time_.py \
879 --tsv \
880 --time-span --rusage \
881 "$@"
882}
883
884maybe-install-wedge() {
885 local how=${1:-legacy}
886 local name=$2
887 local version=$3
888 local wedge_base_dir=$4
889 local where=$5
890
891 if test $where != HOST; then
892 die 'Expected $where to be "HOST"'
893 fi
894
895 local task_file=$WEDGE_LOG_DIR/$name-$version.task.tsv
896 local log_file=$WEDGE_LOG_DIR/$name-$version.log.txt
897
898 echo " TASK $(timestamp) $name $version > $log_file"
899
900 # python3 because it's OUTSIDE the container
901 # Separate columns that could be joined: number of files, total size
902 my-time-tsv --print-header \
903 --field xargs_slot \
904 --field wedge \
905 --field wedge_HREF \
906 --field version \
907 --output $task_file
908
909 if wedge-exists "$name" "$version" "$wedge_base_dir"; then
910 echo "CACHED $(timestamp) $name $version"
911 return
912 fi
913
914 case $how in
915 unboxed)
916 # TODO: I think all the builds should set the install dir, which is the LOCAL output
917 # - wedge unboxed
918 # - wedge unboxed-2025
919 # - wedge boxed-2025
920 # instead of relying on this flag
921 #
922 # So we have (NAME, VERSION, INSTALL_OUT) as our params. That makes sense
923 export WEDGE_2025=1
924 ;;
925 esac
926
927 local -a cmd=( deps/wedge.sh unboxed _build/deps-source/$name/ $version $wedge_base_dir)
928
929 set +o errexit
930 my-time-tsv \
931 --field "$XARGS_SLOT" \
932 --field "$name" \
933 --field "$name-$version.log.txt" \
934 --field "$version" \
935 --append \
936 --output $task_file \
937 -- \
938 "${cmd[@]}" "$@" >$log_file 2>&1
939 local status=$?
940 set -o errexit
941
942 if test "$status" -eq 0; then
943 echo " OK $(timestamp) $name $version"
944 else
945 echo " FAIL $(timestamp) $name $version"
946 fi
947}
948
949dummy-task() {
950 ### For testing log capture
951 local name=$1
952 local version=$2
953
954 echo "Building $name $version"
955
956 # random float between 0 and 3
957 # weirdly we need a seed from bash
958 # https://stackoverflow.com/questions/4048378/random-numbers-generation-with-awk-in-bash-shell
959 local secs
960 secs=$(awk -v seed=$RANDOM 'END { srand(seed); print rand() * 3 }' < /dev/null)
961
962 echo "sleep $secs"
963 sleep $secs
964
965 echo 'stdout'
966 log 'stderr'
967
968 if test $name = 'mksh'; then
969 echo "simulate failure for $name"
970 exit 2
971 fi
972}
973
974dummy-task-wrapper() {
975 # Similar to test/common.sh run-task-with-status, used by
976 # test/{spec,wild}-runner.sh
977 local name=$1
978 local version=$2
979
980 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
981 local log_file=$WEDGE_LOG_DIR/$name.log.txt
982
983 echo " TASK $(timestamp) $name $version > $log_file"
984
985 # python3 because it's OUTSIDE the container
986 # Separate columns that could be joined: number of files, total size
987 my-time-tsv --print-header \
988 --field xargs_slot \
989 --field wedge \
990 --field wedge_HREF \
991 --field version \
992 --output $task_file
993
994 my-time-tsv \
995 --field "$XARGS_SLOT" \
996 --field "$name" \
997 --field "$name.log.txt" \
998 --field "$version" \
999 --append \
1000 --output $task_file \
1001 $0 dummy-task "$@" >$log_file 2>&1 || true
1002
1003 echo " DONE $(timestamp) $name $version"
1004}
1005
1006html-head() {
1007 # python3 because we're outside containers
1008 PYTHONPATH=. python3 doctools/html_head.py "$@"
1009}
1010
1011index-html() {
1012 local tasks_tsv=$1
1013
1014 local base_url='../../../web'
1015 html-head --title 'Wedge Builds' \
1016 "$base_url/ajax.js" \
1017 "$base_url/table/table-sort.js" \
1018 "$base_url/table/table-sort.css" \
1019 "$base_url/base.css"
1020
1021 table-sort-begin 'width60'
1022
1023 cat <<EOF
1024 <p id="home-link">
1025 <a href="/">oils.pub</a>
1026 </p>
1027
1028 <h1>Wedge Builds</h1>
1029EOF
1030
1031 tsv2html3 $tasks_tsv
1032
1033 cat <<EOF
1034 <p>
1035 <a href="tasks.tsv">tasks.tsv</a>
1036 </p>
1037EOF
1038
1039 table-sort-end 'tasks' # ID for sorting
1040}
1041
1042NPROC=$(nproc)
1043#NPROC=1
1044
1045install-wedge-list() {
1046 ### Reads task rows from stdin
1047 local how=${1:-legacy} # unboxed | legacy
1048 local parallel=${2:-}
1049
1050 mkdir -p $WEDGE_LOG_DIR
1051
1052 local -a flags
1053 if test -n "$parallel"; then
1054 log ""
1055 log "=== Installing wedges with $NPROC jobs in parallel"
1056 log ""
1057 flags=( -P $NPROC )
1058 else
1059 log ""
1060 log "=== Installing wedges serially"
1061 log ""
1062 fi
1063
1064 # Reads from stdin
1065 # Note: --process-slot-var requires GNU xargs! busybox args doesn't have it.
1066 #
1067 # $name $version $wedge_dir
1068 xargs "${flags[@]}" -n 4 --process-slot-var=XARGS_SLOT -- $0 maybe-install-wedge "$how"
1069}
1070
1071write-task-report() {
1072 local log_dir=${1:-$WEDGE_LOG_DIR}
1073
1074 local tasks_tsv=$log_dir/tasks.tsv
1075
1076 python3 devtools/tsv_concat.py $log_dir/*.task.tsv > $tasks_tsv
1077 log "Wrote $tasks_tsv"
1078
1079 # TODO: version can be right-justified?
1080 here-schema-tsv-4col >$log_dir/tasks.schema.tsv <<EOF
1081column_name type precision strftime
1082status integer 0 -
1083elapsed_secs float 1 -
1084start_time float 1 %H:%M:%S
1085end_time float 1 %H:%M:%S
1086user_secs float 1 -
1087sys_secs float 1 -
1088max_rss_KiB integer 0 -
1089xargs_slot integer 0 -
1090wedge string 0 -
1091wedge_HREF string 0 -
1092version string 0 -
1093EOF
1094
1095 index-html $tasks_tsv > $log_dir/index.html
1096 log "Wrote $log_dir/index.html"
1097}
1098
1099check-for-failure() {
1100 local tsv=${1:-$WEDGE_LOG_DIR/tasks.tsv}
1101 local allowed_failures=${2:-0}
1102
1103 awk -F $'\t' -v allowed_failures="$allowed_failures" '
1104 NR == 1 {
1105 col_name = $1
1106 if (col_name != "status") {
1107 printf("%s: Expected \"status\" as first column\n", FILENAME)
1108 exit 2
1109 }
1110 }
1111 NR != 1 {
1112 status = $1
1113 if (status != 0) {
1114 printf("%s row %d: got failure %d\n", FILENAME, NR, status)
1115 failures += 1
1116 }
1117 num_rows += 1
1118 }
1119 END {
1120 if (failures == allowed_failures) {
1121 printf("%s: %d tasks succeeded (%d allowed failures)\n", FILENAME, num_rows, allowed_failures)
1122 } else {
1123 printf("%s: Expected %d failures, but got %d\n", FILENAME, allowed_failures, failures)
1124 exit 1
1125 }
1126 }
1127 ' $tsv >& 2
1128}
1129
1130fake-py3-libs-wedge() {
1131 local wedge_out_dir=${1:-}
1132
1133 local name=py3-libs
1134 local version=$PY3_LIBS_VERSION
1135
1136 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
1137 local log_file=$WEDGE_LOG_DIR/$name.log.txt
1138
1139 my-time-tsv --print-header \
1140 --field xargs_slot \
1141 --field wedge \
1142 --field wedge_HREF \
1143 --field version \
1144 --output $task_file
1145
1146 # There is no xargs slot!
1147 my-time-tsv \
1148 --field "-1" \
1149 --field "$name" \
1150 --field "$name.log.txt" \
1151 --field "$version" \
1152 --append \
1153 --output $task_file \
1154 -- \
1155 $0 install-py3-libs '' "$wedge_out_dir" >$log_file 2>&1 || true
1156
1157 echo " FAKE $(timestamp) $name $version"
1158}
1159
1160print-wedge-list() {
1161 local which_wedges=${1:-contrib} # contrib | soil | smoke
1162 local how=${2:-legacy} # boxed | unboxed | legacy
1163
1164 case $which_wedges in
1165 contrib)
1166 contributor-wedges "$how"
1167 ;;
1168 soil)
1169 contributor-wedges "$how"
1170 extra-wedges "$how"
1171 ;;
1172 extra-only)
1173 extra-wedges "$how"
1174 ;;
1175 smoke)
1176 #zsh-wedges "$how"
1177 smoke-wedges "$how"
1178 ;;
1179 cmark) # for testing mkdir /wedge BUG
1180 cmark-wedges "$how"
1181 ;;
1182 *)
1183 die "Invalid which_wedges $which_wedges"
1184 ;;
1185 esac
1186}
1187
1188install-wedges-parallel() {
1189 local which_wedges=${1:-contrib} # contrib | soil | smoke
1190 local how=${2:-legacy} # boxed | unboxed | legacy
1191 local allowed_failures=${3:-0} # sourcehut CI jobs allows failures for now
1192
1193 # For contributor setup: we need to use this BEFORE running build/py.sh all
1194 build/py.sh time-helper
1195
1196 echo " START $(timestamp)"
1197
1198 # Do all of them in parallel
1199 print-wedge-list "$which_wedges" "$how" | install-wedge-list "$how" T
1200
1201 # python3 interpreter is needed to download and install python packages
1202 mkdir -p ../oils.DEPS/bin # must create it
1203 deps/make-bin.sh python3 ../oils.DEPS
1204
1205 local wedge_out_dir
1206 case $how in
1207 unboxed)
1208 wedge_out_dir=$WEDGE_2025_DIR
1209 ;;
1210 legacy)
1211 wedge_out_dir=$USER_WEDGE_DIR
1212 ;;
1213 *)
1214 die "Invalid how $how"
1215 ;;
1216 esac
1217 fake-py3-libs-wedge "$wedge_out_dir"
1218
1219 echo " END $(timestamp)"
1220
1221 write-task-report $WEDGE_LOG_DIR
1222
1223 check-for-failure $WEDGE_LOG_DIR/tasks.tsv "$allowed_failures"
1224}
1225
1226install-wedges-OLD() {
1227 # the /wedge and ~/wedge stuff we got rid of
1228 install-wedges-parallel contrib legacy
1229}
1230
1231install-wedges-contrib-unboxed() {
1232 ### Install in ../oils.DEPS (aka unboxed)
1233
1234 # ../oils.DEPS/wedge
1235 install-wedges-parallel contrib unboxed
1236
1237 # ../oils.DEPS/bin
1238 deps/make-bin.sh contrib
1239}
1240
1241install-wedges() {
1242 ### What we tell users to run
1243
1244 # Migrated as of 2025-10
1245 install-wedges-contrib-unboxed "$@"
1246}
1247
1248install-wedges-soil() {
1249 ### Github Actions dev-setup-debian job calls this
1250 local allowed_failures=${1:-0}
1251
1252 # ../oils.DEPS/wedge
1253 install-wedges-parallel soil unboxed "$allowed_failures"
1254
1255 # ../oils.DEPS/bin
1256 deps/make-bin.sh contrib
1257}
1258
1259#
1260# Sourcehut dev-setup-{debian,fedora,alpine} jobs call these functions
1261#
1262
1263install-wedges-debian-fail() {
1264 install-wedges-soil 2
1265}
1266
1267install-wedges-fedora-fail() {
1268 install-wedges-soil 2
1269}
1270
1271install-wedges-alpine-fail() {
1272 install-wedges-soil 4 # 4 allowed failures for now
1273}
1274
1275#
1276# Wedges built inside a container, for copying into a container
1277#
1278
1279boxed-clean() {
1280 # Source dir is _build/deps-source
1281 time sudo rm -r -f _build/boxed
1282}
1283
1284boxed-uftrace-OLD() {
1285 ### uftrace /wedge build: until we can move it to ../oils.DEPS/wedge
1286
1287 deps/wedge.sh boxed deps/source.medo/uftrace/ '' $ROOT_WEDGE_DIR debian-12
1288}
1289
1290maybe-boxed-wedge() {
1291 local name=$1
1292 local version=$2
1293 local wedge_base_dir=$3 # e.g. oils.DEPS or _build/boxed/wedge?
1294 local distro=$4 # e.g. debian-12 or empty
1295
1296 local task_file=$BOXED_LOG_DIR/$name-$version.task.tsv
1297 local log_file=$BOXED_LOG_DIR/$name-$version.log.txt
1298
1299 echo " TASK $(timestamp) $name $version > $log_file"
1300
1301 # python3 because it's OUTSIDE the container
1302 # Separate columns that could be joined: number of files, total size
1303 my-time-tsv --print-header \
1304 --field xargs_slot \
1305 --field wedge \
1306 --field wedge_HREF \
1307 --field version \
1308 --output $task_file
1309
1310 if boxed-wedge-exists "$name" "$version" "$distro"; then
1311 echo "CACHED $(timestamp) $name $version"
1312 return
1313 fi
1314
1315 #local -a cmd=( deps/wedge.sh boxed-2025 _build/deps-source/$name/ $version)
1316 local -a cmd=(
1317 deps/wedge.sh boxed-2025 deps/source.medo/$name/ "$version" "$wedge_base_dir" "$distro"
1318 )
1319
1320 set +o errexit
1321 my-time-tsv \
1322 --field "$XARGS_SLOT" \
1323 --field "$name" \
1324 --field "$name-$version.log.txt" \
1325 --field "$version" \
1326 --append \
1327 --output $task_file \
1328 -- \
1329 "${cmd[@]}" "$@" >$log_file 2>&1
1330 local status=$?
1331 set -o errexit
1332
1333 if test "$status" -eq 0; then
1334 echo " OK $(timestamp) $name $version"
1335 else
1336 echo " FAIL $(timestamp) $name $version"
1337 fi
1338}
1339
1340do-boxed-wedge-list() {
1341 ### Reads task rows from stdin
1342 local parallel=${1:-}
1343
1344 mkdir -p $BOXED_LOG_DIR
1345
1346 local -a flags
1347 if test -n "$parallel"; then
1348 log ""
1349 log "=== Building boxed wedges with $NPROC jobs in parallel"
1350 log ""
1351 flags=( -P $NPROC )
1352 else
1353 log ""
1354 log "=== Building boxed wedges serially"
1355 log ""
1356 fi
1357
1358 # Reads from stdin
1359 # Note: --process-slot-var requires GNU xargs! busybox args doesn't have it.
1360 #
1361 # $name $version $wedge_dir
1362 xargs "${flags[@]}" -n 4 --process-slot-var=XARGS_SLOT -- $0 maybe-boxed-wedge
1363}
1364
1365_boxed-wedges-2025() {
1366 local which_wedges=${1:-contrib} # contrib | soil | smoke
1367
1368 # deps/wedge.sh reads this
1369 export DOCKER=${2:-podman}
1370
1371 # For contributor setup: we need to use this BEFORE running build/py.sh all
1372 #build/py.sh time-helper
1373
1374 echo " START $(timestamp)"
1375
1376 # Do all of them in parallel
1377 print-wedge-list "$which_wedges" boxed | do-boxed-wedge-list T
1378
1379 # Dockerfile.* calls install-py3-libs from within the container, so we don't need this
1380 # It depends on MyPy
1381 #fake-py3-libs-wedge ../oils.DEPS/wedge
1382
1383 echo " END $(timestamp)"
1384
1385 write-task-report $BOXED_LOG_DIR
1386}
1387
1388boxed-wedges-2025() {
1389 time $0 _boxed-wedges-2025 "$@"
1390}
1391
1392libssl-bug() {
1393 set -x
1394 file=_build/wedge/binary/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/lib-dynload/_ssl.cpython-310-x86_64-linux-gnu.so
1395
1396 # TODO: figure out how to get this to find libssl.so.3, not .so.1.1
1397 ldd $file
1398 echo
1399 ls -l $file
1400}
1401
1402libssl-smoke() {
1403 deps/wedge.sh smoke-test deps/source.medo/python3/ '' '' debian-12
1404}
1405
1406smoke-unboxed-boxed() {
1407 install-wedges smoke legacy
1408 echo
1409 ls -l ~/wedge/oils-for-unix.org/pkg/dash
1410
1411 install-wedges smoke unboxed
1412 echo
1413 ls -l ../oils.DEPS/wedge/dash
1414
1415 boxed-wedges-2025 smoke
1416 echo
1417 ls -l _build/boxed/wedge/dash
1418
1419 # old boxed wedges: they never had xargs automation
1420 #boxed-wedges
1421
1422 # TODO: now invalidate cache, and build again
1423}
1424
1425task-five "$@"