OILS / build / deps.sh View on Github | oilshell.org

1223 lines, 625 significant
1#!/usr/bin/env bash
2#
3# Script for contributors to build dev dependencies -- packaged as cross-distro
4# "wedges". Tested in the Soil CI.
5#
6# Usage:
7# build/deps.sh <function name>
8#
9# Examples:
10# build/deps.sh fetch
11# build/deps.sh install-wedges # for both Python and C++
12#
13# build/deps.sh rm-oils-crap # rm -r -f /wedge ~/wedge to start over
14#
15# TODO: Do we need something faster, just python2, re2c, and cmark?
16#
17# - build/deps.sh fetch-py
18# - build/deps.sh install-wedges-py
19#
20# TODO: Can we make most of them non-root deps? This requires rebuilding
21# containers, which requires podman.
22#
23# rm -r -f ~/wedge # would be better
24
25
26# Check if we're in the right directory
27if [[ ! -d "stdlib/osh" ]]; then
28 echo "Error: This script must be run from the root of the Oil project directory"
29 echo "Please cd to the root directory and try again"
30 exit 1
31fi
32
33: ${LIB_OSH=stdlib/osh}
34if [[ ! -f "$LIB_OSH/bash-strict.sh" ]] || [[ ! -f "$LIB_OSH/task-five.sh" ]]; then
35 echo "Error: Required source files not found in $LIB_OSH/"
36 echo "Expected files:"
37 echo " - $LIB_OSH/bash-strict.sh"
38 echo " - $LIB_OSH/task-five.sh"
39 exit 1
40fi
41
42source $LIB_OSH/bash-strict.sh
43source $LIB_OSH/task-five.sh
44
45REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
46
47source build/dev-shell.sh # python3 in PATH, PY3_LIBS_VERSION
48source deps/from-apt.sh # PY3_BUILD_DEPS
49#source deps/podman.sh
50source test/tsv-lib.sh # tsv-concat
51source web/table/html.sh # table-sort-{begin,end}
52
53# Also in build/dev-shell.sh
54USER_WEDGE_DIR=~/wedge/oils-for-unix.org
55ROOT_WEDGE_DIR=/wedge/oils-for-unix.org
56
57readonly DEPS_SOURCE_DIR=_build/deps-source
58
59readonly RE2C_VERSION=3.0
60readonly RE2C_URL="https://github.com/skvadrik/re2c/releases/download/$RE2C_VERSION/re2c-$RE2C_VERSION.tar.xz"
61
62readonly CMARK_VERSION=0.29.0
63readonly CMARK_URL="https://github.com/commonmark/cmark/archive/$CMARK_VERSION.tar.gz"
64
65readonly PY_FTP_MIRROR="${PY_FTP_MIRROR:-https://www.python.org/ftp}"
66
67readonly PY2_VERSION=2.7.18
68readonly PY2_URL="$PY_FTP_MIRROR/python/$PY2_VERSION/Python-$PY2_VERSION.tar.xz"
69
70readonly PY3_VERSION=3.10.4
71readonly PY3_URL="$PY_FTP_MIRROR/python/$PY3_VERSION/Python-$PY3_VERSION.tar.xz"
72
73readonly BASH_VER=4.4 # don't clobber BASH_VERSION
74readonly BASH_URL="https://www.oilshell.org/blob/spec-bin/bash-$BASH_VER.tar.gz"
75
76# Another version of bash to test
77readonly BASH5_VER=5.2.21
78readonly BASH5_URL="https://www.oilshell.org/blob/spec-bin/bash-$BASH5_VER.tar.gz"
79
80readonly DASH_VERSION=0.5.10.2
81readonly DASH_URL="https://www.oilshell.org/blob/spec-bin/dash-$DASH_VERSION.tar.gz"
82
83readonly ZSH_VERSION=5.1.1
84readonly ZSH_URL="https://www.oilshell.org/blob/spec-bin/zsh-$ZSH_VERSION.tar.xz"
85
86readonly MKSH_VERSION=R52c
87readonly MKSH_URL="https://www.oilshell.org/blob/spec-bin/mksh-$MKSH_VERSION.tgz"
88
89readonly BUSYBOX_VERSION='1.35.0'
90readonly BUSYBOX_URL="https://www.oilshell.org/blob/spec-bin/busybox-$BUSYBOX_VERSION.tar.bz2"
91
92readonly YASH_VERSION=2.49
93readonly YASH_URL="https://www.oilshell.org/blob/spec-bin/yash-$YASH_VERSION.tar.xz"
94
95readonly MYPY_GIT_URL=https://github.com/python/mypy
96readonly MYPY_VERSION=0.780
97
98readonly PY3_LIBS=~/wedge/oils-for-unix.org/pkg/py3-libs/$MYPY_VERSION
99
100# Version 2.4.0 from 2021-10-06 was the last version that supported Python 2
101# https://github.com/PyCQA/pyflakes/blob/main/NEWS.rst
102readonly PYFLAKES_VERSION=2.4.0
103#readonly PYFLAKES_URL='https://files.pythonhosted.org/packages/15/60/c577e54518086e98470e9088278247f4af1d39cb43bcbd731e2c307acd6a/pyflakes-2.4.0.tar.gz'
104# 2023-07: Mirrored to avoid network problem on broome during release
105readonly PYFLAKES_URL='https://www.oilshell.org/blob/pyflakes-2.4.0.tar.gz'
106
107readonly BLOATY_VERSION=1.1
108readonly BLOATY_URL='https://github.com/google/bloaty/releases/download/v1.1/bloaty-1.1.tar.bz2'
109
110readonly UFTRACE_VERSION=0.13
111readonly UFTRACE_URL='https://github.com/namhyung/uftrace/archive/refs/tags/v0.13.tar.gz'
112
113readonly SOUFFLE_VERSION=2.4.1
114readonly SOUFFLE_URL=https://github.com/souffle-lang/souffle/archive/refs/tags/2.4.1.tar.gz
115
116log() {
117 echo "$@" >& 2
118}
119
120die() {
121 log "$0: fatal: $@"
122 exit 1
123}
124
125rm-oils-crap() {
126 ### When you want to start over
127
128 rm -r -f -v ~/wedge
129 sudo rm -r -f -v /wedge
130}
131
132# Note: git is an implicit dependency -- that's how we got the repo in the
133# first place!
134
135# python2-dev is no longer available on Debian 12
136# python-dev also seems gone
137#
138# wget: for fetching wedges (not on Debian by default!)
139# tree: tiny package that's useful for showing what we installed
140# g++: essential
141# libreadline-dev: needed for the build/prepare.sh Python build.
142# gawk: used by spec-runner.sh for the special match() function.
143# cmake: for cmark
144# PY3_BUILD_DEPS - I think these will be used for building the Python 2 wedge
145# as well
146readonly -a WEDGE_DEPS_DEBIAN=(
147 bzip2
148 wget
149 tree
150 gawk
151 g++
152 ninja-build
153 cmake
154 libreadline-dev
155 systemtap-sdt-dev
156
157 # for Souffle, flex and bison
158 #flex bison
159
160 "${PY3_BUILD_DEPS[@]}"
161)
162
163readonly -a WEDGE_DEPS_ALPINE=(
164 bzip2
165 xz
166
167 wget tree gawk
168
169 gcc g++
170 ninja-build
171 # https://pkgs.alpinelinux.org/packages?name=ninja-is-really-ninja&branch=v3.19&repo=&arch=&maintainer=
172 ninja-is-really-ninja
173 cmake
174
175 readline-dev
176 zlib-dev
177 libffi-dev
178 openssl-dev
179
180 ncurses-dev
181
182 # for Souffle, flex and bison
183 #flex bison
184)
185
186readonly -a WEDGE_DEPS_FEDORA=(
187
188 # Weird, Fedora doesn't have these by default!
189 hostname
190 tar
191 bzip2
192
193 # https://packages.fedoraproject.org/pkgs/wget/wget/
194 wget
195 # https://packages.fedoraproject.org/pkgs/tree-pkg/tree/
196 tree
197 gawk
198
199 # https://packages.fedoraproject.org/pkgs/gcc/gcc/
200 gcc gcc-c++
201
202 ninja-build
203 cmake
204
205 readline-devel
206
207 # Like PY3_BUILD_DEPS
208 # https://packages.fedoraproject.org/pkgs/zlib/zlib-devel/
209 zlib-devel
210 # https://packages.fedoraproject.org/pkgs/libffi/libffi-devel/
211 libffi-devel
212 # https://packages.fedoraproject.org/pkgs/openssl/openssl-devel/
213 openssl-devel
214
215 # For building zsh from source?
216 # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
217 ncurses-devel
218 #libcap-devel
219
220 # still have a job control error compiling bash
221 # https://packages.fedoraproject.org/pkgs/glibc/glibc-devel/
222 # glibc-devel
223
224 libasan
225)
226
227readonly -a WEDGE_DEPS_ARCH=(
228 # https://archlinux.org/packages/core/x86_64/bzip2/
229 bzip2
230
231 # https://archlinux.org/packages/extra/x86_64/wget/
232 wget
233
234 # https://archlinux.org/packages/extra/x86_64/tree/
235 tree
236
237 # https://archlinux.org/packages/core/x86_64/gawk/
238 gawk
239
240 # https://archlinux.org/packages/core/x86_64/gcc/
241 gcc
242
243 # https://archlinux.org/packages/community/x86_64/ninja/
244 ninja
245
246 # https://archlinux.org/packages/extra/x86_64/cmake/
247 cmake
248
249 # https://archlinux.org/packages/core/x86_64/readline/
250 readline
251
252 # https://archlinux.org/packages/core/x86_64/zlib/
253 zlib
254
255 # https://archlinux.org/packages/core/x86_64/libffi/
256 libffi
257
258 # https://archlinux.org/packages/core/x86_64/openssl/
259 openssl
260
261 # https://archlinux.org/packages/core/x86_64/ncurses/
262 ncurses
263
264 # Development headers are included in the main packages on Arch,
265 # unlike other distros that separate them into -dev/-devel packages
266
267 # Python 2 from the AUR
268 # https://aur.archlinux.org/packages/python2
269 base-devel # needed for building packages from the AUR
270
271)
272
273
274install-debian-packages() {
275 ### Packages for build/py.sh all, building wedges, etc.
276
277 set -x # show what needs sudo
278
279 # pass -y for say gitpod
280 sudo apt "$@" install "${WEDGE_DEPS_DEBIAN[@]}"
281 set +x
282
283 # maybe pass -y through
284 test/spec-bin.sh install-shells-with-apt "$@"
285}
286
287install-ubuntu-packages() {
288 ### Debian and Ubuntu packages are the same; this function is suggested on the wiki
289 install-debian-packages "$@"
290}
291
292wedge-deps-debian() {
293 # Install packages without prompt
294
295 # 2024-02 - there was an Ubuntu update, and we started needing this
296 sudo apt-get -y update
297
298 install-debian-packages -y
299}
300
301wedge-deps-fedora() {
302 # https://linuxconfig.org/install-development-tools-on-redhat-8
303 # Trying to get past compile errors
304 # sudo dnf group install --assumeyes 'Development Tools'
305
306 sudo dnf install --assumeyes "${WEDGE_DEPS_FEDORA[@]}"
307}
308
309wedge-deps-alpine() {
310 # https://linuxconfig.org/install-development-tools-on-redhat-8
311 # Trying to get past compile errors
312 # sudo dnf group install --assumeyes 'Development Tools'
313
314 sudo apk add "${WEDGE_DEPS_ALPINE[@]}"
315}
316
317wedge-deps-arch() {
318 # Install packages without prompt
319
320 # First sync the package database
321 sudo pacman -Sy
322
323 # Then install packages
324 for pkg in "${WEDGE_DEPS_ARCH[@]}"; do
325 # Only install if not already installed
326 if ! pacman -Qi "$pkg" >/dev/null 2>&1; then
327 sudo pacman --noconfirm -S "$pkg"
328 fi
329 done
330}
331
332#
333# Unused patch, was experiment for Fedora
334#
335
336get-typed-ast-patch() {
337 curl -o deps/typed_ast.patch https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96.patch
338}
339
340# Work around typed_ast bug:
341# https://github.com/python/typed_ast/issues/169
342#
343# Apply this patch
344# https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96
345#
346# typed_ast is tarred up though
347patch-typed-ast() {
348 local package_dir=_cache/py3-libs
349 local patch=$PWD/deps/typed_ast.patch
350
351 pushd $package_dir
352 cat $patch
353 echo
354
355 local dir=typed_ast-1.4.3
356 local tar=typed_ast-1.4.3.tar.gz
357
358 echo OLD
359 ls -l $tar
360 echo
361
362 rm -r -f -v $dir
363 tar -x -z < $tar
364
365 pushd $dir
366 patch -p1 < $patch
367 popd
368 #find $dir
369
370 # Create a new one
371 tar --create --gzip --file $tar typed_ast-1.4.3
372
373 echo NEW
374 ls -l $tar
375 echo
376
377 popd
378}
379
380#
381# Fetch
382#
383
384download-to() {
385 local dir=$1
386 local url=$2
387 wget --no-clobber --directory-prefix "$dir" "$url"
388}
389
390maybe-extract() {
391 local wedge_dir=$1
392 local tar_name=$2
393 local out_dir=$3
394
395 if test -d "$wedge_dir/$out_dir"; then
396 log "Not extracting because $wedge_dir/$out_dir exists"
397 return
398 fi
399
400 local tar=$wedge_dir/$tar_name
401 case $tar_name in
402 *.gz|*.tgz) # mksh ends with .tgz
403 flag='--gzip'
404 ;;
405 *.bz2)
406 flag='--bzip2'
407 ;;
408 *.xz)
409 flag='--xz'
410 ;;
411 *)
412 die "tar with unknown extension: $tar_name"
413 ;;
414 esac
415
416 tar --extract $flag --file $tar --directory $wedge_dir
417}
418
419clone-mypy() {
420 ### replaces deps/from-git
421 local dest_dir=$1
422 local version=${2:-$MYPY_VERSION}
423
424 local dest=$dest_dir/mypy-$version
425 if test -d $dest; then
426 log "Not cloning because $dest exists"
427 return
428 fi
429
430 # v$VERSION is a tag, not a branch
431
432 # size optimization: --depth=1 --shallow-submodules
433 # https://git-scm.com/docs/git-clone
434
435 git clone --recursive --branch v$version \
436 --depth=1 --shallow-submodules \
437 $MYPY_GIT_URL $dest
438
439 # TODO: verify commit checksum
440}
441
442copy-source-medo() {
443 mkdir -p $DEPS_SOURCE_DIR
444
445 # Copy the whole tree, including the .treeptr files
446 cp --verbose --recursive --no-target-directory \
447 deps/source.medo/ $DEPS_SOURCE_DIR/
448}
449
450fetch-spec-bin() {
451 download-to $DEPS_SOURCE_DIR/bash "$BASH_URL"
452 maybe-extract $DEPS_SOURCE_DIR/bash "$(basename $BASH_URL)" bash-$BASH_VER
453
454 download-to $DEPS_SOURCE_DIR/bash "$BASH5_URL"
455 maybe-extract $DEPS_SOURCE_DIR/bash "$(basename $BASH5_URL)" bash-$BASH5_VER
456
457 download-to $DEPS_SOURCE_DIR/dash "$DASH_URL"
458 maybe-extract $DEPS_SOURCE_DIR/dash "$(basename $DASH_URL)" dash-$DASH_VERSION
459
460 download-to $DEPS_SOURCE_DIR/zsh "$ZSH_URL"
461 maybe-extract $DEPS_SOURCE_DIR/zsh "$(basename $ZSH_URL)" zsh-$ZSH_VERSION
462
463 download-to $DEPS_SOURCE_DIR/mksh "$MKSH_URL"
464 maybe-extract $DEPS_SOURCE_DIR/mksh "$(basename $MKSH_URL)" mksh-$MKSH_VERSION
465
466 download-to $DEPS_SOURCE_DIR/busybox "$BUSYBOX_URL"
467 maybe-extract $DEPS_SOURCE_DIR/busybox "$(basename $BUSYBOX_URL)" busybox-$BUSYBOX_VERSION
468
469 download-to $DEPS_SOURCE_DIR/yash "$YASH_URL"
470 maybe-extract $DEPS_SOURCE_DIR/yash "$(basename $YASH_URL)" yash-$YASH_VERSION
471
472 # Patch: this tarball doesn't follow the convention $name-$version
473 if test -d $DEPS_SOURCE_DIR/mksh/mksh; then
474 pushd $DEPS_SOURCE_DIR/mksh
475 mv -v mksh mksh-$MKSH_VERSION
476 popd
477 fi
478}
479
480fetch() {
481 local py_only=${1:-}
482
483 # For now, simulate what 'medo expand deps/source.medo _build/deps-source'
484 # would do: fetch compressed tarballs designated by .treeptr files, and
485 # expand them.
486
487 # _build/deps-source/
488 # re2c/
489 # WEDGE
490 # re2c-3.0/ # expanded .tar.xz file
491
492 copy-source-medo
493
494 download-to $DEPS_SOURCE_DIR/re2c "$RE2C_URL"
495 download-to $DEPS_SOURCE_DIR/cmark "$CMARK_URL"
496 maybe-extract $DEPS_SOURCE_DIR/re2c "$(basename $RE2C_URL)" re2c-$RE2C_VERSION
497 maybe-extract $DEPS_SOURCE_DIR/cmark "$(basename $CMARK_URL)" cmark-$CMARK_VERSION
498
499 if test -n "$py_only"; then
500 log "Fetched dependencies for 'build/py.sh'"
501 return
502 fi
503
504 download-to $DEPS_SOURCE_DIR/pyflakes "$PYFLAKES_URL"
505 maybe-extract $DEPS_SOURCE_DIR/pyflakes "$(basename $PYFLAKES_URL)" \
506 pyflakes-$PYFLAKES_VERSION
507
508 download-to $DEPS_SOURCE_DIR/python2 "$PY2_URL"
509 download-to $DEPS_SOURCE_DIR/python3 "$PY3_URL"
510 maybe-extract $DEPS_SOURCE_DIR/python2 "$(basename $PY2_URL)" Python-$PY2_VERSION
511 maybe-extract $DEPS_SOURCE_DIR/python3 "$(basename $PY3_URL)" Python-$PY3_VERSION
512
513 fetch-spec-bin
514
515 # bloaty and uftrace are for benchmarks, in containers
516 download-to $DEPS_SOURCE_DIR/bloaty "$BLOATY_URL"
517 download-to $DEPS_SOURCE_DIR/uftrace "$UFTRACE_URL"
518 maybe-extract $DEPS_SOURCE_DIR/bloaty "$(basename $BLOATY_URL)" bloaty-$BLOATY_VERSION
519 maybe-extract $DEPS_SOURCE_DIR/uftrace "$(basename $UFTRACE_URL)" uftrace-$UFTRACE_VERSION
520
521 # This is in $DEPS_SOURCE_DIR to COPY into containers, which mycpp will directly import.
522
523 # It's also copied into a wedge in install-wedges.
524 clone-mypy $DEPS_SOURCE_DIR/mypy
525
526 if false; then
527 download-to $DEPS_SOURCE_DIR/souffle "$SOUFFLE_URL"
528 maybe-extract $DEPS_SOURCE_DIR/souffle "$(basename $SOUFFLE_URL)" souffle-$SOUFFLE_VERSION
529 fi
530
531 if command -v tree > /dev/null; then
532 tree -L 2 $DEPS_SOURCE_DIR
533 fi
534}
535
536fetch-py() {
537 fetch py_only
538}
539
540mirror-pyflakes() {
541 ### Workaround for network error during release
542 scp \
543 $DEPS_SOURCE_DIR/pyflakes/"$(basename $PYFLAKES_URL)" \
544 oilshell.org:oilshell.org/blob/
545}
546
547mirror-python() {
548 ### Can't reach python.org from some machines
549 scp \
550 $DEPS_SOURCE_DIR/python2/"$(basename $PY2_URL)" \
551 oilshell.org:oilshell.org/blob/
552
553 scp \
554 $DEPS_SOURCE_DIR/python3/"$(basename $PY3_URL)" \
555 oilshell.org:oilshell.org/blob/
556}
557
558wedge-exists() {
559 ### Does an installed wedge already exist?
560
561 local name=$1
562 local version=$2
563 local wedge_dir=${3:-/wedge/oils-for-unix.org}
564
565 local installed=$wedge_dir/pkg/$name/$version
566
567 if test -d $installed; then
568 log "$installed already exists"
569 return 0
570 else
571 return 1
572 fi
573}
574
575#
576# Install
577#
578
579# TODO: py3-libs needs to be a WEDGE, so that that you can run
580# 'wedge build deps/source.medo/py3-libs/' and then get it in
581#
582# _build/wedge/{absolute,relative} # which one?
583#
584# It needs a BUILD DEPENDENCY on:
585# - the python3 wedge, so you can do python3 -m pip install.
586# - the mypy repo, which has test-requirements.txt
587
588download-py3-libs() {
589 ### Download source/binary packages, AFTER python3 is installed
590
591 # Note that this is NOT source code; there is binary code, e.g. in
592 # lxml-*.whl
593
594 local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
595 local py_package_dir=_cache/py3-libs
596 mkdir -p $py_package_dir
597
598 # Avoids a warning, but doesn't fix typed_ast
599 #python3 -m pip download -d $py_package_dir wheel
600
601 python3 -m pip download -d $py_package_dir -r $mypy_dir/test-requirements.txt
602 python3 -m pip download -d $py_package_dir pexpect
603}
604
605install-py3-libs-in-venv() {
606 local venv_dir=$1
607 local mypy_dir=$2 # This is a param for host build vs. container build
608 local package_dir=_cache/py3-libs
609
610 source $venv_dir/bin/activate # enter virtualenv
611
612 # 2023-07 note: we're installing yapf in a DIFFERENT venv, because it
613 # conflicts with MyPy deps!
614 # "ERROR: pip's dependency resolver does not currently take into account all
615 # the packages that are installed."
616
617 # --find-links uses a "cache dir" for packages (weird flag name!)
618
619 # Avoids a warning, but doesn't fix typed_ast
620 #time python3 -m pip install --find-links $package_dir wheel
621
622 upgrade-typed-ast $mypy_dir/mypy-requirements.txt
623
624 # for mycpp/
625 time python3 -m pip install --find-links $package_dir -r $mypy_dir/test-requirements.txt
626
627 # pexpect: for spec/stateful/*.py
628 time python3 -m pip install --find-links $package_dir pexpect
629}
630
631upgrade-typed-ast() {
632 local file=$1
633 sed -i 's/typed_ast.*/typed_ast==1.5.0/' $file
634}
635
636test-typed-ast() {
637 local dir=~/wedge/oils-for-unix.org/pkg/mypy/0.780
638
639 cp -v $dir/mypy-requirements.txt _tmp
640
641 local file=_tmp/mypy-requirements.txt
642 cat $file
643 #echo
644
645 # 1.5.0 fixed this bug
646 # https://github.com/python/typed_ast/issues/169
647
648 upgrade-typed-ast $file
649 echo
650 cat $file
651}
652
653install-py3-libs-from-cache() {
654
655 # As well as end users
656
657 local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
658
659 local py3
660 py3=$(command -v python3)
661 case $py3 in
662 *wedge/oils-for-unix.org/*)
663 ;;
664 *)
665 die "python3 is '$py3', but expected it to be in a wedge"
666 ;;
667 esac
668
669 log "Ensuring pip is installed (interpreter $(command -v python3)"
670 python3 -m ensurepip
671
672 local venv_dir=$USER_WEDGE_DIR/pkg/py3-libs/$PY3_LIBS_VERSION
673 log "Creating venv in $venv_dir"
674
675 # Note: the bin/python3 in this venv is a symlink to python3 in $PATH, i.e.
676 # the /wedge we just built
677 python3 -m venv $venv_dir
678
679 log "Installing MyPy deps in venv"
680
681 # Run in a subshell because it mutates shell state
682 $0 install-py3-libs-in-venv $venv_dir $mypy_dir
683}
684
685install-py3-libs() {
686 ### Invoked by Dockerfile.cpp-small, etc.
687 local mypy_dir=${1:-}
688
689 download-py3-libs "$mypy_dir"
690 install-py3-libs-from-cache "$mypy_dir"
691}
692
693# zsh notes
694 # Fedora compiler error
695 # zsh ./configure is NOT detecting 'boolcodes', and then it has a broken
696 # fallback in Src/Modules/termcap.c that causes a compile error! It seems
697 # like ncurses-devel should fix this, but it doesn't
698 #
699 # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
700 #
701 # from /home/build/oil/_build/deps-source/zsh/zsh-5.1.1/Src/Modules/termcap.c:38:
702 # /usr/include/term.h:783:56: note: previous declaration of ‘boolcodes’ with type ‘const char * const[]’
703 # 783 | extern NCURSES_EXPORT_VAR(NCURSES_CONST char * const ) boolcodes[];
704 #
705 # I think the ./configure is out of sync with the actual build?
706
707
708# TODO:
709# - $ROOT_WEDGE_DIR vs. $USER_WEDGE_DIR is duplicating information that's
710# already in each WEDGE file
711
712py-wedges() {
713 ### for build/py.sh all
714
715 echo cmark $CMARK_VERSION $ROOT_WEDGE_DIR
716 echo re2c $RE2C_VERSION $ROOT_WEDGE_DIR
717 echo python2 $PY2_VERSION $ROOT_WEDGE_DIR
718 echo pyflakes $PYFLAKES_VERSION $USER_WEDGE_DIR
719}
720
721cpp-wedges() {
722 ### for ninja / mycpp translation
723
724 echo python3 $PY3_VERSION $ROOT_WEDGE_DIR
725 echo mypy $MYPY_VERSION $USER_WEDGE_DIR
726
727 # py3-libs has a built time dep on both python3 and MyPy, so we're doing it
728 # separately for now
729 #echo py3-libs $PY3_LIBS_VERSION $USER_WEDGE_DIR
730}
731
732spec-bin-wedges() {
733 ### for test/spec-py.sh osh-all
734
735 echo dash $DASH_VERSION $USER_WEDGE_DIR
736 echo bash $BASH_VER $USER_WEDGE_DIR
737 echo bash $BASH5_VER $USER_WEDGE_DIR
738 echo mksh $MKSH_VERSION $USER_WEDGE_DIR
739 echo zsh $ZSH_VERSION $USER_WEDGE_DIR
740 echo busybox $BUSYBOX_VERSION $USER_WEDGE_DIR
741 echo yash $YASH_VERSION $USER_WEDGE_DIR
742}
743
744contributor-wedges() {
745 py-wedges
746 cpp-wedges
747 spec-bin-wedges
748}
749
750extra-wedges() {
751 # Contributors don't need uftrace, bloaty, and probably R-libs
752 # Although R-libs could be useful for benchmarks
753
754 # Test both outside the contianer, as well as inside?
755 echo uftrace $UFTRACE_VERSION $ROOT_WEDGE_DIR
756 echo bloaty $BLOATY_VERSION $ROOT_WEDGE_DIR
757
758 #echo souffle $SOUFFLE_VERSION $USER_WEDGE_DIR
759}
760
761timestamp() {
762 date '+%H:%M:%S'
763}
764
765my-time-tsv() {
766 python3 benchmarks/time_.py \
767 --tsv \
768 --time-span --rusage \
769 "$@"
770}
771
772maybe-install-wedge() {
773 local name=$1
774 local version=$2
775 local wedge_dir=$3 # e.g. $USER_WEDGE_DIR or empty
776
777 local task_file=$WEDGE_LOG_DIR/$name-$version.task.tsv
778 local log_file=$WEDGE_LOG_DIR/$name-$version.log.txt
779
780 echo " TASK $(timestamp) $name $version > $log_file"
781
782 # python3 because it's OUTSIDE the container
783 # Separate columns that could be joined: number of files, total size
784 my-time-tsv --print-header \
785 --field xargs_slot \
786 --field wedge \
787 --field wedge_HREF \
788 --field version \
789 --output $task_file
790
791 if wedge-exists "$name" "$version" "$wedge_dir"; then
792 echo "CACHED $(timestamp) $name $version"
793 return
794 fi
795
796 local -a cmd=( deps/wedge.sh unboxed _build/deps-source/$name/ $version)
797
798 set +o errexit
799 my-time-tsv \
800 --field "$XARGS_SLOT" \
801 --field "$name" \
802 --field "$name-$version.log.txt" \
803 --field "$version" \
804 --append \
805 --output $task_file \
806 "${cmd[@]}" "$@" >$log_file 2>&1
807 local status=$?
808 set -o errexit
809
810 if test "$status" -eq 0; then
811 echo " OK $(timestamp) $name $version"
812 else
813 echo " FAIL $(timestamp) $name $version"
814 fi
815}
816
817dummy-task() {
818 ### For testing log capture
819 local name=$1
820 local version=$2
821
822 echo "Building $name $version"
823
824 # random float between 0 and 3
825 # weirdly we need a seed from bash
826 # https://stackoverflow.com/questions/4048378/random-numbers-generation-with-awk-in-bash-shell
827 local secs
828 secs=$(awk -v seed=$RANDOM 'END { srand(seed); print rand() * 3 }' < /dev/null)
829
830 echo "sleep $secs"
831 sleep $secs
832
833 echo 'stdout'
834 log 'stderr'
835
836 if test $name = 'mksh'; then
837 echo "simulate failure for $name"
838 exit 2
839 fi
840}
841
842readonly WEDGE_LOG_DIR=_build/wedge/logs
843
844dummy-task-wrapper() {
845 # Similar to test/common.sh run-task-with-status, used by
846 # test/{spec,wild}-runner.sh
847 local name=$1
848 local version=$2
849
850 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
851 local log_file=$WEDGE_LOG_DIR/$name.log.txt
852
853 echo " TASK $(timestamp) $name $version > $log_file"
854
855 # python3 because it's OUTSIDE the container
856 # Separate columns that could be joined: number of files, total size
857 my-time-tsv --print-header \
858 --field xargs_slot \
859 --field wedge \
860 --field wedge_HREF \
861 --field version \
862 --output $task_file
863
864 my-time-tsv \
865 --field "$XARGS_SLOT" \
866 --field "$name" \
867 --field "$name.log.txt" \
868 --field "$version" \
869 --append \
870 --output $task_file \
871 $0 dummy-task "$@" >$log_file 2>&1 || true
872
873 echo " DONE $(timestamp) $name $version"
874}
875
876html-head() {
877 # python3 because we're outside containers
878 PYTHONPATH=. python3 doctools/html_head.py "$@"
879}
880
881index-html() {
882 local tasks_tsv=$1
883
884 local base_url='../../../web'
885 html-head --title 'Wedge Builds' \
886 "$base_url/ajax.js" \
887 "$base_url/table/table-sort.js" \
888 "$base_url/table/table-sort.css" \
889 "$base_url/base.css"\
890
891 table-sort-begin 'width60'
892
893 cat <<EOF
894 <p id="home-link">
895 <a href="/">oilshell.org</a>
896 </p>
897
898 <h1>Wedge Builds</h1>
899EOF
900
901 tsv2html3 $tasks_tsv
902
903 cat <<EOF
904 <p>
905 <a href="tasks.tsv">tasks.tsv</a>
906 </p>
907EOF
908
909 table-sort-end 'tasks' # ID for sorting
910}
911
912NPROC=$(nproc)
913#NPROC=1
914
915install-wedge-list() {
916 ### Reads task rows from stdin
917 local parallel=${1:-}
918
919 mkdir -p _build/wedge/logs
920
921 local -a flags
922 if test -n "$parallel"; then
923 log ""
924 log "=== Installing wedges with $NPROC jobs in parallel"
925 log ""
926 flags=( -P $NPROC )
927 else
928 log ""
929 log "=== Installing wedges serially"
930 log ""
931 fi
932
933 # Reads from stdin
934 # Note: --process-slot-var requires GNU xargs! busybox args doesn't have it.
935 #
936 # $name $version $wedge_dir
937 xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 maybe-install-wedge
938
939 #xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 dummy-task-wrapper
940}
941
942write-task-report() {
943 local tasks_tsv=_build/wedge/logs/tasks.tsv
944
945 python3 devtools/tsv_concat.py $WEDGE_LOG_DIR/*.task.tsv > $tasks_tsv
946 log "Wrote $tasks_tsv"
947
948 # TODO: version can be right-justified?
949 here-schema-tsv-4col >_build/wedge/logs/tasks.schema.tsv <<EOF
950column_name type precision strftime
951status integer 0 -
952elapsed_secs float 1 -
953user_secs float 1 -
954start_time float 1 %H:%M:%S
955end_time float 1 %H:%M:%S
956sys_secs float 1 -
957max_rss_KiB integer 0 -
958xargs_slot integer 0 -
959wedge string 0 -
960wedge_HREF string 0 -
961version string 0 -
962EOF
963
964 index-html $tasks_tsv > $WEDGE_LOG_DIR/index.html
965 log "Wrote $WEDGE_LOG_DIR/index.html"
966}
967
968install-spec-bin-fast() {
969 spec-bin-wedges | install-wedge-list T
970 write-task-report
971}
972
973fake-py3-libs-wedge() {
974 local name=py3-libs
975 local version=$PY3_LIBS_VERSION
976
977 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
978 local log_file=$WEDGE_LOG_DIR/$name.log.txt
979
980 my-time-tsv --print-header \
981 --field xargs_slot \
982 --field wedge \
983 --field wedge_HREF \
984 --field version \
985 --output $task_file
986
987 # There is no xargs slot!
988 my-time-tsv \
989 --field "-1" \
990 --field "$name" \
991 --field "$name.log.txt" \
992 --field "$version" \
993 --append \
994 --output $task_file \
995 $0 install-py3-libs >$log_file 2>&1 || true
996
997 echo " FAKE $(timestamp) $name $version"
998}
999
1000install-wedges() {
1001 local extra=${1:-}
1002
1003 # For contributor setup: we need to use this BEFORE running build/py.sh all
1004 build/py.sh time-helper
1005
1006 echo " START $(timestamp)"
1007
1008 # Do all of them in parallel
1009 if test -n "$extra"; then
1010 { contributor-wedges; extra-wedges; } | install-wedge-list T
1011 else
1012 contributor-wedges | install-wedge-list T
1013 fi
1014
1015 fake-py3-libs-wedge
1016 echo " END $(timestamp)"
1017
1018 write-task-report
1019}
1020
1021install-wedges-soil() {
1022 install-wedges extra
1023}
1024
1025install-wedges-fast() {
1026 ### Alias for compatibility
1027 install-wedges "$@"
1028}
1029
1030#
1031# Unboxed wedge builds
1032#
1033
1034uftrace-host() {
1035 ### built on demand; run $0 first
1036
1037 # BUG: doesn't detect python3
1038 # WEDGE tells me that it depends on pkg-config
1039 # 'apt-get install pkgconf' gets it
1040 # TODO: Should use python3 WEDGE instead of SYSTEM python3?
1041 deps/wedge.sh unboxed _build/deps-source/uftrace
1042}
1043
1044bloaty-host() {
1045 deps/wedge.sh unboxed _build/deps-source/bloaty
1046}
1047
1048R-libs-host() {
1049 deps/wedge.sh unboxed _build/deps-source/R-libs
1050}
1051
1052#
1053# Wedges built inside a container, for copying into a container
1054#
1055
1056boxed-wedges() {
1057 #### host _build/wedge/binary -> guest container /wedge or ~/wedge
1058
1059 #export-podman
1060
1061 # TODO:
1062 #
1063 # - Add equivalents of spec-bin
1064 # - Use the same manifest as install-wedges
1065 # - so then you can delete the _build/wedge dir to re-run it
1066 # - use xargs -n 1 so it's done serially
1067
1068 # - Do these lazily like we do in install-wedges
1069
1070 # We can test if the dir _build/wedge/binary/oils-for-unix.org/pkg/FOO exists
1071 # if wedge-exists "$name" "$version" "$wedge_dir"; then
1072 # echo "CACHED $(timestamp) $name $version"
1073 # return
1074 # fi
1075
1076 if false; then
1077 deps/wedge.sh boxed deps/source.medo/time-helper
1078 deps/wedge.sh boxed deps/source.medo/cmark/
1079 deps/wedge.sh boxed deps/source.medo/re2c/
1080 deps/wedge.sh boxed deps/source.medo/python3/
1081 fi
1082
1083 if false; then
1084 deps/wedge.sh boxed deps/source.medo/bloaty/
1085 fi
1086
1087 if true; then
1088 # build with debian-12, because soil-benchmarks2 is, because it has R
1089 deps/wedge.sh boxed deps/source.medo/uftrace/ '' debian-12
1090 # python2 needed everywhere
1091 #deps/wedge.sh boxed deps/source.medo/python2/ '' debian-12
1092
1093 # TODO: build with debian-12
1094 # Used in {benchmarks,benchmarks2,other-tests}
1095 #deps/wedge.sh boxed deps/source.medo/R-libs/ '' debian-12
1096 fi
1097}
1098
1099boxed-spec-bin() {
1100 if false; then
1101 deps/wedge.sh boxed deps/source.medo/bash '4.4'
1102 fi
1103
1104 if true; then
1105 deps/wedge.sh boxed deps/source.medo/bash '5.2.21'
1106 fi
1107
1108 if true; then
1109 deps/wedge.sh boxed deps/source.medo/dash
1110 deps/wedge.sh boxed deps/source.medo/mksh
1111 fi
1112
1113 if true; then
1114 # Note: zsh requires libncursesw5-dev
1115 deps/wedge.sh boxed deps/source.medo/zsh
1116
1117 deps/wedge.sh boxed deps/source.medo/busybox
1118
1119 # Problem with out of tree build, as above. Skipping for now
1120 deps/wedge.sh boxed deps/source.medo/yash
1121 echo
1122 fi
1123}
1124
1125#
1126# Report
1127#
1128
1129commas() {
1130 # Wow I didn't know this :a trick
1131 #
1132 # OK this is a label and a loop, which makes sense. You can't do it with
1133 # pure regex.
1134 #
1135 # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
1136 # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
1137 sed ':a;s/\b\([0-9]\+\)\([0-9]\{3\}\)\b/\1,\2/;ta'
1138}
1139
1140wedge-sizes() {
1141 local tmp=_tmp/wedge-sizes.txt
1142
1143 # -b is --bytes, but use short flag for busybox compat
1144 du -s -b /wedge/*/*/* ~/wedge/*/*/* | awk '
1145 { print $0 # print the line
1146 total_bytes += $1 # accumulate
1147 }
1148END { print total_bytes " TOTAL" }
1149' > $tmp
1150
1151 # printf justifies du output
1152 cat $tmp | commas | xargs -n 2 printf '%15s %s\n'
1153 echo
1154
1155 #du -s --si /wedge/*/*/* ~/wedge/*/*/*
1156 #echo
1157}
1158
1159wedge-report() {
1160 # 4 levels deep shows the package
1161 if command -v tree > /dev/null; then
1162 tree -L 4 /wedge ~/wedge
1163 echo
1164 fi
1165
1166 wedge-sizes
1167
1168 local tmp=_tmp/wedge-manifest.txt
1169
1170 echo 'Biggest files'
1171 if ! find /wedge ~/wedge -type f -a -printf '%10s %P\n' > $tmp; then
1172 # busybox find doesn't have -printf
1173 echo 'find -printf failed'
1174 return
1175 fi
1176
1177 set +o errexit # ignore SIGPIPE
1178 sort -n --reverse $tmp | head -n 20 | commas
1179 set -o errexit
1180
1181 echo
1182
1183 # Show the most common file extensions
1184 #
1185 # I feel like we should be able to get rid of .a files? That's 92 MB, second
1186 # most common
1187 #
1188 # There are also duplicate .a files for Python -- should look at how distros
1189 # get rid of those
1190
1191 cat $tmp | python3 -c '
1192import os, sys, collections
1193
1194bytes = collections.Counter()
1195files = collections.Counter()
1196
1197for line in sys.stdin:
1198 size, path = line.split(None, 1)
1199 path = path.strip() # remove newline
1200 _, ext = os.path.splitext(path)
1201 size = int(size)
1202
1203 bytes[ext] += size
1204 files[ext] += 1
1205
1206#print(bytes)
1207#print(files)
1208
1209n = 20
1210
1211print("Most common file types")
1212for ext, count in files.most_common()[:n]:
1213 print("%10d %s" % (count, ext))
1214
1215print()
1216
1217print("Total bytes by file type")
1218for ext, total_bytes in bytes.most_common()[:n]:
1219 print("%10d %s" % (total_bytes, ext))
1220' | commas
1221}
1222
1223task-five "$@"