| 1 | #!/usr/bin/env bash
|
| 2 | #
|
| 3 | # Build the dev version of Oils on CPython.
|
| 4 | # This is in contrast to oils-for-unix C++ build
|
| 5 | #
|
| 6 | # Usage:
|
| 7 | # build/py.sh <function name>
|
| 8 |
|
| 9 | : ${LIB_OSH=stdlib/osh}
|
| 10 | source $LIB_OSH/bash-strict.sh
|
| 11 | source $LIB_OSH/task-five.sh
|
| 12 |
|
| 13 | REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
|
| 14 | source build/common.sh # log, $CLANGXX
|
| 15 |
|
| 16 | if test -z "${IN_NIX_SHELL:-}"; then
|
| 17 | source build/dev-shell.sh # to run 're2c'
|
| 18 | fi
|
| 19 |
|
| 20 | export PYTHONPATH='.:vendor/'
|
| 21 |
|
| 22 | ubuntu-deps() {
|
| 23 | ### Alias for backward compatility
|
| 24 | build/deps.sh install-ubuntu-packages
|
| 25 | }
|
| 26 |
|
| 27 | # This is what Python uses on OS X.
|
| 28 | #
|
| 29 | # https://www.thrysoee.dk/editline/
|
| 30 | install-libedit() {
|
| 31 | sudo apt install libedit-dev
|
| 32 | }
|
| 33 |
|
| 34 | libedit-flags() {
|
| 35 | pkg-config --libs --cflags libedit
|
| 36 | }
|
| 37 |
|
| 38 | install-py3() {
|
| 39 | pip3 install mypy
|
| 40 | }
|
| 41 |
|
| 42 | destroy-pip() {
|
| 43 | rm -r -f -v ~/.cache/pip ~/.local/lib/python2.7
|
| 44 | }
|
| 45 |
|
| 46 | # Needed for the release process, but not the dev process.
|
| 47 | # TODO: remove in favor of wedges in deps/
|
| 48 | release-ubuntu-deps() {
|
| 49 | # For the release to run test/report.R, you need r-base-core too.
|
| 50 | # cloc is used for line counts
|
| 51 | # valgrind/cachegrind for benchmarks
|
| 52 | sudo apt-get install r-base-core cloc valgrind
|
| 53 | }
|
| 54 |
|
| 55 | # 3/2021: For installing dplyr on Ubuntu Xenial 16.04 LTS, which has an old R version
|
| 56 | # Following these instructions
|
| 57 | # https://cloud.r-project.org/bin/linux/ubuntu/README.html
|
| 58 |
|
| 59 | # 5/2021: Upgraded to Ubuntu Bionic, which has R 3.4.4. So it looks like I no
|
| 60 | # longer need this.
|
| 61 | #
|
| 62 | # 2/2023: I need this again because R 3.4.4 is too old for dplyr.
|
| 63 | #
|
| 64 | # https://cloud.r-project.org/bin/linux/ubuntu/
|
| 65 |
|
| 66 | _install-new-r() {
|
| 67 | # update indices
|
| 68 | apt update -qq
|
| 69 |
|
| 70 | # install two helper packages we need
|
| 71 | apt install --no-install-recommends software-properties-common dirmngr
|
| 72 |
|
| 73 | # import the signing key (by Michael Rutter) for these repo
|
| 74 | apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9
|
| 75 |
|
| 76 | # add the R 4.0 repo from CRAN -- adjust 'focal' to 'groovy' or 'bionic' as needed
|
| 77 |
|
| 78 | local ubuntu_version
|
| 79 | ubuntu_version=$(lsb_release -cs)
|
| 80 | add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $ubuntu_version-cran40/"
|
| 81 |
|
| 82 | # Hm I had to run this manually and I got R 4.0
|
| 83 | # 2021-04: Hm this had to be run twice
|
| 84 | apt install --no-install-recommends r-base
|
| 85 | }
|
| 86 |
|
| 87 | install-new-r() {
|
| 88 | sudo $0 _install-new-r "$@"
|
| 89 | }
|
| 90 |
|
| 91 | const-mypy-gen() {
|
| 92 | local out=_devbuild/gen/id_kind_asdl.py
|
| 93 | frontend/consts_gen.py mypy > $out
|
| 94 | log " (frontend/consts_gen) -> $out"
|
| 95 |
|
| 96 | out=_devbuild/gen/id_kind.py
|
| 97 | frontend/consts_gen.py py-consts > $out
|
| 98 | log " (frontend/consts_gen) -> $out"
|
| 99 | }
|
| 100 |
|
| 101 | option-mypy-gen() {
|
| 102 | local out=_devbuild/gen/option_asdl.py
|
| 103 | frontend/option_gen.py mypy > $out
|
| 104 | log " (frontend/option_gen) -> $out"
|
| 105 | }
|
| 106 |
|
| 107 | flag-gen-mypy() {
|
| 108 | local out=_devbuild/gen/arg_types.py
|
| 109 | frontend/flag_gen.py mypy > $out
|
| 110 | log " (frontend/flag_gen) -> $out"
|
| 111 | }
|
| 112 |
|
| 113 | # Helper
|
| 114 | gen-asdl-py() {
|
| 115 | local asdl_path=$1 # e.g. frontend/syntax.asdl
|
| 116 |
|
| 117 | local name
|
| 118 | name=$(basename $asdl_path .asdl)
|
| 119 |
|
| 120 | local tmp=_tmp/${name}_asdl.py
|
| 121 | local out=_devbuild/gen/${name}_asdl.py
|
| 122 |
|
| 123 | # abbrev module is optional
|
| 124 | asdl/asdl_main.py mypy "$@" > $tmp
|
| 125 |
|
| 126 | # BUG: MUST BE DONE ATOMICALLY; otherwise the Python interpreter can
|
| 127 | # import an empty file!
|
| 128 | mv $tmp $out
|
| 129 |
|
| 130 | log "$asdl_path -> (asdl_main) -> $out"
|
| 131 | }
|
| 132 |
|
| 133 | asdl-metrics() {
|
| 134 | # sum types with the most variants:
|
| 135 | #
|
| 136 | # 20 expr
|
| 137 | # 22 command
|
| 138 | # 28 value
|
| 139 |
|
| 140 | for schema in */*.asdl; do
|
| 141 | asdl/asdl_main.py metrics $schema
|
| 142 | done | sort -n
|
| 143 | }
|
| 144 |
|
| 145 | asdl-closure() {
|
| 146 | local type=${1:-command}
|
| 147 |
|
| 148 | local out=_tmp/closure-$type.txt
|
| 149 |
|
| 150 | asdl/asdl_main.py closure frontend/syntax.asdl $type > $out
|
| 151 |
|
| 152 | tail $out
|
| 153 | echo
|
| 154 |
|
| 155 | # Should only appear ONCE, if we're accounting for shared variants
|
| 156 | if grep SingleQuoted $out; then
|
| 157 | echo 'FAIL: shared variant bug'
|
| 158 | return 1
|
| 159 | fi
|
| 160 |
|
| 161 | wc -l $out
|
| 162 | echo
|
| 163 |
|
| 164 | echo "Wrote $out"
|
| 165 | }
|
| 166 |
|
| 167 | py-codegen() {
|
| 168 | # note: filename must come first
|
| 169 | # hnode.asdl has REQUIRED fields so it's --py-init-N
|
| 170 | gen-asdl-py 'asdl/hnode.asdl' --no-pretty-print-methods --py-init-N
|
| 171 |
|
| 172 | gen-asdl-py 'frontend/types.asdl'
|
| 173 | # depends on syntax.asdl
|
| 174 | gen-asdl-py 'core/runtime.asdl'
|
| 175 | gen-asdl-py 'core/value.asdl'
|
| 176 | gen-asdl-py 'data_lang/htm8.asdl'
|
| 177 | gen-asdl-py 'data_lang/nil8.asdl'
|
| 178 | gen-asdl-py 'display/pretty.asdl'
|
| 179 | gen-asdl-py 'mycpp/mycpp.asdl'
|
| 180 |
|
| 181 | gen-asdl-py 'tools/find/find.asdl'
|
| 182 |
|
| 183 | const-mypy-gen # depends on bool_arg_type_e, generates Id_t
|
| 184 |
|
| 185 | # This does __import__ of syntax_abbrev.py, which depends on Id. We could
|
| 186 | # use the AST module later?
|
| 187 | gen-asdl-py 'frontend/syntax.asdl' \
|
| 188 | --abbrev-module='frontend.syntax_abbrev'
|
| 189 |
|
| 190 | option-mypy-gen
|
| 191 | flag-gen-mypy
|
| 192 |
|
| 193 | # Experiment
|
| 194 | gen-asdl-py 'yaks/yaks.asdl'
|
| 195 |
|
| 196 | # For tests
|
| 197 | gen-asdl-py 'mycpp/examples/expr.asdl'
|
| 198 | }
|
| 199 |
|
| 200 | py-asdl-examples() {
|
| 201 | # dependency of typed_demo
|
| 202 | gen-asdl-py 'asdl/examples/demo_lib.asdl'
|
| 203 | gen-asdl-py 'asdl/examples/typed_demo.asdl'
|
| 204 |
|
| 205 | gen-asdl-py 'asdl/examples/shared_variant.asdl'
|
| 206 | gen-asdl-py 'asdl/examples/typed_arith.asdl' \
|
| 207 | --abbrev-module='asdl.examples.typed_arith_abbrev'
|
| 208 | }
|
| 209 |
|
| 210 | oil-cpp() {
|
| 211 | ### STUB for backward compatibility
|
| 212 |
|
| 213 | build/cpp.sh all
|
| 214 | }
|
| 215 |
|
| 216 | py-ext() {
|
| 217 | ### Build a Python extension
|
| 218 |
|
| 219 | local name=$1
|
| 220 | local setup_script=$2
|
| 221 |
|
| 222 | log " ($setup_script) -> $name.so"
|
| 223 |
|
| 224 | local arch
|
| 225 | arch=$(uname -m)
|
| 226 |
|
| 227 | # send compiler output to stdout, which goes to a log
|
| 228 | $setup_script --quiet build_ext --inplace 2>&1
|
| 229 |
|
| 230 | #file $name.so
|
| 231 | }
|
| 232 |
|
| 233 | py-ext-test() {
|
| 234 | ### Run a test and log it
|
| 235 |
|
| 236 | # TODO: Fold this into some kind of Ninja test runner?
|
| 237 | # Or just rely on test/unit.sh all?
|
| 238 |
|
| 239 | local test_path=$1 # Or a function
|
| 240 | shift
|
| 241 |
|
| 242 | local log_path=_test/unit/$test_path.log
|
| 243 | mkdir -p $(dirname $log_path)
|
| 244 |
|
| 245 | set +o errexit
|
| 246 | $test_path "$@" >$log_path 2>&1
|
| 247 | local status=$?
|
| 248 | set -o errexit
|
| 249 |
|
| 250 | if test $status -eq 0; then
|
| 251 | log " OK $log_path"
|
| 252 | else
|
| 253 | echo
|
| 254 | cat $log_path
|
| 255 | echo
|
| 256 | die " FAIL $log_path"
|
| 257 | fi
|
| 258 | }
|
| 259 |
|
| 260 | pylibc() {
|
| 261 | rm -f libc.so
|
| 262 |
|
| 263 | py-ext libc pyext/setup_libc.py
|
| 264 |
|
| 265 | # Skip unit tests on Alpine for now
|
| 266 | # musl libc doesn't have extended globs
|
| 267 | if uname -a | grep -F Alpine; then
|
| 268 | return
|
| 269 | fi
|
| 270 |
|
| 271 | py-ext-test pyext/libc_test.py "$@"
|
| 272 | }
|
| 273 |
|
| 274 | fanos() {
|
| 275 | rm -f fanos.so
|
| 276 |
|
| 277 | py-ext fanos pyext/setup_fanos.py
|
| 278 | py-ext-test pyext/fanos_test.py "$@"
|
| 279 | }
|
| 280 |
|
| 281 | fastfunc() {
|
| 282 | rm -f fastfunc.so
|
| 283 |
|
| 284 | py-ext fastfunc pyext/setup_fastfunc.py
|
| 285 | py-ext-test pyext/fastfunc_test.py "$@"
|
| 286 | }
|
| 287 |
|
| 288 | #
|
| 289 | # For frontend/match.py
|
| 290 | #
|
| 291 |
|
| 292 | lexer-gen() { frontend/lexer_gen.py "$@"; }
|
| 293 |
|
| 294 | print-regex() { lexer-gen print-regex; }
|
| 295 | print-all() { lexer-gen print-all; }
|
| 296 |
|
| 297 | # Structure:
|
| 298 | #
|
| 299 | # _gen
|
| 300 | # frontend/
|
| 301 | # id.asdl_c.h
|
| 302 | # types.asdl_c.h
|
| 303 | # match.re2c.h
|
| 304 | # _build/
|
| 305 | # tmp/
|
| 306 | # frontend/
|
| 307 | # match.re2c.in
|
| 308 | # bin/
|
| 309 | # oils_for_unix_raw.mycpp.cc
|
| 310 |
|
| 311 | # re2c native.
|
| 312 | osh-lex-gen-native() {
|
| 313 | local in=$1
|
| 314 | local out=$2
|
| 315 | # Turn on all warnings and make them native.
|
| 316 | # The COMMENT state can match an empty string at the end of a line, e.g.
|
| 317 | # '#\n'. So we have to turn that warning off.
|
| 318 | re2c -W -Wno-match-empty-string -Werror -o $out $in
|
| 319 | }
|
| 320 |
|
| 321 | fastmatch() {
|
| 322 | local gen_dir=_gen/frontend
|
| 323 | mkdir -p _gen/_tmp $gen_dir
|
| 324 |
|
| 325 | # C version of frontend/types.asdl
|
| 326 | local out=$gen_dir/types.asdl_c.h
|
| 327 | asdl/asdl_main.py c frontend/types.asdl "$@" > $out
|
| 328 | log " (asdl_main c) -> $out"
|
| 329 |
|
| 330 | # C version of id_kind
|
| 331 | local out=$gen_dir/id_kind.asdl_c.h
|
| 332 | frontend/consts_gen.py c > $out
|
| 333 | log " (frontend/consts_gen c) -> $out"
|
| 334 |
|
| 335 | # Fast matcher
|
| 336 | local tmp=_gen/_tmp/match.re2c-input.h
|
| 337 | local out=_gen/frontend/match.re2c.h
|
| 338 | lexer-gen c > $tmp
|
| 339 | log " (lexer_gen) -> $tmp"
|
| 340 |
|
| 341 | osh-lex-gen-native $tmp $out
|
| 342 | log "$tmp -> (re2c) -> $out"
|
| 343 | }
|
| 344 |
|
| 345 | fastlex() {
|
| 346 | fastmatch
|
| 347 |
|
| 348 | # Why do we need this? It gets stale otherwise.
|
| 349 | rm -f fastlex.so
|
| 350 |
|
| 351 | py-ext fastlex pyext/setup_fastlex.py
|
| 352 | py-ext-test pyext/fastlex_test.py
|
| 353 | }
|
| 354 |
|
| 355 | line-input() {
|
| 356 | # Why do we need this? It gets stale otherwise.
|
| 357 | rm -f line_input.so
|
| 358 |
|
| 359 | py-ext line_input pyext/setup_line_input.py
|
| 360 | py-ext-test pyext/line_input_test.py
|
| 361 | }
|
| 362 |
|
| 363 | posix_() {
|
| 364 | rm -f posix_.so
|
| 365 |
|
| 366 | py-ext posix_ pyext/setup_posix.py
|
| 367 | py-ext-test pyext/posix_test.py
|
| 368 | }
|
| 369 |
|
| 370 | py-source() {
|
| 371 | ### Generate Python source code
|
| 372 |
|
| 373 | mkdir -p _tmp _devbuild/gen
|
| 374 |
|
| 375 | # need -r because Python 3 puts a __pycache__ here
|
| 376 | log 'Removing _devbuild/gen/*'
|
| 377 | rm -r -f _devbuild/gen/*
|
| 378 |
|
| 379 | # So modules are importable.
|
| 380 | touch _devbuild/__init__.py _devbuild/gen/__init__.py
|
| 381 |
|
| 382 | py-codegen # depends on Id
|
| 383 |
|
| 384 | # Only for testing.
|
| 385 | py-asdl-examples
|
| 386 |
|
| 387 | # Needed on Travis.
|
| 388 | ysh-grammar
|
| 389 | find-grammar
|
| 390 | demo-grammar # for mycpp/examples/pgen2_demo
|
| 391 | }
|
| 392 |
|
| 393 | # No fastlex, because we don't want to require re2c installation.
|
| 394 | py-extensions() {
|
| 395 | local minimal=${1:-}
|
| 396 |
|
| 397 | # Parallel build
|
| 398 | mkdir -p _tmp/pyext
|
| 399 |
|
| 400 | local -a tasks=(pylibc line-input posix_ fanos fastfunc)
|
| 401 | if test -z "$minimal"; then
|
| 402 | tasks+=(fastlex)
|
| 403 | fi
|
| 404 |
|
| 405 | local -A pid_map=()
|
| 406 | for task in "${tasks[@]}"; do
|
| 407 | local log="_tmp/pyext/$task.log"
|
| 408 | $task > $log &
|
| 409 | pid_map[$!]=$task
|
| 410 | log " PYEXT $log"
|
| 411 | done
|
| 412 | #log PIDS "${!pid_map[@]}"
|
| 413 |
|
| 414 | local failed=''
|
| 415 | for pid in "${!pid_map[@]}"; do
|
| 416 | #echo "WAIT $pid"
|
| 417 |
|
| 418 | # Funny dance to get exit code
|
| 419 | set +o errexit
|
| 420 | wait $pid
|
| 421 | status=$?
|
| 422 | set -o errexit
|
| 423 |
|
| 424 | if test $status -ne 0; then
|
| 425 | local task=${pid_map[$pid]}
|
| 426 | echo
|
| 427 | echo "*** Building '$task' failed: _tmp/pyext/$task.log:"
|
| 428 | echo
|
| 429 | cat "_tmp/pyext/$task.log"
|
| 430 | failed=T
|
| 431 | fi
|
| 432 | done
|
| 433 |
|
| 434 | if test -n "$failed"; then
|
| 435 | return 1
|
| 436 | fi
|
| 437 | }
|
| 438 |
|
| 439 | configure-for-dev() {
|
| 440 | # _OIL_DEV does two things:
|
| 441 | # -D GC_TIMING
|
| 442 | # Skip expensive "OVM" tarball stuff
|
| 443 | # Note: this causes a spurious LONG_BIT error in build/ovm-compile.sh
|
| 444 | # when detecting 'python-headers' for the oils-ref tarball
|
| 445 | _OIL_DEV=1 ./configure
|
| 446 | }
|
| 447 |
|
| 448 | minimal() {
|
| 449 | configure-for-dev
|
| 450 |
|
| 451 | build/stamp.sh write-git-commit
|
| 452 |
|
| 453 | py-source
|
| 454 | py-extensions MINIMAL
|
| 455 |
|
| 456 | cat <<EOF
|
| 457 |
|
| 458 | *****
|
| 459 | '$0 minimal' succeeded
|
| 460 |
|
| 461 | It allows you to run and modify Oils quickly, but the lexer will be slow and
|
| 462 | the help builtin won't work.
|
| 463 |
|
| 464 | '$0 all' requires re2c and libcmark.so. (Issue #513 is related, ask
|
| 465 | on #oil-dev)
|
| 466 | *****
|
| 467 | EOF
|
| 468 | }
|
| 469 |
|
| 470 | ysh-grammar() {
|
| 471 | mkdir -p _gen/ysh
|
| 472 | touch _gen/__init__.py _gen/ysh/__init__.py
|
| 473 |
|
| 474 | ysh/grammar_gen.py py ysh/grammar.pgen2 _devbuild/gen
|
| 475 | }
|
| 476 |
|
| 477 | find-grammar() {
|
| 478 | ysh/grammar_gen.py py tools/find/find.pgen2 _devbuild/gen
|
| 479 | }
|
| 480 |
|
| 481 | demo-grammar() {
|
| 482 | ysh/grammar_gen.py py mycpp/examples/arith.pgen2 _devbuild/gen
|
| 483 | }
|
| 484 |
|
| 485 | time-helper() {
|
| 486 | local out=${1:-_devbuild/bin/time-helper}
|
| 487 | local cflags=${2:-} # e.g. -fsanitize=address
|
| 488 |
|
| 489 | local in=benchmarks/time-helper.c
|
| 490 |
|
| 491 | mkdir -p $(dirname $out)
|
| 492 |
|
| 493 | cc -std=c99 -Wall $cflags -o $out $in
|
| 494 | log " CC $in"
|
| 495 | }
|
| 496 |
|
| 497 | all() {
|
| 498 | configure-for-dev
|
| 499 |
|
| 500 | rm -f *.so # 12/2019: to clear old symlinks, maybe get rid of
|
| 501 |
|
| 502 | build/stamp.sh write-git-commit
|
| 503 |
|
| 504 | py-source
|
| 505 | py-extensions
|
| 506 | time-helper
|
| 507 |
|
| 508 | # help topics and chapter links are extracted from doc/ref
|
| 509 | build/doc.sh all-ref
|
| 510 | }
|
| 511 |
|
| 512 | gitpod-minimal() {
|
| 513 | ubuntu-deps '-y' # skip prompt
|
| 514 | minimal
|
| 515 | test/spec.sh smoke
|
| 516 |
|
| 517 | set -x
|
| 518 | bin/osh -c 'echo hi'
|
| 519 | }
|
| 520 |
|
| 521 | show-cpython-patches() {
|
| 522 | # 2025-01: The patches for the "OVM" build
|
| 523 | #
|
| 524 | # After getting rid of build/oil-defs, this is significant, but maintainable.
|
| 525 | git diff 1d2b97384e14d65b1241f67fd995277f5508db28..HEAD Python-2.7.13/
|
| 526 | }
|
| 527 |
|
| 528 | name=$(basename $0)
|
| 529 | if test "$name" = 'py.sh'; then
|
| 530 | task-five "$@"
|
| 531 | fi
|