OILS / build / py.sh View on Github | oils.pub

513 lines, 283 significant
1#!/usr/bin/env bash
2#
3# Build the dev version of Oils on CPython.
4# This is in contrast to oils-for-unix C++ build
5#
6# Usage:
7# build/py.sh <function name>
8
9: ${LIB_OSH=stdlib/osh}
10source $LIB_OSH/bash-strict.sh
11source $LIB_OSH/task-five.sh
12
13REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
14source build/common.sh # log, $CLANGXX
15
16if test -z "${IN_NIX_SHELL:-}"; then
17 source build/dev-shell.sh # to run 're2c'
18fi
19
20export PYTHONPATH='.:vendor/'
21
22ubuntu-deps() {
23 ### Alias for backward compatility
24 build/deps.sh install-ubuntu-packages
25}
26
27# This is what Python uses on OS X.
28#
29# https://www.thrysoee.dk/editline/
30install-libedit() {
31 sudo apt install libedit-dev
32}
33
34libedit-flags() {
35 pkg-config --libs --cflags libedit
36}
37
38install-py3() {
39 pip3 install mypy
40}
41
42destroy-pip() {
43 rm -r -f -v ~/.cache/pip ~/.local/lib/python2.7
44}
45
46# Needed for the release process, but not the dev process.
47# TODO: remove in favor of wedges in deps/
48release-ubuntu-deps() {
49 # For the release to run test/report.R, you need r-base-core too.
50 # cloc is used for line counts
51 # valgrind/cachegrind for benchmarks
52 sudo apt-get install r-base-core cloc valgrind
53}
54
55# 3/2021: For installing dplyr on Ubuntu Xenial 16.04 LTS, which has an old R version
56# Following these instructions
57# https://cloud.r-project.org/bin/linux/ubuntu/README.html
58
59# 5/2021: Upgraded to Ubuntu Bionic, which has R 3.4.4. So it looks like I no
60# longer need this.
61#
62# 2/2023: I need this again because R 3.4.4 is too old for dplyr.
63#
64# https://cloud.r-project.org/bin/linux/ubuntu/
65
66_install-new-r() {
67 # update indices
68 apt update -qq
69
70 # install two helper packages we need
71 apt install --no-install-recommends software-properties-common dirmngr
72
73 # import the signing key (by Michael Rutter) for these repo
74 apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9
75
76 # add the R 4.0 repo from CRAN -- adjust 'focal' to 'groovy' or 'bionic' as needed
77
78 local ubuntu_version
79 ubuntu_version=$(lsb_release -cs)
80 add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $ubuntu_version-cran40/"
81
82 # Hm I had to run this manually and I got R 4.0
83 # 2021-04: Hm this had to be run twice
84 apt install --no-install-recommends r-base
85}
86
87install-new-r() {
88 sudo $0 _install-new-r "$@"
89}
90
91const-mypy-gen() {
92 local out=_devbuild/gen/id_kind_asdl.py
93 frontend/consts_gen.py mypy > $out
94 log " (frontend/consts_gen) -> $out"
95
96 out=_devbuild/gen/id_kind.py
97 frontend/consts_gen.py py-consts > $out
98 log " (frontend/consts_gen) -> $out"
99}
100
101option-mypy-gen() {
102 local out=_devbuild/gen/option_asdl.py
103 frontend/option_gen.py mypy > $out
104 log " (frontend/option_gen) -> $out"
105}
106
107flag-gen-mypy() {
108 local out=_devbuild/gen/arg_types.py
109 frontend/flag_gen.py mypy > $out
110 log " (frontend/flag_gen) -> $out"
111}
112
113# Helper
114gen-asdl-py() {
115 local asdl_path=$1 # e.g. frontend/syntax.asdl
116
117 local name
118 name=$(basename $asdl_path .asdl)
119
120 local tmp=_tmp/${name}_asdl.py
121 local out=_devbuild/gen/${name}_asdl.py
122
123 # abbrev module is optional
124 asdl/asdl_main.py mypy "$@" > $tmp
125
126 # BUG: MUST BE DONE ATOMICALLY; otherwise the Python interpreter can
127 # import an empty file!
128 mv $tmp $out
129
130 log "$asdl_path -> (asdl_main) -> $out"
131}
132
133asdl-metrics() {
134 # sum types with the most variants:
135 #
136 # 20 expr
137 # 22 command
138 # 28 value
139
140 for schema in */*.asdl; do
141 asdl/asdl_main.py metrics $schema
142 done | sort -n
143}
144
145asdl-command-t() {
146 asdl/asdl_main.py command_t frontend/syntax.asdl
147}
148
149py-codegen() {
150 # note: filename must come first
151 # hnode.asdl has REQUIRED fields so it's --py-init-N
152 gen-asdl-py 'asdl/hnode.asdl' --no-pretty-print-methods --py-init-N
153
154 gen-asdl-py 'frontend/types.asdl'
155 # depends on syntax.asdl
156 gen-asdl-py 'core/runtime.asdl'
157 gen-asdl-py 'core/value.asdl'
158 gen-asdl-py 'data_lang/htm8.asdl'
159 gen-asdl-py 'data_lang/nil8.asdl'
160 gen-asdl-py 'display/pretty.asdl'
161 gen-asdl-py 'mycpp/mycpp.asdl'
162
163 gen-asdl-py 'tools/find/find.asdl'
164
165 const-mypy-gen # depends on bool_arg_type_e, generates Id_t
166
167 # This does __import__ of syntax_abbrev.py, which depends on Id. We could
168 # use the AST module later?
169 gen-asdl-py 'frontend/syntax.asdl' \
170 --abbrev-module='frontend.syntax_abbrev'
171
172 option-mypy-gen
173 flag-gen-mypy
174
175 # Experiment
176 gen-asdl-py 'yaks/yaks.asdl'
177
178 # For tests
179 gen-asdl-py 'mycpp/examples/expr.asdl'
180}
181
182py-asdl-examples() {
183 # dependency of typed_demo
184 gen-asdl-py 'asdl/examples/demo_lib.asdl'
185 gen-asdl-py 'asdl/examples/typed_demo.asdl'
186
187 gen-asdl-py 'asdl/examples/shared_variant.asdl'
188 gen-asdl-py 'asdl/examples/typed_arith.asdl' \
189 --abbrev-module='asdl.examples.typed_arith_abbrev'
190}
191
192oil-cpp() {
193 ### STUB for backward compatibility
194
195 build/cpp.sh all
196}
197
198py-ext() {
199 ### Build a Python extension
200
201 local name=$1
202 local setup_script=$2
203
204 log " ($setup_script) -> $name.so"
205
206 local arch
207 arch=$(uname -m)
208
209 # send compiler output to stdout, which goes to a log
210 $setup_script --quiet build_ext --inplace 2>&1
211
212 #file $name.so
213}
214
215py-ext-test() {
216 ### Run a test and log it
217
218 # TODO: Fold this into some kind of Ninja test runner?
219 # Or just rely on test/unit.sh all?
220
221 local test_path=$1 # Or a function
222 shift
223
224 local log_path=_test/unit/$test_path.log
225 mkdir -p $(dirname $log_path)
226
227 set +o errexit
228 $test_path "$@" >$log_path 2>&1
229 local status=$?
230 set -o errexit
231
232 if test $status -eq 0; then
233 log " OK $log_path"
234 else
235 echo
236 cat $log_path
237 echo
238 die " FAIL $log_path"
239 fi
240}
241
242pylibc() {
243 rm -f libc.so
244
245 py-ext libc pyext/setup_libc.py
246
247 # Skip unit tests on Alpine for now
248 # musl libc doesn't have extended globs
249 if uname -a | grep -F Alpine; then
250 return
251 fi
252
253 py-ext-test pyext/libc_test.py "$@"
254}
255
256fanos() {
257 rm -f fanos.so
258
259 py-ext fanos pyext/setup_fanos.py
260 py-ext-test pyext/fanos_test.py "$@"
261}
262
263fastfunc() {
264 rm -f fastfunc.so
265
266 py-ext fastfunc pyext/setup_fastfunc.py
267 py-ext-test pyext/fastfunc_test.py "$@"
268}
269
270#
271# For frontend/match.py
272#
273
274lexer-gen() { frontend/lexer_gen.py "$@"; }
275
276print-regex() { lexer-gen print-regex; }
277print-all() { lexer-gen print-all; }
278
279# Structure:
280#
281# _gen
282# frontend/
283# id.asdl_c.h
284# types.asdl_c.h
285# match.re2c.h
286# _build/
287# tmp/
288# frontend/
289# match.re2c.in
290# bin/
291# oils_for_unix_raw.mycpp.cc
292
293# re2c native.
294osh-lex-gen-native() {
295 local in=$1
296 local out=$2
297 # Turn on all warnings and make them native.
298 # The COMMENT state can match an empty string at the end of a line, e.g.
299 # '#\n'. So we have to turn that warning off.
300 re2c -W -Wno-match-empty-string -Werror -o $out $in
301}
302
303fastmatch() {
304 local gen_dir=_gen/frontend
305 mkdir -p _gen/_tmp $gen_dir
306
307 # C version of frontend/types.asdl
308 local out=$gen_dir/types.asdl_c.h
309 asdl/asdl_main.py c frontend/types.asdl "$@" > $out
310 log " (asdl_main c) -> $out"
311
312 # C version of id_kind
313 local out=$gen_dir/id_kind.asdl_c.h
314 frontend/consts_gen.py c > $out
315 log " (frontend/consts_gen c) -> $out"
316
317 # Fast matcher
318 local tmp=_gen/_tmp/match.re2c-input.h
319 local out=_gen/frontend/match.re2c.h
320 lexer-gen c > $tmp
321 log " (lexer_gen) -> $tmp"
322
323 osh-lex-gen-native $tmp $out
324 log "$tmp -> (re2c) -> $out"
325}
326
327fastlex() {
328 fastmatch
329
330 # Why do we need this? It gets stale otherwise.
331 rm -f fastlex.so
332
333 py-ext fastlex pyext/setup_fastlex.py
334 py-ext-test pyext/fastlex_test.py
335}
336
337line-input() {
338 # Why do we need this? It gets stale otherwise.
339 rm -f line_input.so
340
341 py-ext line_input pyext/setup_line_input.py
342 py-ext-test pyext/line_input_test.py
343}
344
345posix_() {
346 rm -f posix_.so
347
348 py-ext posix_ pyext/setup_posix.py
349 py-ext-test pyext/posix_test.py
350}
351
352py-source() {
353 ### Generate Python source code
354
355 mkdir -p _tmp _devbuild/gen
356
357 # need -r because Python 3 puts a __pycache__ here
358 log 'Removing _devbuild/gen/*'
359 rm -r -f _devbuild/gen/*
360
361 # So modules are importable.
362 touch _devbuild/__init__.py _devbuild/gen/__init__.py
363
364 py-codegen # depends on Id
365
366 # Only for testing.
367 py-asdl-examples
368
369 # Needed on Travis.
370 ysh-grammar
371 find-grammar
372 demo-grammar # for mycpp/examples/pgen2_demo
373}
374
375# No fastlex, because we don't want to require re2c installation.
376py-extensions() {
377 local minimal=${1:-}
378
379 # Parallel build
380 mkdir -p _tmp/pyext
381
382 local -a tasks=(pylibc line-input posix_ fanos fastfunc)
383 if test -z "$minimal"; then
384 tasks+=(fastlex)
385 fi
386
387 local -A pid_map=()
388 for task in "${tasks[@]}"; do
389 local log="_tmp/pyext/$task.log"
390 $task > $log &
391 pid_map[$!]=$task
392 log " PYEXT $log"
393 done
394 #log PIDS "${!pid_map[@]}"
395
396 local failed=''
397 for pid in "${!pid_map[@]}"; do
398 #echo "WAIT $pid"
399
400 # Funny dance to get exit code
401 set +o errexit
402 wait $pid
403 status=$?
404 set -o errexit
405
406 if test $status -ne 0; then
407 local task=${pid_map[$pid]}
408 echo
409 echo "*** Building '$task' failed: _tmp/pyext/$task.log:"
410 echo
411 cat "_tmp/pyext/$task.log"
412 failed=T
413 fi
414 done
415
416 if test -n "$failed"; then
417 return 1
418 fi
419}
420
421configure-for-dev() {
422 # _OIL_DEV does two things:
423 # -D GC_TIMING
424 # Skip expensive "OVM" tarball stuff
425 # Note: this causes a spurious LONG_BIT error in build/ovm-compile.sh
426 # when detecting 'python-headers' for the oils-ref tarball
427 _OIL_DEV=1 ./configure
428}
429
430minimal() {
431 configure-for-dev
432
433 build/stamp.sh write-git-commit
434
435 py-source
436 py-extensions MINIMAL
437
438 cat <<EOF
439
440*****
441'$0 minimal' succeeded
442
443 It allows you to run and modify Oils quickly, but the lexer will be slow and
444 the help builtin won't work.
445
446'$0 all' requires re2c and libcmark.so. (Issue #513 is related, ask
447on #oil-dev)
448*****
449EOF
450}
451
452ysh-grammar() {
453 mkdir -p _gen/ysh
454 touch _gen/__init__.py _gen/ysh/__init__.py
455
456 ysh/grammar_gen.py py ysh/grammar.pgen2 _devbuild/gen
457}
458
459find-grammar() {
460 ysh/grammar_gen.py py tools/find/find.pgen2 _devbuild/gen
461}
462
463demo-grammar() {
464 ysh/grammar_gen.py py mycpp/examples/arith.pgen2 _devbuild/gen
465}
466
467time-helper() {
468 local out=${1:-_devbuild/bin/time-helper}
469 local cflags=${2:-} # e.g. -fsanitize=address
470
471 local in=benchmarks/time-helper.c
472
473 mkdir -p $(dirname $out)
474
475 cc -std=c99 -Wall $cflags -o $out $in
476 log " CC $in"
477}
478
479all() {
480 configure-for-dev
481
482 rm -f *.so # 12/2019: to clear old symlinks, maybe get rid of
483
484 build/stamp.sh write-git-commit
485
486 py-source
487 py-extensions
488 time-helper
489
490 # help topics and chapter links are extracted from doc/ref
491 build/doc.sh all-ref
492}
493
494gitpod-minimal() {
495 ubuntu-deps '-y' # skip prompt
496 minimal
497 test/spec.sh smoke
498
499 set -x
500 bin/osh -c 'echo hi'
501}
502
503show-cpython-patches() {
504 # 2025-01: The patches for the "OVM" build
505 #
506 # After getting rid of build/oil-defs, this is significant, but maintainable.
507 git diff 1d2b97384e14d65b1241f67fd995277f5508db28..HEAD Python-2.7.13/
508}
509
510name=$(basename $0)
511if test "$name" = 'py.sh'; then
512 task-five "$@"
513fi