1 | #!/usr/bin/env bash
|
2 | #
|
3 | # Build the dev version of Oil on CPython.
|
4 | # This is in contrast to oils-for-unix and the oil.ovm build.
|
5 | #
|
6 | # Usage:
|
7 | # build/py.sh <function name>
|
8 |
|
9 | : ${LIB_OSH=stdlib/osh}
|
10 | source $LIB_OSH/bash-strict.sh
|
11 | source $LIB_OSH/task-five.sh
|
12 |
|
13 | REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
|
14 | source build/common.sh # log, $CLANGXX
|
15 |
|
16 | if test -z "${IN_NIX_SHELL:-}"; then
|
17 | source build/dev-shell.sh # to run 're2c'
|
18 | fi
|
19 |
|
20 | export PYTHONPATH='.:vendor/'
|
21 |
|
22 | ubuntu-deps() {
|
23 | ### Alias for backward compatility
|
24 | build/deps.sh install-ubuntu-packages
|
25 | }
|
26 |
|
27 | # This is what Python uses on OS X.
|
28 | #
|
29 | # https://www.thrysoee.dk/editline/
|
30 | install-libedit() {
|
31 | sudo apt install libedit-dev
|
32 | }
|
33 |
|
34 | libedit-flags() {
|
35 | pkg-config --libs --cflags libedit
|
36 | }
|
37 |
|
38 | install-py3() {
|
39 | pip3 install mypy
|
40 | }
|
41 |
|
42 | destroy-pip() {
|
43 | rm -r -f -v ~/.cache/pip ~/.local/lib/python2.7
|
44 | }
|
45 |
|
46 | # Needed for the release process, but not the dev process.
|
47 | # TODO: remove in favor of wedges in deps/
|
48 | release-ubuntu-deps() {
|
49 | # For the release to run test/report.R, you need r-base-core too.
|
50 | # cloc is used for line counts
|
51 | # valgrind/cachegrind for benchmarks
|
52 | sudo apt-get install r-base-core cloc valgrind
|
53 | }
|
54 |
|
55 | # 3/2021: For installing dplyr on Ubuntu Xenial 16.04 LTS, which has an old R version
|
56 | # Following these instructions
|
57 | # https://cloud.r-project.org/bin/linux/ubuntu/README.html
|
58 |
|
59 | # 5/2021: Upgraded to Ubuntu Bionic, which has R 3.4.4. So it looks like I no
|
60 | # longer need this.
|
61 | #
|
62 | # 2/2023: I need this again because R 3.4.4 is too old for dplyr.
|
63 | #
|
64 | # https://cloud.r-project.org/bin/linux/ubuntu/
|
65 |
|
66 | _install-new-r() {
|
67 | # update indices
|
68 | apt update -qq
|
69 |
|
70 | # install two helper packages we need
|
71 | apt install --no-install-recommends software-properties-common dirmngr
|
72 |
|
73 | # import the signing key (by Michael Rutter) for these repo
|
74 | apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9
|
75 |
|
76 | # add the R 4.0 repo from CRAN -- adjust 'focal' to 'groovy' or 'bionic' as needed
|
77 |
|
78 | local ubuntu_version
|
79 | ubuntu_version=$(lsb_release -cs)
|
80 | add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $ubuntu_version-cran40/"
|
81 |
|
82 | # Hm I had to run this manually and I got R 4.0
|
83 | # 2021-04: Hm this had to be run twice
|
84 | apt install --no-install-recommends r-base
|
85 | }
|
86 |
|
87 | install-new-r() {
|
88 | sudo $0 _install-new-r "$@"
|
89 | }
|
90 |
|
91 | const-mypy-gen() {
|
92 | local out=_devbuild/gen/id_kind_asdl.py
|
93 | frontend/consts_gen.py mypy > $out
|
94 | log " (frontend/consts_gen) -> $out"
|
95 |
|
96 | out=_devbuild/gen/id_kind.py
|
97 | frontend/consts_gen.py py-consts > $out
|
98 | log " (frontend/consts_gen) -> $out"
|
99 | }
|
100 |
|
101 | option-mypy-gen() {
|
102 | local out=_devbuild/gen/option_asdl.py
|
103 | frontend/option_gen.py mypy > $out
|
104 | log " (frontend/option_gen) -> $out"
|
105 | }
|
106 |
|
107 | flag-gen-mypy() {
|
108 | local out=_devbuild/gen/arg_types.py
|
109 | frontend/flag_gen.py mypy > $out
|
110 | log " (frontend/flag_gen) -> $out"
|
111 | }
|
112 |
|
113 | # Helper
|
114 | gen-asdl-py() {
|
115 | local asdl_path=$1 # e.g. frontend/syntax.asdl
|
116 |
|
117 | local name
|
118 | name=$(basename $asdl_path .asdl)
|
119 |
|
120 | local tmp=_tmp/${name}_asdl.py
|
121 | local out=_devbuild/gen/${name}_asdl.py
|
122 |
|
123 | # abbrev module is optional
|
124 | asdl/asdl_main.py mypy "$@" > $tmp
|
125 |
|
126 | # BUG: MUST BE DONE ATOMICALLY; otherwise the Python interpreter can
|
127 | # import an empty file!
|
128 | mv $tmp $out
|
129 |
|
130 | log "$asdl_path -> (asdl_main) -> $out"
|
131 | }
|
132 |
|
133 | py-codegen() {
|
134 | # note: filename must come first
|
135 | # hnode.asdl has REQUIRED fields so it's --py-init-N
|
136 | gen-asdl-py 'asdl/hnode.asdl' --no-pretty-print-methods --py-init-N
|
137 |
|
138 | gen-asdl-py 'frontend/types.asdl'
|
139 | # depends on syntax.asdl
|
140 | gen-asdl-py 'core/runtime.asdl'
|
141 | gen-asdl-py 'core/value.asdl'
|
142 | gen-asdl-py 'data_lang/htm8.asdl'
|
143 | gen-asdl-py 'data_lang/nil8.asdl'
|
144 | gen-asdl-py 'display/pretty.asdl'
|
145 | gen-asdl-py 'mycpp/mycpp.asdl'
|
146 |
|
147 | gen-asdl-py 'tools/find/find.asdl'
|
148 |
|
149 | const-mypy-gen # depends on bool_arg_type_e, generates Id_t
|
150 |
|
151 | # This does __import__ of syntax_abbrev.py, which depends on Id. We could
|
152 | # use the AST module later?
|
153 | gen-asdl-py 'frontend/syntax.asdl' \
|
154 | --abbrev-module='frontend.syntax_abbrev'
|
155 |
|
156 | option-mypy-gen
|
157 | flag-gen-mypy
|
158 |
|
159 | # Experiment
|
160 | gen-asdl-py 'yaks/yaks.asdl'
|
161 |
|
162 | # For tests
|
163 | gen-asdl-py 'mycpp/examples/expr.asdl'
|
164 | }
|
165 |
|
166 | py-asdl-examples() {
|
167 | # dependency of typed_demo
|
168 | gen-asdl-py 'asdl/examples/demo_lib.asdl'
|
169 | gen-asdl-py 'asdl/examples/typed_demo.asdl'
|
170 |
|
171 | gen-asdl-py 'asdl/examples/shared_variant.asdl'
|
172 | gen-asdl-py 'asdl/examples/typed_arith.asdl' \
|
173 | --abbrev-module='asdl.examples.typed_arith_abbrev'
|
174 | }
|
175 |
|
176 | oil-cpp() {
|
177 | ### STUB for backward compatibility
|
178 |
|
179 | build/cpp.sh all
|
180 | }
|
181 |
|
182 | py-ext() {
|
183 | ### Build a Python extension
|
184 |
|
185 | local name=$1
|
186 | local setup_script=$2
|
187 |
|
188 | log " ($setup_script) -> $name.so"
|
189 |
|
190 | local arch
|
191 | arch=$(uname -m)
|
192 |
|
193 | # global opts come first
|
194 | $setup_script --quiet build_ext --inplace
|
195 |
|
196 | #file $name.so
|
197 | }
|
198 |
|
199 | py-ext-test() {
|
200 | ### Run a test and log it
|
201 |
|
202 | # TODO: Fold this into some kind of Ninja test runner?
|
203 | # Or just rely on test/unit.sh all?
|
204 |
|
205 | local test_path=$1 # Or a function
|
206 | shift
|
207 |
|
208 | local log_path=_test/unit/$test_path.log
|
209 | mkdir -p $(dirname $log_path)
|
210 |
|
211 | set +o errexit
|
212 | $test_path "$@" >$log_path 2>&1
|
213 | local status=$?
|
214 | set -o errexit
|
215 |
|
216 | if test $status -eq 0; then
|
217 | log "OK $log_path"
|
218 | else
|
219 | echo
|
220 | cat $log_path
|
221 | echo
|
222 | die "FAIL $log_path"
|
223 | fi
|
224 | }
|
225 |
|
226 | pylibc() {
|
227 | rm -f libc.so
|
228 |
|
229 | py-ext libc pyext/setup_libc.py
|
230 |
|
231 | # Skip unit tests on Alpine for now
|
232 | # musl libc doesn't have extended globs
|
233 | if uname -a | grep -F Alpine; then
|
234 | return
|
235 | fi
|
236 |
|
237 | py-ext-test pyext/libc_test.py "$@"
|
238 | }
|
239 |
|
240 | fanos() {
|
241 | rm -f fanos.so
|
242 |
|
243 | py-ext fanos pyext/setup_fanos.py
|
244 | py-ext-test pyext/fanos_test.py "$@"
|
245 | }
|
246 |
|
247 | fastfunc() {
|
248 | rm -f fastfunc.so
|
249 |
|
250 | py-ext fastfunc pyext/setup_fastfunc.py
|
251 | py-ext-test pyext/fastfunc_test.py "$@"
|
252 | }
|
253 |
|
254 | #
|
255 | # For frontend/match.py
|
256 | #
|
257 |
|
258 | lexer-gen() { frontend/lexer_gen.py "$@"; }
|
259 |
|
260 | print-regex() { lexer-gen print-regex; }
|
261 | print-all() { lexer-gen print-all; }
|
262 |
|
263 | # Structure:
|
264 | #
|
265 | # _gen
|
266 | # frontend/
|
267 | # id.asdl_c.h
|
268 | # types.asdl_c.h
|
269 | # match.re2c.h
|
270 | # _build/
|
271 | # tmp/
|
272 | # frontend/
|
273 | # match.re2c.in
|
274 | # bin/
|
275 | # oils_for_unix_raw.mycpp.cc
|
276 |
|
277 | # re2c native.
|
278 | osh-lex-gen-native() {
|
279 | local in=$1
|
280 | local out=$2
|
281 | # Turn on all warnings and make them native.
|
282 | # The COMMENT state can match an empty string at the end of a line, e.g.
|
283 | # '#\n'. So we have to turn that warning off.
|
284 | re2c -W -Wno-match-empty-string -Werror -o $out $in
|
285 | }
|
286 |
|
287 | fastmatch() {
|
288 | local gen_dir=_gen/frontend
|
289 | mkdir -p _gen/_tmp $gen_dir
|
290 |
|
291 | # C version of frontend/types.asdl
|
292 | local out=$gen_dir/types.asdl_c.h
|
293 | asdl/asdl_main.py c frontend/types.asdl "$@" > $out
|
294 | log " (asdl_main c) -> $out"
|
295 |
|
296 | # C version of id_kind
|
297 | local out=$gen_dir/id_kind.asdl_c.h
|
298 | frontend/consts_gen.py c > $out
|
299 | log " (frontend/consts_gen c) -> $out"
|
300 |
|
301 | # Fast matcher
|
302 | local tmp=_gen/_tmp/match.re2c-input.h
|
303 | local out=_gen/frontend/match.re2c.h
|
304 | lexer-gen c > $tmp
|
305 | log " (lexer_gen) -> $tmp"
|
306 |
|
307 | osh-lex-gen-native $tmp $out
|
308 | log "$tmp -> (re2c) -> $out"
|
309 | }
|
310 |
|
311 | fastlex() {
|
312 | fastmatch
|
313 |
|
314 | # Why do we need this? It gets stale otherwise.
|
315 | rm -f fastlex.so
|
316 |
|
317 | py-ext fastlex pyext/setup_fastlex.py
|
318 | py-ext-test pyext/fastlex_test.py
|
319 | }
|
320 |
|
321 | line-input() {
|
322 | # Why do we need this? It gets stale otherwise.
|
323 | rm -f line_input.so
|
324 |
|
325 | py-ext line_input pyext/setup_line_input.py
|
326 | py-ext-test pyext/line_input_test.py
|
327 | }
|
328 |
|
329 | posix_() {
|
330 | rm -f posix_.so
|
331 |
|
332 | py-ext posix_ pyext/setup_posix.py
|
333 | py-ext-test pyext/posix_test.py
|
334 | }
|
335 |
|
336 | py-source() {
|
337 | ### Generate Python source code
|
338 |
|
339 | mkdir -p _tmp _devbuild/gen
|
340 |
|
341 | # need -r because Python 3 puts a __pycache__ here
|
342 | log 'Removing _devbuild/gen/*'
|
343 | rm -r -f _devbuild/gen/*
|
344 |
|
345 | # So modules are importable.
|
346 | touch _devbuild/__init__.py _devbuild/gen/__init__.py
|
347 |
|
348 | py-codegen # depends on Id
|
349 |
|
350 | # Only for testing.
|
351 | py-asdl-examples
|
352 |
|
353 | # Needed on Travis.
|
354 | ysh-grammar
|
355 | find-grammar
|
356 | demo-grammar # for mycpp/examples/pgen2_demo
|
357 | }
|
358 |
|
359 | # No fastlex, because we don't want to require re2c installation.
|
360 | py-extensions() {
|
361 | pylibc
|
362 | line-input
|
363 | posix_
|
364 | fanos
|
365 | fastfunc
|
366 | }
|
367 |
|
368 | configure-for-dev() {
|
369 | # _OIL_DEV does two things:
|
370 | # -D GC_TIMING
|
371 | # Skip expensive "OVM" tarball stuff
|
372 | _OIL_DEV=1 ./configure
|
373 | }
|
374 |
|
375 | minimal() {
|
376 | configure-for-dev
|
377 |
|
378 | build/stamp.sh write-git-commit
|
379 |
|
380 | py-source
|
381 | py-extensions
|
382 |
|
383 | cat <<EOF
|
384 |
|
385 | *****
|
386 | '$0 minimal' succeeded
|
387 |
|
388 | It allows you to run and modify Oil quickly, but the lexer will be slow and
|
389 | the help builtin won't work.
|
390 |
|
391 | '$0 all' requires re2c and libcmark.so. (Issue #513 is related, ask
|
392 | on #oil-dev)
|
393 | *****
|
394 | EOF
|
395 | }
|
396 |
|
397 | ysh-grammar() {
|
398 | mkdir -p _gen/ysh
|
399 | touch _gen/__init__.py _gen/ysh/__init__.py
|
400 |
|
401 | ysh/grammar_gen.py py ysh/grammar.pgen2 _devbuild/gen
|
402 | }
|
403 |
|
404 | find-grammar() {
|
405 | ysh/grammar_gen.py py tools/find/find.pgen2 _devbuild/gen
|
406 | }
|
407 |
|
408 | demo-grammar() {
|
409 | ysh/grammar_gen.py py mycpp/examples/arith.pgen2 _devbuild/gen
|
410 | }
|
411 |
|
412 | time-helper() {
|
413 | local out=${1:-_devbuild/bin/time-helper}
|
414 | local in=benchmarks/time-helper.c
|
415 |
|
416 | mkdir -p $(dirname $out)
|
417 |
|
418 | cc -std=c99 -Wall -o $out $in
|
419 | log " CC $in"
|
420 | }
|
421 |
|
422 | all() {
|
423 | configure-for-dev
|
424 |
|
425 | rm -f *.so # 12/2019: to clear old symlinks, maybe get rid of
|
426 |
|
427 | build/stamp.sh write-git-commit
|
428 |
|
429 | py-source
|
430 | py-extensions # no re2c
|
431 |
|
432 | # requires re2c: deps/from-tar.sh layer-re2c
|
433 | fastlex
|
434 | time-helper
|
435 |
|
436 | # help topics and chapter links are extracted from doc/ref
|
437 | build/doc.sh all-ref
|
438 | }
|
439 |
|
440 | gitpod-minimal() {
|
441 | ubuntu-deps '-y' # skip prompt
|
442 | minimal
|
443 | test/spec.sh smoke
|
444 |
|
445 | set -x
|
446 | bin/osh -c 'echo hi'
|
447 | }
|
448 |
|
449 | show-cpython-patches() {
|
450 | # 2025-01: The patches for the "OVM" build
|
451 | #
|
452 | # After getting rid of build/oil-defs, this is significant, but maintainable.
|
453 | git diff 1d2b97384e14d65b1241f67fd995277f5508db28..HEAD Python-2.7.13/
|
454 | }
|
455 |
|
456 | if test $(basename $0) = 'py.sh'; then
|
457 | task-five "$@"
|
458 | fi
|