OILS / build / dynamic-deps.sh View on Github | oils.pub

318 lines, 136 significant
1#!/usr/bin/env bash
2#
3# Calculate and filter deps of Python apps.
4#
5# Usage:
6# build/dynamic-deps.sh <function name>
7
8set -o nounset
9set -o pipefail
10set -o errexit
11
12source build/py2.sh # python2 needed below for dynamic_deps.py
13
14# Two fixes:
15# - cd -P: Resolve symlinks in the current working directory. This is needed
16# to make `grep $REPO_ROOT...` in `repo-filter (build/dynamic-deps.sh)` work.
17# Later, `repo-filter` and related parts may be rewritten using AWK to handle
18# it better.
19# - ${BASH_SOURCE[0]} rather than $0: because this file is sourced by
20# ./NINJA-config.sh, in which case $0 has one less /
21REPO_ROOT=$(cd -P "$(dirname ${BASH_SOURCE[0]})/.."; pwd)
22
23readonly PY_PATH='.:vendor/'
24
25# Temporary
26readonly DIR=_build/NINJA
27
28make-egrep() {
29 # match chars until # or space, and line must be non-empty
30 gawk '
31 match($0, /([^# ]*)/, m) {
32 contents = m[0]
33 if (contents) { # skip empty lines
34 print(contents)
35 }
36 }
37 '
38}
39
40write-filters() {
41 ### Write filename filters in the egrep -f format
42
43 # For ./NINJA-config.sh to use.
44 # This style lets us add comments.
45
46 # For asdl.asdl_main and other tools
47 make-egrep >build/default.deps-filter.txt <<'EOF'
48__init__.py
49typing.py # vendor/typing.py isn't imported normally
50EOF
51
52 #
53 # DEFAULT typecheck and translate filter
54 #
55 make-egrep >build/default.typecheck-filter.txt <<'EOF'
56__init__.py
57typing.py
58
59# OrderedDict is polymorphic
60pylib/collections_.py
61
62# lots of polymorphic stuff etc.
63mycpp/mylib.py
64EOF
65
66 make-egrep >build/default.translate-filter.txt <<'EOF'
67# generated code shouldn't be translated
68_devbuild/
69_gen/
70
71asdl/py.* # pybase.py ported by hand to C++
72pylib/py.*
73
74mycpp/iolib.py # Implemented in gc_iolib.{h,cC}
75mycpp/mops.py # Implemented in gc_mops.{h,cC}
76EOF
77
78 #
79 # OILS typecheck and translate filter
80 #
81 make-egrep >bin/oils_for_unix.typecheck-filter.txt <<'EOF'
82__init__.py
83typing.py
84
85# OrderedDict is polymorphic
86pylib/collections_.py
87
88# lots of polymorphic stuff etc.
89mycpp/mylib.py
90
91# TODO: move or remove these
92tools/deps.py
93tools/readlink.py
94EOF
95
96 # On top of the typecheck filter, exclude these from translation. They are
97 # not inputs to mycpp.
98
99 make-egrep >bin/oils_for_unix.translate-filter.txt <<'EOF'
100# generated code shouldn't be translated
101_devbuild/
102_gen/
103
104# definitions that are used by */*_gen.py
105.*_def\.py
106.*_spec\.py
107
108asdl/py.* # pybase.py ported by hand to C++
109
110core/py.* # pyos.py, pyutil.py ported by hand to C++
111core/optview\.py # core/optview_gen.py
112
113data_lang/py.* # pyj8.py
114
115frontend/py.*\.py # py_readline.py ported by hand to C++
116frontend/consts.py # frontend/consts_gen.py
117frontend/match.py # frontend/lexer_gen.py
118
119mycpp/iolib.py # Implemented in gc_iolib.{h,cC}
120mycpp/mops.py # Implemented in gc_mops.{h,cC}
121
122pgen2/grammar.py # These files are re-done in C++
123pgen2/pnode.py
124pgen2/token.py
125
126# should be py_path_stat.py, because it's ported by hand to C++
127pylib/path_stat.py
128
129# should be py_bool_stat.py, because it's ported by hand to C++
130osh/bool_stat.py
131EOF
132
133 wc -l */*.*-filter.txt
134}
135
136repo-filter() {
137 ### Select files from the dynamic_deps.py output
138
139 # select what's in the repo; eliminating stdlib stuff
140 # eliminate _cache for mycpp running under Python-3.10
141 #
142 # 2025-10 fix: trailing slash in $REPO_ROOT/ matters, because otherwise
143 # __future__.py in oils-for-unix/oils.DEPS is included, when we want it excluded
144 grep -F -v "$REPO_ROOT/_cache" | grep -F "$REPO_ROOT/" | awk '{ print $2 }'
145}
146
147exclude-files() {
148 local filter_file=$1
149 grep -E -v -f $filter_file
150}
151
152mysort() {
153 LC_ALL=C sort
154}
155
156# Code generators
157list-gen() {
158 ls */*_gen.py
159}
160
161#
162# Invocations of dynamic_deps
163#
164
165py2-manifest() {
166 local py_module=$1
167 local dir=$2
168 PYTHONPATH=$PY_PATH /usr/bin/env python2 \
169 build/dynamic_deps.py py-manifest $py_module \
170 > $dir/all-pairs.txt
171}
172
173py3-manifest() {
174 local dir=$1
175 python3 build/dynamic_deps.py py-manifest $module > $dir/all-pairs.txt
176}
177
178py-tool() {
179 local py_module=$1
180
181 local dir=$DIR/$py_module
182 mkdir -p $dir
183
184 py2-manifest $py_module $dir
185
186 cat $dir/all-pairs.txt | repo-filter | exclude-files build/default.deps-filter.txt | mysort \
187 > $dir/deps.txt
188
189 echo "DEPS $dir/deps.txt"
190}
191
192typecheck-translate() {
193 local py_module=$1
194 local typecheck_filter=${2:-}
195 local translate_filter=${3:-}
196
197 local py_rel_path=${py_module//'.'/'/'}
198 if test -z "$typecheck_filter"; then
199 local custom="$py_rel_path.typecheck-filter.txt"
200 if test -f "$custom"; then
201 typecheck_filter=$custom
202 else
203 typecheck_filter=build/default.typecheck-filter.txt
204 fi
205 fi
206
207 if test -z "$translate_filter"; then
208 local custom="$py_rel_path.translate-filter.txt"
209 if test -f "$custom"; then
210 translate_filter=$custom
211 else
212 translate_filter=build/default.translate-filter.txt
213 fi
214 fi
215
216 if false; then
217 echo " | PY $py_module"
218 echo " | TYPECHECK $typecheck_filter"
219 echo " | TRANSLATE $translate_filter"
220 fi
221
222 local dir=$DIR/$py_module
223
224 mkdir -p $dir
225
226 py2-manifest $py_module $dir
227
228 set +o errexit
229 cat $dir/all-pairs.txt | repo-filter | exclude-files $typecheck_filter | mysort \
230 > $dir/typecheck.txt
231
232 cat $dir/typecheck.txt | exclude-files $translate_filter | mysort \
233 > $dir/translate.txt
234
235 echo DEPS $dir/*
236}
237
238# mycpp and pea deps are committed to git instead of in _build/NINJA/ because
239# users might not have Python 3.10
240
241write-pea() {
242 local module='pea.pea_main'
243 local dir=prebuilt/ninja/$module
244 mkdir -p $dir
245
246 source build/dev-shell.sh # python3
247
248 # Can't use vendor/typing.py
249 py3-manifest $dir
250
251 cat $dir/all-pairs.txt | repo-filter | mysort | tee $dir/deps.txt
252
253 echo
254 echo $dir/*
255}
256
257write-mycpp() {
258 local module='mycpp.mycpp_main'
259 local dir=prebuilt/ninja/$module
260 mkdir -p $dir
261
262 if false; then
263 ( source $MYCPP_VENV/bin/activate
264 PYTHONPATH=$REPO_ROOT:$REPO_ROOT/mycpp:$MYPY_REPO maybe-our-python3 \
265 build/dynamic_deps.py py-manifest $module > $dir/all-pairs.txt
266 )
267 fi
268
269 # TODO: it would be nicer to put this at the top of the file, but we get
270 # READONLY errors.
271 source build/dev-shell.sh
272
273 py3-manifest $dir
274
275 local deps=$dir/deps.txt
276 cat $dir/all-pairs.txt \
277 | grep -v oilshell/oil_DEPS \
278 | repo-filter \
279 | exclude-files build/default.deps-filter.txt \
280 | mysort \
281 | tee $deps
282
283 echo
284 echo $dir/*
285}
286
287mycpp-example-parse() {
288 ### Manifests for mycpp/examples/parse are committed to git
289
290 local dir=$DIR/parse
291 mkdir -p $dir
292
293 py2-manifest mycpp.examples.parse $dir
294
295 local ty=mycpp/examples/parse.typecheck.txt
296 local tr=mycpp/examples/parse.translate.txt
297
298 # TODO: remove oils-for-unix
299 cat $dir/all-pairs.txt | repo-filter |
300 exclude-files build/default.typecheck-filter.txt | mysort > $ty
301
302 cat $ty | exclude-files build/default.translate-filter.txt > $tr
303
304 wc -l $ty $tr
305
306 #head $ty $tr
307}
308
309pea-hack() {
310 # Leave out help_.py for Soil
311 grep -v '_devbuild/gen/help_meta.py' $DIR/bin.oils_for_unix/typecheck.txt \
312 > pea/oils-typecheck.txt
313}
314
315# Sourced by NINJA-config.sh
316if test $(basename $0) = 'dynamic-deps.sh'; then
317 "$@"
318fi