OILS / build / dynamic-deps.sh View on Github | oilshell.org

251 lines, 98 significant
1#!/usr/bin/env bash
2#
3# Calculate and filter deps of Python apps.
4#
5# Usage:
6# build/dynamic-deps.sh <function name>
7
8set -o nounset
9set -o pipefail
10set -o errexit
11
12REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
13
14readonly PY_PATH='.:vendor/'
15
16# Temporary
17readonly DIR=_build/NINJA
18
19# In git
20readonly FILTER_DIR='prebuilt/dynamic-deps'
21
22make-egrep() {
23 # match chars until # or space, and line must be non-empty
24 gawk '
25 match($0, /([^# ]*)/, m) {
26 contents = m[0]
27 if (contents) { # skip empty lines
28 print(contents)
29 }
30 }
31 '
32}
33
34write-filters() {
35 ### Write filename filters in the egrep -f format
36
37 # For ./NINJA-config.sh to use.
38 # This style lets us add comments.
39
40 # For asdl.asdl_main and other tools
41 make-egrep >$FILTER_DIR/filter-py-tool.txt <<'EOF'
42__init__.py
43typing.py # vendor/typing.py isn't imported normally
44EOF
45
46 # Don't typecheck these files.
47
48 make-egrep >$FILTER_DIR/filter-typecheck.txt <<'EOF'
49__init__.py
50typing.py
51
52# OrderedDict is polymorphic
53pylib/collections_.py
54
55# lots of polymorphic stuff etc.
56mycpp/mylib.py
57
58# TODO: move or remove these
59tools/deps.py
60tools/readlink.py
61EOF
62
63 # On top of the typecheck filter, exclude these from translation. They are
64 # not inputs to mycpp.
65
66 make-egrep >$FILTER_DIR/filter-translate.txt <<'EOF'
67# generated code shouldn't be translated
68_devbuild/
69_gen/
70
71# definitions that are used by */*_gen.py
72.*_def\.py
73.*_spec\.py
74
75asdl/py.* # pybase.py ported by hand to C++
76
77core/py.* # pyos.py, pyutil.py ported by hand to C++
78core/optview\.py # core/optview_gen.py
79
80data_lang/py.* # pyj8.py
81
82frontend/py.*\.py # py_readline.py ported by hand to C++
83frontend/consts.py # frontend/consts_gen.py
84frontend/match.py # frontend/lexer_gen.py
85
86mycpp/iolib.py # Implemented in gc_iolib.{h,cC}
87mycpp/mops.py # Implemented in gc_mops.{h,cC}
88
89pgen2/grammar.py # These files are re-done in C++
90pgen2/pnode.py
91pgen2/token.py
92
93# should be py_path_stat.py, because it's ported by hand to C++
94pylib/path_stat.py
95
96# should be py_bool_stat.py, because it's ported by hand to C++
97osh/bool_stat.py
98EOF
99
100 wc -l $FILTER_DIR/filter-*
101}
102
103repo-filter() {
104 ### Select files from the dynamic_deps.py output
105
106 # select what's in the repo; eliminating stdlib stuff
107 # eliminate _cache for mycpp running under Python-3.10
108 fgrep -v "$REPO_ROOT/_cache" | fgrep "$REPO_ROOT" | awk '{ print $2 }'
109}
110
111exclude-filter() {
112 ### Exclude repo-relative paths
113
114 local filter_name=$1
115
116 egrep -v -f $FILTER_DIR/filter-$filter_name.txt
117}
118
119mysort() {
120 LC_ALL=C sort
121}
122
123#
124# Programs
125#
126
127py-tool() {
128 ### Used by NINJA-config.sh
129 local py_module=$1
130
131 local dir=$DIR/$py_module
132 mkdir -p $dir
133
134 PYTHONPATH=$PY_PATH /usr/bin/env python2 \
135 build/dynamic_deps.py py-manifest $py_module \
136 > $dir/all-pairs.txt
137
138 cat $dir/all-pairs.txt | repo-filter | exclude-filter py-tool | mysort \
139 > $dir/deps.txt
140
141 echo "DEPS $dir/deps.txt"
142}
143
144asdl-target-lang() {
145 local py_module='asdl.target_lang_test'
146
147 # typing backport:
148 # https://pypi.org/project/typing/
149 #
150 # Brings in collections.py, copy.py, functools.py, heapq.py, keyword.py,
151 # weakref.py
152 # This might be OK?
153 # Right now we only depends on Lib/stat.py and Lib/__future__.py, but we can
154 # grow these
155
156 PYTHONPATH=$PY_PATH /usr/bin/env python2 \
157 build/dynamic_deps.py py-manifest $py_module
158}
159
160# Code generators
161list-gen() {
162 ls */*_gen.py
163}
164
165# mycpp and pea deps are committed to git instead of in _build/NINJA/ because
166# users might not have Python 3.10
167
168write-pea() {
169 # PYTHONPATH=$PY_PATH
170 local module='pea.pea_main'
171 local dir=prebuilt/ninja/$module
172 mkdir -p $dir
173
174 source build/dev-shell.sh # python3
175
176 # Can't use vendor/typing.py
177 PYTHONPATH=. python3 \
178 build/dynamic_deps.py py-manifest $module \
179 > $dir/all-pairs.txt
180
181 cat $dir/all-pairs.txt | repo-filter | mysort | tee $dir/deps.txt
182
183 echo
184 echo $dir/*
185}
186
187write-mycpp() {
188 local module='mycpp.mycpp_main'
189 local dir=prebuilt/ninja/$module
190 mkdir -p $dir
191
192 if false; then
193 ( source $MYCPP_VENV/bin/activate
194 PYTHONPATH=$REPO_ROOT:$REPO_ROOT/mycpp:$MYPY_REPO maybe-our-python3 \
195 build/dynamic_deps.py py-manifest $module > $dir/all-pairs.txt
196 )
197 fi
198
199 # TODO: it would be nicer to put this at the top of the file, but we get
200 # READONLY errors.
201 source build/dev-shell.sh
202
203 python3 build/dynamic_deps.py py-manifest $module > $dir/all-pairs.txt
204
205 local deps=$dir/deps.txt
206 cat $dir/all-pairs.txt \
207 | grep -v oilshell/oil_DEPS \
208 | repo-filter \
209 | exclude-filter py-tool \
210 | mysort \
211 | tee $deps
212
213 # EXTRA FILE
214 echo '_bin/datalog/dataflow' >> $deps
215
216 echo
217 echo $dir/*
218}
219
220mycpp-example-parse() {
221 ### Manifests for mycpp/examples/parse are committed to git
222
223 local dir=$DIR/parse
224 mkdir -p $dir
225
226 PYTHONPATH=$PY_PATH /usr/bin/env python2 \
227 build/dynamic_deps.py py-manifest mycpp.examples.parse \
228 > $dir/all-pairs.txt
229
230 local ty=mycpp/examples/parse.typecheck.txt
231 local tr=mycpp/examples/parse.translate.txt
232
233 cat $dir/all-pairs.txt | repo-filter | exclude-filter typecheck | mysort > $ty
234
235 cat $ty | exclude-filter translate > $tr
236
237 wc -l $ty $tr
238
239 #head $ty $tr
240}
241
242pea-hack() {
243 # Leave out help_.py for Soil
244 grep -v '_devbuild/gen/help_meta.py' $DIR/bin.oils_for_unix/typecheck.txt \
245 > pea/oils-typecheck.txt
246}
247
248# Sourced by NINJA-config.sh
249if test $(basename $0) = 'dynamic-deps.sh'; then
250 "$@"
251fi