OILS / build / ovm-compile.sh View on Github | oils.pub

431 lines, 168 significant
1#!/usr/bin/env bash
2#
3# Compile OVM tarball.
4#
5# Usage:
6# build/ovm-compile.sh <function name>
7
8set -o nounset
9set -o pipefail
10set -o errexit
11shopt -s strict:all 2>/dev/null || true # dogfood for OSH
12
13REPO_ROOT=$(cd $(dirname $0)/..; pwd)
14readonly REPO_ROOT
15
16source build/common.sh
17
18source-detected-config-or-die() {
19 if ! source _build/detected-config.sh; then
20 # Make this error stand out.
21 echo
22 echo "FATAL: can't find _build/detected-config.h. Run './configure'"
23 echo
24 exit 1
25 fi
26}
27
28# NOTES on trying to delete certain modules:
29#
30# _warnings.c: There weren't that many; it probably could be deleted.
31# bufferobject.c: the types.py module uses it.
32# Python-ast.h: pythonrun.c uses it in several places (mod_ty), and a lot of
33# stuff uses pythonrun.c.
34# pythonrun.c: lots interpreter flags and interpreter initialization caused
35# link errors.
36# pyctype.c: Tables needed for many string operations.
37
38# getargs.c: needed for Python-C API, e.g. PyArg_ParseTuple.
39# dtoa.c: not tried, but I assume that %.3f for 'time' uses it.
40
41
42readonly OVM_PYTHON_OBJS='
43Python/_warnings.c
44Python/bltinmodule.c
45Python/ceval.c
46Python/errors.c
47Python/getargs.c
48Python/getcompiler.c
49Python/getplatform.c
50Python/getversion.c
51Python/import.c
52Python/marshal.c
53Python/modsupport.c
54Python/mystrtoul.c
55Python/mysnprintf.c
56Python/pyarena.c
57Python/pyctype.c
58Python/pyfpe.c
59Python/pystate.c
60Python/pythonrun.c
61Python/random.c
62Python/structmember.c
63Python/sysmodule.c
64Python/traceback.c
65Python/pystrtod.c
66Python/dtoa.c
67Python/pymath.c
68'
69# NOTE: pystrtod.c needs some floating point functions in pymath.c
70
71OBJECT_OBJS='
72Objects/abstract.c
73Objects/boolobject.c
74Objects/bufferobject.c
75Objects/bytes_methods.c
76Objects/capsule.c
77Objects/cellobject.c
78Objects/classobject.c
79Objects/cobject.c
80Objects/codeobject.c
81Objects/descrobject.c
82Objects/enumobject.c
83Objects/exceptions.c
84Objects/genobject.c
85Objects/fileobject.c
86Objects/floatobject.c
87Objects/frameobject.c
88Objects/funcobject.c
89Objects/intobject.c
90Objects/iterobject.c
91Objects/listobject.c
92Objects/longobject.c
93Objects/dictobject.c
94Objects/methodobject.c
95Objects/moduleobject.c
96Objects/object.c
97Objects/obmalloc.c
98Objects/rangeobject.c
99Objects/setobject.c
100Objects/sliceobject.c
101Objects/stringobject.c
102Objects/structseq.c
103Objects/tupleobject.c
104Objects/typeobject.c
105Objects/weakrefobject.c
106'
107
108# Non-standard lib stuff.
109MODULE_OBJS='
110Modules/main.c
111Modules/gcmodule.c
112'
113
114# The stuff in Modules/Setup.dist, signalmodule.c. NOTE: In Python,
115# signalmodule.c is specified in Modules/Setup.config, which comes from
116# 'configure' output.
117MODOBJS='
118Modules/errnomodule.c
119Modules/pwdmodule.c
120Modules/_sre.c
121Modules/_weakref.c
122Modules/zipimport.c
123Modules/signalmodule.c
124'
125
126# Parser/myreadline.c is needed for raw_input() to work. There is a dependency
127# from Python/bltinmodule.c to it.
128OVM_LIBRARY_OBJS="
129Modules/getbuildinfo.c
130Parser/myreadline.c
131$OBJECT_OBJS
132$OVM_PYTHON_OBJS
133$MODULE_OBJS
134$MODOBJS
135"
136
137readonly EMPTY_STR='""'
138
139# Stub out a few variables
140readonly PREPROC_FLAGS=(
141 -D OVM_MAIN \
142 -D PYTHONPATH="$EMPTY_STR" \
143 -D VERSION="$EMPTY_STR" \
144 -D VPATH="$EMPTY_STR" \
145 -D Py_BUILD_CORE \
146 # Python already has support for disabling complex numbers!
147 -D WITHOUT_COMPLEX
148)
149
150# NOTE: build/oil-defs is hard-coded to the oil.ovm app. We're abandoning
151# hello.ovm and opy.ovm for now, but those can easily be added later. We
152# haven't mangled the CPython source!
153readonly INCLUDE_PATHS=(
154 -I . # for pyconfig.h
155 -I .. # for _gen/frontend/id_kind_asdl_c.h etc.
156 -I Include
157 -I ../build/oil-defs
158)
159readonly CC=${CC:-cc} # cc should be on POSIX systems
160
161# BASE_CFLAGS is copied by observation from what configure.ac does on my Ubuntu
162# 16.04 system. Then we check if it works on Alpine Linux too.
163
164# "Python violates C99 rules, by casting between incompatible pointer types.
165# GCC may generate bad code as a result of that, so use -fno-strict-aliasing if
166# supported."
167# - gcc 4.x and Clang need -fwrapv
168
169# TODO:
170# - -DNDEBUG is also passed. That turns off asserts. Do we want that?
171# - We should auto-detect the flags in configure, or simplify the source so it
172# isn't necessary. Python's configure.ac sometimes does it by compiling a test
173# file; at other times it does it by grepping $CC --help.
174
175# pyext/fanos.c needs -std=c99
176BASE_CFLAGS='-fno-strict-aliasing -fwrapv -Wall -Wstrict-prototypes -std=c99'
177
178# These flags are disabled for OS X. I would have thought it would work in
179# Clang? It works with both GCC and Clang on Linux.
180# https://stackoverflow.com/questions/6687630/how-to-remove-unused-c-c-symbols-with-gcc-and-ld
181#BASE_CFLAGS="$BASE_CFLAGS -fdata-sections -ffunction-sections"
182
183# Needed after cpython-defs filtering.
184BASE_CFLAGS="$BASE_CFLAGS -Wno-unused-variable -Wno-unused-function"
185readonly BASE_CFLAGS
186
187BASE_LDFLAGS=''
188# Disabled for OS X
189# BASE_LDFLAGS='-Wl,--gc-sections'
190
191# The user should be able to customize CFLAGS, but it shouldn't disable what's
192# in BASE_CFLAGS.
193readonly CFLAGS=${CFLAGS:-}
194readonly LDFLAGS=${LDFLAGS:-}
195
196build() {
197 local out=${1:-$PY27/ovm2}
198 local module_init=${2:-$PY27/Modules/config.c}
199 local main_name=${3:-_tmp/hello/main_name.c}
200 local c_module_srcs=${4:-_tmp/hello/c-module-srcs.txt}
201 shift 4
202
203 local abs_out=$PWD/$out
204 local abs_module_init=$PWD/$module_init
205 local abs_main_name=$PWD/$main_name
206 local abs_c_module_srcs=$PWD/$c_module_srcs
207
208 #echo $OVM_LIBRARY_OBJS
209
210 # HAVE_READLINE defined in detected-config.sh.
211 source-detected-config-or-die
212
213 pushd $PY27
214
215 local readline_flags=''
216 if [[ "$HAVE_READLINE" -eq 1 ]]; then
217 # Readline interface for tokenizer.c and [raw_]input() in bltinmodule.c.
218 # For now, we are using raw_input() for the REPL. TODO: Parameterize this!
219 # We should create a special no_readline_raw_input().
220
221 c_module_src_list=$(cat $abs_c_module_srcs)
222
223 if [[ -n "$READLINE_DIR" ]]; then
224 readline_flags+="-L $READLINE_DIR/lib -I $READLINE_DIR/include "
225 fi
226
227 # NOTE: pyconfig.h has HAVE_LIBREADLINE but doesn't appear to use it?
228 readline_flags+="-l readline -D HAVE_READLINE"
229 else
230 # don't fail
231 c_module_src_list=$(grep -E -v '/readline.c|/line_input.c' $abs_c_module_srcs || true)
232 fi
233
234 # $PREFIX comes from ./configure and defaults to /usr/local.
235 # $EXEC_PREFIX is a GNU thing and used in getpath.c. Could probably get rid
236 # of it.
237
238 time $CC \
239 ${BASE_CFLAGS} \
240 ${CFLAGS} \
241 "${INCLUDE_PATHS[@]}" \
242 "${PREPROC_FLAGS[@]}" \
243 -D PREFIX="\"$PREFIX\"" \
244 -D EXEC_PREFIX="\"$PREFIX\"" \
245 -o $abs_out \
246 $OVM_LIBRARY_OBJS \
247 $abs_module_init \
248 $abs_main_name \
249 $c_module_src_list \
250 Modules/ovm.c \
251 -l m \
252 ${BASE_LDFLAGS} \
253 ${LDFLAGS} \
254 $readline_flags \
255 "$@"
256
257 # NOTE:
258 # -l readline -l termcap -- for Python readline. Hm it builds without -l
259 # termcap.
260 # -l z WOULD be needed for zlibmodule.c, but we don't need it because our zip
261 # file has no compression -- see build/make_zip.py with ZIP_STORED.
262 # zipimport works fine without this.
263}
264
265# build the optimized one. Makefile uses -O3.
266
267# Clang -O2 is 1.37 MB. 18 seconds to compile.
268# -m32 is 1.12 MB. But I probably have to redefine a few things because
269# there are more warnings.
270# -O3 is 1.40 MB.
271
272# GCC -O2 is 1.35 MB. 21 seconds to compile.
273
274build-dbg() {
275 build "$@" -O0 -g -D OVM_DEBUG
276}
277
278# This will be stripped later.
279build-opt() {
280 # frame pointer for perf. Otherwise stack traces are messed up!
281 # http://www.brendangregg.com/FlameGraphs/cpuflamegraphs.html#C But why
282 # isn't debuginfo enough? Because it's a recursive function?
283 # Does this make things slower? Do I need a "perf" build?
284 build "$@" -O3 -fno-omit-frame-pointer
285}
286
287#
288# Source Release (uses same files
289#
290
291add-py27() {
292 xargs -I {} -- echo $PY27/{}
293}
294
295python-sources() {
296 echo "$OVM_LIBRARY_OBJS" | add-py27
297}
298
299_headers() {
300 local c_module_srcs=${1:-_tmp/hello/c-module-srcs.txt}
301 local abs_c_module_srcs=$PWD/$c_module_srcs
302
303 cd $PY27
304
305 # -MM: no system headers
306 gcc \
307 "${INCLUDE_PATHS[@]}" \
308 "${PREPROC_FLAGS[@]}" \
309 -MM $OVM_LIBRARY_OBJS \
310 Modules/ovm.c \
311 $(cat $abs_c_module_srcs)
312}
313
314# NOTE: 91 headers in Include, but only 81 referenced here. So it's worth it.
315# These are probably for the parser.
316#
317# NOTE: We also should get rid of asdl.h and so forth.
318
319python-headers() {
320 local c_module_srcs=$1
321
322 # 1. -MM outputs Makefile fragments, so egrep turns those into proper lines.
323 #
324 # 2. The user should generated detected-config.h, so remove it.
325 #
326 # 3. # gcc outputs paths like
327 # Python-2.7.13/Python/../Objects/stringlib/stringdefs.h
328 # but 'Python/..' causes problems for tar.
329 #
330
331 # NOTE: need .def for build/oil-defs.
332 _headers $c_module_srcs \
333 | egrep --only-matching '[^ ]+\.(h|def)' \
334 | grep -v '_build/detected-config.h' \
335 | sed 's|^Python/../||' \
336 | sort | uniq | add-py27
337}
338
339make-tar() {
340 local app_name=${1:-hello}
341 local bytecode_zip=${2:-bytecode-cpython.zip}
342 local out=${3:-_release/hello.tar}
343
344 local version_file
345 case $app_name in
346 oil)
347 version_file=oils-version.txt
348 ;;
349 hello)
350 version_file=build/testdata/hello-version.txt
351 ;;
352 *)
353 die "Unknown app $app_name"
354 exit 1
355 ;;
356 esac
357 local version=$(head -n 1 $version_file)
358
359 echo "Creating $app_name version $version"
360
361 local c_module_srcs=_build/$app_name/c-module-srcs.txt
362
363 # Add oil-0.0.0/ to the beginning of every path.
364 local sed_expr="s,^,${app_name}-${version}/,"
365
366 # Differences between tarball and repo:
367 #
368 # - build/portable-rules.mk is intentionally not included in the release tarball.
369 # The Makefile can and should operate without it.
370 #
371 # - We include intermediate files like c-module-srcs.txt, so we don't have to
372 # ship tools dynamic_deps.py. The end-user build shouldn't depend on Python.
373
374 # Note: python-headers runs gcc -M, including pyconfig.h and
375 # _build/detected-config.h.
376
377 tar --create --transform "$sed_expr" --file $out \
378 LICENSE.txt \
379 INSTALL-old.txt \
380 configure \
381 install \
382 uninstall \
383 Makefile \
384 doc/osh.1 \
385 build/ovm-compile.sh \
386 build/ovm-actions.sh \
387 build/clean.sh \
388 build/common.sh \
389 build/detect-*.c \
390 _build/$app_name/$bytecode_zip \
391 _build/$app_name/*.c \
392 $PY27/LICENSE \
393 $PY27/Modules/ovm.c \
394 $c_module_srcs \
395 $(cat $c_module_srcs | add-py27) \
396 $(python-headers $c_module_srcs) \
397 $(python-sources)
398
399 ls -l $out
400}
401
402# 123K lines.
403# Excluding MODOBJS, it's 104K lines.
404#
405# Biggest: posixmodule,unicodeobject,typeobject,ceval.
406#
407# Remove tmpnam from posixmodule, other cruft.
408#
409# Big ones to rid of: unicodeobject.c, import.c
410# codecs and codecsmodule? There is some non-unicode stuff there though.
411#
412# Probably need unicode for compatibility with modules and web frameworks
413# especially.
414
415count-c-lines() {
416 pushd $PY27
417 wc -l $OVM_LIBRARY_OBJS | sort -n
418
419 # 90 files.
420 # NOTE: To count headers, use the tar file.
421 echo
422 echo 'Files:'
423 { for i in $OVM_LIBRARY_OBJS; do
424 echo $i
425 done
426 } | wc -l
427
428 popd
429}
430
431"$@"