OILS / deps / from-apt.sh View on Github | oils.pub

430 lines, 214 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# deps/from-apt.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10# These are needed for bootstrapping pip in Python 3.10
11# (Also used by build/py.sh ubuntu-deps)
12#
13# For building Python 3.10 with working 'pip install'
14# libssl-dev: to download packages
15# libffi-dev: for working setuptools
16# zlib1g-dev: needed for 'import zlib'
17declare -a PY3_BUILD_DEPS=(libssl-dev libffi-dev zlib1g-dev)
18
19# for deps/from-R.sh
20declare -a R_BUILD_DEPS=(
21 r-base-core # R interpreter
22
23 # ICU for the R stringi package. This makes compilation faster; otherwise
24 # it tries to compile it from source.
25 # https://stringi.gagolewski.com/install.html
26 libicu-dev
27)
28
29install-R() {
30 ### For manual use OUTSIDE container
31 apt-install "${R_BUILD_DEPS[@]}"
32}
33
34# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md#run---mounttypecache
35
36# TODO: Use this for ALL images
37apt-install() {
38 ### Helper to slim down images
39
40 apt-get install -y --no-install-recommends "$@"
41}
42
43init-deb-cache() {
44 rm -f /etc/apt/apt.conf.d/docker-clean
45 echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
46}
47
48readonly -a BUILD_PACKAGES=(
49 gcc
50 g++ # re2c is C++
51 make # to build re2c
52
53 # for cmark
54 cmake
55
56 # cmake -G Ninja can be used
57 ninja-build
58
59 # For 'deps/wedge.sh unboxed-install'
60 sudo
61
62 # uftrace configure uses pkg-config to find python3 flags
63 pkg-config
64
65 # 2024-06: I think this is necessary for the boxed build of zsh
66 # Not sure why the wedge-setup-debian VM build doesn't need it. Oh probably
67 # because Github Actions pre-installs many packages for you.
68 libncursesw5-dev
69
70 # for USDT probes
71 systemtap-sdt-dev
72
73 # Dependencies for building our own Python3 wedge. Otherwise 'pip install'
74 # won't work.
75 # TODO: We should move 'pip install' to build time.
76 "${PY3_BUILD_DEPS[@]}"
77
78 # For installing R packages
79 "${R_BUILD_DEPS[@]}"
80)
81
82layer-wedge-bootstrap-debian() {
83 apt-get update
84
85 # Pass aditional deps
86 apt-install "${BUILD_PACKAGES[@]}" "$@"
87}
88
89layer-wedge-bootstrap-debian-10() {
90 # Default packages
91 layer-wedge-bootstrap-debian
92}
93
94layer-wedge-bootstrap-debian-12() {
95 local -a uftrace_packages=(
96 # uftrace configure detects with #include "Python.h"
97 python3-dev
98 # shared library for uftrace to do dlopen()
99 # requires path in uftrace source
100 libpython3.11
101
102 #libpython3.7
103 )
104
105 layer-wedge-bootstrap-debian "${uftrace_packages[@]}"
106}
107
108layer-python-symlink() {
109 ### A special layer for building CPython; done as root
110 ln -s -f -v /usr/bin/python2 /usr/bin/python
111}
112
113layer-debian-10() {
114 # Can't install packages in Debian without this
115 apt-get update # uses /var/lib/apt
116
117 # uses /var/cache/apt
118 apt-install git python2
119}
120
121layer-debian-12() {
122 # Can't install packages in Debian without this
123 apt-get update # uses /var/lib/apt
124
125 # uses /var/cache/apt
126 # Soil's time-tsv3 can run under python3 too
127 apt-install git python3
128}
129
130layer-locales() {
131 apt-install locales
132 # uncomment in a file
133 sed -i 's/# en_US.UTF-8/en_US.UTF-8/' /etc/locale.gen
134 locale-gen --purge en_US.UTF-8
135}
136
137wild() {
138 # for build/py.sh all
139 local -a packages=(
140 gcc # 'cc' for compiling Python extensions
141 g++ # for C++ tarball
142
143 python2-dev
144 libreadline-dev
145 curl # wait for cpp-tarball
146 ca-certificates # curl https - could do this outside container
147 )
148
149 apt-install "${packages[@]}"
150}
151
152dev-minimal() {
153 local -a packages=(
154 # TODO: remove
155 python2-dev # for building Python extensions
156 python-setuptools # Python 2, for flake8
157 python-pip # flake8 typing
158
159 gcc # for building Python extensions
160 libreadline-dev
161
162 python3-setuptools # mypy
163 python3-pip
164 # 2023-07: somehow this became necessary to pip3 install typed_ast, a MyPy
165 # dep, which recently updated to version 1.5.5
166 python3-dev
167
168 # Note: osh-minimal task needs shells; testing WITHOUT spec-bin shells
169 busybox-static mksh zsh
170
171 gawk
172
173 # 'ps' used by spec tests
174 procps
175 # for oil-spec task
176 jq
177 )
178
179 apt-install "${packages[@]}"
180}
181
182pea() {
183 # For installing MyPy
184 # apt-install python3-pip
185
186 echo 'None'
187}
188
189other-tests() {
190 local -a packages=(
191 libreadline-dev
192 python2-dev # osh2oil needs build/py.sh minimal
193
194 # Compilers for R. TODO: try removing after wedge
195 gcc g++
196
197 make # to build py27.grammar.marshal, ugh
198
199 r-base-core
200 )
201
202 apt-install "${packages[@]}"
203}
204
205cpp-small() {
206 local -a packages=(
207 # for build/py.sh all
208 libreadline-dev
209 python2-dev
210
211 # To compile Oil
212 g++
213 ninja-build
214
215 # For 32-bit binaries with -m32
216 gcc-multilib
217 g++-multilib
218
219 # For some tests
220 gawk
221
222 # for MyPy git clone https://. TODO: remove when the build is hermetic
223 ca-certificates
224
225 # for test/ltrace
226 ltrace
227
228 # for USDT probes
229 systemtap-sdt-dev
230 )
231
232 apt-install "${packages[@]}"
233}
234
235test-image() {
236 ### For testing builds, not run on CI
237
238 # We don't really need build deps?
239 apt-install build-essential "${PY3_BUILD_DEPS[@]}"
240}
241
242benchmarks() {
243 ### For benchmarks
244
245 local -a packages=(
246 # for build/py.sh all
247 libreadline-dev
248 #python2-dev
249
250 # To build Oils
251 g++
252 ninja-build
253 make # to build R packages
254
255 # to create _test/index.html
256 gawk
257
258 # for stable benchmarks. TODO: could move osh-parser cachegrind to benchmarks2
259 valgrind
260
261 # benchmarks compare system shells -- they don't use our spec-bin? In case
262 # there are perf bugs caused by the build
263 busybox-static mksh zsh
264
265 # retrieving deps like benchmarks/osh-runtime -- TODO: move to build time
266 wget
267 bzip2 # extracting benchmarks/osh-runtime
268 xz-utils
269
270 # For analyzing benchmarks.
271 r-base-core
272
273 # pgrep used by test/stateful in interactive task
274 # TODO: Could move both Python and C++ to their own image
275 # That will be a good use case once we have
276 procps
277 )
278
279 apt-install "${packages[@]}"
280}
281
282bloaty() {
283 local -a packages=(
284 g++ # for C++ tarball
285 curl # wait for cpp-tarball
286 ca-certificates # curl https - could do this outside container
287 )
288
289 apt-install "${packages[@]}"
290}
291
292benchmarks2() {
293 ### uftrace needs a Python plugin
294
295 local -a packages=(
296 curl # fetch C++ tarball
297 g++ # build it
298
299 # uftrace needs a Python 3 plugin
300 # This is different than 'python3' or 'python3.11' -- it's only the shared
301 # lib?
302 libpython3.11
303
304 # for stable benchmarks.
305 valgrind
306
307 # Analyze uftrace
308 r-base-core
309 )
310
311 apt-install "${packages[@]}"
312}
313
314cpp-spec() {
315 ### For cpp-spec
316
317 local -a packages=(
318 # for build/py.sh all
319 libreadline-dev
320 python2-dev
321
322 # To build Oil
323 g++
324 ninja-build
325
326 # to create _test/index.html
327 gawk
328
329 # spec tests use these
330 procps
331 jq
332
333 # for MyPy git clone https://. TODO: remove when the build is hermetic
334 ca-certificates
335 )
336
337 apt-install "${packages[@]}"
338}
339
340clang() {
341 ### For cpp-coverage
342
343 local -a packages=(
344 # For build/py.sh minimal
345 libreadline-dev
346 python2-dev
347
348 # Compile Oils
349 g++
350 ninja-build
351
352 xz-utils # to extract Clang
353
354 # for MyPy git clone https://. TODO: remove when the build is hermetic
355 ca-certificates
356 )
357
358 apt-install "${packages[@]}"
359}
360
361ovm-tarball() {
362 local -a packages=(
363 # build/py.sh all
364 libreadline-dev
365 python2-dev
366
367 # retrieving spec-bin -- TODO: move to build time
368 wget
369 # for wget https://. TODO: remove when the build is hermetic
370 ca-certificates
371
372 # when spec tests use 'time', dash falls back on 'time' command
373 'time'
374
375 # TODO: probably can remove C++ compiler now that re2c is a wedge
376 gcc
377 g++
378
379 # for cmark
380 cmake
381 # to build Python-2.7.13 (could be a wedge)
382 make
383
384 xz-utils # extract e.g. zsh/yash tarballs
385 bzip2 # extract e.g. busybox tarball
386
387 # for syscall measurements
388 strace
389
390 # used by test/spec-runner.sh
391 gawk
392 )
393
394 apt-install "${packages[@]}"
395}
396
397app-tests() {
398 local -a packages=(
399 # build/py.sh all
400 libreadline-dev
401 python2-dev
402
403 # retrieving spec-bin -- TODO: move to build time
404 wget
405 # for wget https://. TODO: remove when the build is hermetic
406 ca-certificates
407
408 curl # wait for cpp-tarball
409
410 gcc
411 g++ # for C++ tarball
412
413 # to build ble.sh
414 make
415 # used by ble.sh
416 gawk
417 procps
418
419 # for ble.sh contra
420 libx11-dev
421 libxft-dev
422 libncursesw5-dev
423 )
424
425 apt-install "${packages[@]}"
426}
427
428if test $(basename $0) = 'from-apt.sh'; then
429 "$@"
430fi