OILS / deps / from-apt.sh View on Github | oils.pub

418 lines, 202 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# deps/from-apt.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10# These are needed for bootstrapping pip in Python 3.10
11# (Also used by build/deps.sh wedge-deps-{debian,alpine,fedora})
12#
13# For building Python 3.10 with working 'pip install'
14# libssl-dev: to download packages
15# libffi-dev: for working setuptools
16# zlib1g-dev: needed for 'import zlib'
17declare -a PY3_BUILD_DEPS=(libssl-dev libffi-dev zlib1g-dev)
18
19# Needed to run unit tests, e.g. 'import readline'
20declare -a PY2_BUILD_DEPS=(libreadline-dev)
21
22# for deps/from-R.sh
23declare -a R_BUILD_DEPS=(
24 r-base-core # R interpreter
25
26 # ICU for the R stringi package. This makes compilation faster; otherwise
27 # it tries to compile it from source.
28 # https://stringi.gagolewski.com/install.html
29 libicu-dev
30)
31
32declare -a SPEC_TEST_DEPS=(
33 jq # YSH spec tests invoke this
34 procps # for 'ps'
35)
36
37install-R() {
38 ### For manual use OUTSIDE container
39 apt-install "${R_BUILD_DEPS[@]}"
40}
41
42# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md#run---mounttypecache
43
44# TODO: Use this for ALL images
45apt-install() {
46 ### Helper to slim down images
47
48 apt-get install -y --no-install-recommends "$@"
49}
50
51init-deb-cache() {
52 rm -f /etc/apt/apt.conf.d/docker-clean
53 echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
54}
55
56readonly -a BUILD_PACKAGES=(
57 gcc
58 g++ # re2c is C++
59 make # to build re2c
60
61 # for cmark
62 cmake
63
64 # cmake -G Ninja can be used
65 ninja-build
66
67 # For 'deps/wedge.sh unboxed-install'
68 sudo
69
70 # uftrace configure uses pkg-config to find python3 flags
71 pkg-config
72
73 # 2024-06: I think this is necessary for the boxed build of zsh
74 # Not sure why the wedge-setup-debian VM build doesn't need it. Oh probably
75 # because Github Actions pre-installs many packages for you.
76 libncursesw5-dev
77
78 # for USDT probes
79 systemtap-sdt-dev
80
81 "${PY2_BUILD_DEPS[@]}"
82
83 # Dependencies for building our own Python3 wedge. Otherwise 'pip install'
84 # won't work.
85 # TODO: We should move 'pip install' to build time.
86 "${PY3_BUILD_DEPS[@]}"
87
88 # For installing R packages
89 "${R_BUILD_DEPS[@]}"
90)
91
92layer-wedge-bootstrap-debian() {
93 apt-get update
94
95 # Pass additional deps
96 apt-install \
97 "${BUILD_PACKAGES[@]}" \
98 "$@"
99}
100
101layer-wedge-bootstrap-debian-10() {
102 # Default packages
103 layer-wedge-bootstrap-debian
104}
105
106layer-wedge-bootstrap-debian-12() {
107 local -a uftrace_packages=(
108 # uftrace configure detects with #include "Python.h"
109 python3-dev
110 # shared library for uftrace to do dlopen()
111 # requires path in uftrace source
112 libpython3.11
113
114 #libpython3.7
115 )
116
117 layer-wedge-bootstrap-debian "${uftrace_packages[@]}"
118}
119
120layer-debian-10() {
121 # Can't install packages in Debian without this
122 apt-get update # uses /var/lib/apt
123
124 # uses /var/cache/apt
125 apt-install git python2
126}
127
128layer-debian-12() {
129 # Can't install packages in Debian without this
130 apt-get update # uses /var/lib/apt
131
132 # uses /var/cache/apt
133 # Soil's time-tsv3 can run under python3 too
134 apt-install git python3
135}
136
137layer-locales() {
138 set -x
139 apt-install locales
140
141 # uncomment in a file
142 sed -i 's/# en_US.UTF-8/en_US.UTF-8/' /etc/locale.gen
143 locale-gen --purge en_US.UTF-8
144}
145
146wild() {
147 # for build/py.sh all
148 local -a packages=(
149 gcc # 'cc' for compiling Python extensions
150 g++ # for C++ tarball
151
152 libreadline-dev
153 curl # wait for cpp-tarball
154 ca-certificates # curl https - could do this outside container
155 )
156
157 apt-install "${packages[@]}"
158}
159
160dev-minimal() {
161 local -a packages=(
162 gcc # for building Python extensions
163 libreadline-dev
164
165 python3-setuptools # mypy
166 python3-pip
167 # 2023-07: somehow this became necessary to pip3 install typed_ast, a MyPy
168 # dep, which recently updated to version 1.5.5
169 python3-dev
170
171 # Note: osh-minimal task needs shells; testing WITHOUT spec-bin shells
172 busybox-static mksh zsh
173
174 gawk
175
176 # 'ps' used by spec tests
177 procps
178
179 "${SPEC_TEST_DEPS[@]}"
180 )
181
182 apt-install "${packages[@]}"
183}
184
185other-tests() {
186 local -a packages=(
187 libreadline-dev
188
189 # Compilers for R. TODO: try removing after wedge
190 gcc g++
191
192 make # to build py27.grammar.marshal, ugh
193
194 r-base-core
195 )
196
197 apt-install "${packages[@]}"
198}
199
200cpp-small() {
201 local -a packages=(
202 # for build/py.sh all
203 libreadline-dev
204
205 # To compile Oils
206 g++
207 ninja-build
208
209 # For 32-bit binaries with -m32
210 gcc-multilib
211 g++-multilib
212
213 # For some tests
214 gawk
215
216 # for MyPy git clone https://. TODO: remove when the build is hermetic
217 ca-certificates
218
219 # for test/ltrace
220 ltrace
221
222 # for USDT probes
223 systemtap-sdt-dev
224 )
225
226 apt-install "${packages[@]}"
227}
228
229test-image() {
230 ### For testing builds, not run on CI
231
232 # We don't really need build deps?
233 apt-install build-essential "${PY3_BUILD_DEPS[@]}"
234}
235
236benchmarks() {
237 ### For benchmarks
238
239 local -a packages=(
240 # for build/py.sh all
241 libreadline-dev
242
243 # To build Oils
244 g++
245 ninja-build
246 make # to build R packages
247
248 # to create _test/index.html
249 gawk
250
251 # for stable benchmarks. TODO: could move osh-parser cachegrind to benchmarks2
252 valgrind
253
254 # benchmarks compare system shells -- they don't use our spec-bin? In case
255 # there are perf bugs caused by the build
256 busybox-static mksh zsh
257
258 # retrieving deps like benchmarks/osh-runtime -- TODO: move to build time
259 wget
260 bzip2 # extracting benchmarks/osh-runtime
261 xz-utils
262
263 # For analyzing benchmarks.
264 r-base-core
265
266 # pgrep used by test/stateful in interactive task
267 # TODO: Could move both Python and C++ to their own image
268 # That will be a good use case once we have
269 procps
270 )
271
272 apt-install "${packages[@]}"
273}
274
275bloaty() {
276 local -a packages=(
277 g++ # for C++ tarball
278 curl # wait for cpp-tarball
279 ca-certificates # curl https - could do this outside container
280 )
281
282 apt-install "${packages[@]}"
283}
284
285benchmarks2() {
286 ### uftrace needs a Python plugin
287
288 local -a packages=(
289 curl # fetch C++ tarball
290 g++ # build it
291
292 # uftrace needs a Python 3 plugin
293 # This is different than 'python3' or 'python3.11' -- it's only the shared
294 # lib?
295 libpython3.11
296
297 # for stable benchmarks.
298 valgrind
299
300 # Analyze uftrace
301 r-base-core
302 )
303
304 apt-install "${packages[@]}"
305}
306
307cpp-spec() {
308 ### For cpp-spec
309
310 local -a packages=(
311 # for build/py.sh all
312 libreadline-dev
313
314 # To build Oils
315 g++
316 ninja-build
317
318 # to create _test/index.html
319 gawk
320
321 "${SPEC_TEST_DEPS[@]}"
322
323 # for MyPy git clone https://. TODO: remove when the build is hermetic
324 ca-certificates
325 )
326
327 apt-install "${packages[@]}"
328}
329
330clang() {
331 ### For cpp-coverage
332
333 local -a packages=(
334 # For build/py.sh minimal
335 libreadline-dev
336 #python2-dev
337
338 # Compile Oils
339 g++
340 ninja-build
341
342 xz-utils # to extract Clang
343
344 # for MyPy git clone https://. TODO: remove when the build is hermetic
345 ca-certificates
346 )
347
348 apt-install "${packages[@]}"
349}
350
351ovm-tarball() {
352 local -a packages=(
353 # build/py.sh all
354 libreadline-dev
355
356 # retrieving spec-bin -- TODO: move to build time
357 wget
358 # for wget https://. TODO: remove when the build is hermetic
359 ca-certificates
360
361 # when spec tests use 'time', dash falls back on 'time' command
362 'time'
363
364 # TODO: probably can remove C++ compiler now that re2c is a wedge
365 gcc
366 g++
367
368 # for cmark
369 cmake
370 # to build Python-2.7.13 (could be a wedge)
371 make
372
373 xz-utils # extract e.g. zsh/yash tarballs
374 bzip2 # extract e.g. busybox tarball
375
376 # for syscall measurements
377 strace
378
379 # used by test/spec-runner.sh
380 gawk
381 )
382
383 apt-install "${packages[@]}"
384}
385
386app-tests() {
387 local -a packages=(
388 # build/py.sh all
389 libreadline-dev
390
391 # retrieving spec-bin -- TODO: move to build time
392 wget
393 # for wget https://. TODO: remove when the build is hermetic
394 ca-certificates
395
396 curl # wait for cpp-tarball
397
398 gcc
399 g++ # for C++ tarball
400
401 # to build ble.sh
402 make
403 # used by ble.sh
404 gawk
405 procps
406
407 # for ble.sh contra
408 libx11-dev
409 libxft-dev
410 libncursesw5-dev
411 )
412
413 apt-install "${packages[@]}"
414}
415
416if test $(basename $0) = 'from-apt.sh'; then
417 "$@"
418fi