OILS / deps / from-apt.sh View on Github | oils.pub

444 lines, 216 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# deps/from-apt.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10# These are needed for bootstrapping pip in Python 3.10
11# (Also used by build/py.sh ubuntu-deps)
12#
13# For building Python 3.10 with working 'pip install'
14# libssl-dev: to download packages
15# libffi-dev: for working setuptools
16# zlib1g-dev: needed for 'import zlib'
17declare -a PY3_BUILD_DEPS=(libssl-dev libffi-dev zlib1g-dev)
18
19# for deps/from-R.sh
20declare -a R_BUILD_DEPS=(
21 r-base-core # R interpreter
22
23 # ICU for the R stringi package. This makes compilation faster; otherwise
24 # it tries to compile it from source.
25 # https://stringi.gagolewski.com/install.html
26 libicu-dev
27)
28
29install-R() {
30 ### For manual use OUTSIDE container
31 apt-install "${R_BUILD_DEPS[@]}"
32}
33
34# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md#run---mounttypecache
35
36# TODO: Use this for ALL images
37apt-install() {
38 ### Helper to slim down images
39
40 apt-get install -y --no-install-recommends "$@"
41}
42
43init-deb-cache() {
44 rm -f /etc/apt/apt.conf.d/docker-clean
45 echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
46}
47
48readonly -a BUILD_PACKAGES=(
49 gcc
50 g++ # re2c is C++
51 make # to build re2c
52
53 # for cmark
54 cmake
55
56 # cmake -G Ninja can be used
57 ninja-build
58
59 # For 'deps/wedge.sh unboxed-install'
60 sudo
61
62 # uftrace configure uses pkg-config to find python3 flags
63 pkg-config
64
65 # 2024-06: I think this is necessary for the boxed build of zsh
66 # Not sure why the wedge-setup-debian VM build doesn't need it. Oh probably
67 # because Github Actions pre-installs many packages for you.
68 libncursesw5-dev
69
70 # for USDT probes
71 systemtap-sdt-dev
72
73 # Dependencies for building our own Python3 wedge. Otherwise 'pip install'
74 # won't work.
75 # TODO: We should move 'pip install' to build time.
76 "${PY3_BUILD_DEPS[@]}"
77
78 # For installing R packages
79 "${R_BUILD_DEPS[@]}"
80)
81
82layer-wedge-bootstrap-debian() {
83 apt-get update
84
85 # Pass additional deps
86 apt-install \
87 "${BUILD_PACKAGES[@]}" \
88 "$@"
89}
90
91layer-wedge-bootstrap-debian-10() {
92 # Default packages
93 layer-wedge-bootstrap-debian
94}
95
96layer-wedge-bootstrap-debian-12() {
97 local -a uftrace_packages=(
98 # uftrace configure detects with #include "Python.h"
99 python3-dev
100 # shared library for uftrace to do dlopen()
101 # requires path in uftrace source
102 libpython3.11
103
104 #libpython3.7
105 )
106
107 layer-wedge-bootstrap-debian "${uftrace_packages[@]}"
108}
109
110layer-python-symlink() {
111 ### make /usr/bin/python and /usr/bin/python2
112
113 local py2=/wedge/oils-for-unix.org/pkg/python2/2.7.18/bin/python
114
115 # For building CPython in soil-ovm-tarball; done as root
116 ln -s -f -v $py2 /usr/bin/python
117
118 # For other dev tools
119 ln -s -f -v $py2 /usr/bin/python2
120}
121
122layer-debian-10() {
123 # Can't install packages in Debian without this
124 apt-get update # uses /var/lib/apt
125
126 # uses /var/cache/apt
127 apt-install git python2
128}
129
130layer-debian-12() {
131 # Can't install packages in Debian without this
132 apt-get update # uses /var/lib/apt
133
134 # uses /var/cache/apt
135 # Soil's time-tsv3 can run under python3 too
136 apt-install git python3
137}
138
139layer-locales() {
140 set -x
141 apt-install locales
142
143 # doesn't work either
144 # apt-install locales-all
145
146 # uncomment in a file
147 sed -i 's/# en_US.UTF-8/en_US.UTF-8/' /etc/locale.gen
148 locale-gen --purge en_US.UTF-8
149}
150
151wild() {
152 # for build/py.sh all
153 local -a packages=(
154 gcc # 'cc' for compiling Python extensions
155 g++ # for C++ tarball
156
157 #python2-dev
158 libreadline-dev
159 curl # wait for cpp-tarball
160 ca-certificates # curl https - could do this outside container
161 )
162
163 apt-install "${packages[@]}"
164}
165
166dev-minimal() {
167 local -a packages=(
168 # TODO: remove
169 python2-dev # for building Python extensions
170 python-setuptools # Python 2, for flake8
171 python-pip # flake8 typing
172
173 gcc # for building Python extensions
174 libreadline-dev
175
176 python3-setuptools # mypy
177 python3-pip
178 # 2023-07: somehow this became necessary to pip3 install typed_ast, a MyPy
179 # dep, which recently updated to version 1.5.5
180 python3-dev
181
182 # Note: osh-minimal task needs shells; testing WITHOUT spec-bin shells
183 busybox-static mksh zsh
184
185 gawk
186
187 # 'ps' used by spec tests
188 procps
189 # for oil-spec task
190 jq
191 )
192
193 apt-install "${packages[@]}"
194}
195
196pea() {
197 # For installing MyPy
198 # apt-install python3-pip
199
200 echo 'None'
201}
202
203other-tests() {
204 local -a packages=(
205 libreadline-dev
206 python2-dev # osh2oil needs build/py.sh minimal
207
208 # Compilers for R. TODO: try removing after wedge
209 gcc g++
210
211 make # to build py27.grammar.marshal, ugh
212
213 r-base-core
214 )
215
216 apt-install "${packages[@]}"
217}
218
219cpp-small() {
220 local -a packages=(
221 # for build/py.sh all
222 libreadline-dev
223 python2-dev
224
225 # To compile Oil
226 g++
227 ninja-build
228
229 # For 32-bit binaries with -m32
230 gcc-multilib
231 g++-multilib
232
233 # For some tests
234 gawk
235
236 # for MyPy git clone https://. TODO: remove when the build is hermetic
237 ca-certificates
238
239 # for test/ltrace
240 ltrace
241
242 # for USDT probes
243 systemtap-sdt-dev
244 )
245
246 apt-install "${packages[@]}"
247}
248
249test-image() {
250 ### For testing builds, not run on CI
251
252 # We don't really need build deps?
253 apt-install build-essential "${PY3_BUILD_DEPS[@]}"
254}
255
256benchmarks() {
257 ### For benchmarks
258
259 local -a packages=(
260 # for build/py.sh all
261 libreadline-dev
262 #python2-dev
263
264 # To build Oils
265 g++
266 ninja-build
267 make # to build R packages
268
269 # to create _test/index.html
270 gawk
271
272 # for stable benchmarks. TODO: could move osh-parser cachegrind to benchmarks2
273 valgrind
274
275 # benchmarks compare system shells -- they don't use our spec-bin? In case
276 # there are perf bugs caused by the build
277 busybox-static mksh zsh
278
279 # retrieving deps like benchmarks/osh-runtime -- TODO: move to build time
280 wget
281 bzip2 # extracting benchmarks/osh-runtime
282 xz-utils
283
284 # For analyzing benchmarks.
285 r-base-core
286
287 # pgrep used by test/stateful in interactive task
288 # TODO: Could move both Python and C++ to their own image
289 # That will be a good use case once we have
290 procps
291 )
292
293 apt-install "${packages[@]}"
294}
295
296bloaty() {
297 local -a packages=(
298 g++ # for C++ tarball
299 curl # wait for cpp-tarball
300 ca-certificates # curl https - could do this outside container
301 )
302
303 apt-install "${packages[@]}"
304}
305
306benchmarks2() {
307 ### uftrace needs a Python plugin
308
309 local -a packages=(
310 curl # fetch C++ tarball
311 g++ # build it
312
313 # uftrace needs a Python 3 plugin
314 # This is different than 'python3' or 'python3.11' -- it's only the shared
315 # lib?
316 libpython3.11
317
318 # for stable benchmarks.
319 valgrind
320
321 # Analyze uftrace
322 r-base-core
323 )
324
325 apt-install "${packages[@]}"
326}
327
328cpp-spec() {
329 ### For cpp-spec
330
331 local -a packages=(
332 # for build/py.sh all
333 libreadline-dev
334 python2-dev
335
336 # To build Oil
337 g++
338 ninja-build
339
340 # to create _test/index.html
341 gawk
342
343 # spec tests use these
344 procps
345 jq
346
347 # for MyPy git clone https://. TODO: remove when the build is hermetic
348 ca-certificates
349 )
350
351 apt-install "${packages[@]}"
352}
353
354clang() {
355 ### For cpp-coverage
356
357 local -a packages=(
358 # For build/py.sh minimal
359 libreadline-dev
360 python2-dev
361
362 # Compile Oils
363 g++
364 ninja-build
365
366 xz-utils # to extract Clang
367
368 # for MyPy git clone https://. TODO: remove when the build is hermetic
369 ca-certificates
370 )
371
372 apt-install "${packages[@]}"
373}
374
375ovm-tarball() {
376 local -a packages=(
377 # build/py.sh all
378 libreadline-dev
379 #python2-dev
380
381 # retrieving spec-bin -- TODO: move to build time
382 wget
383 # for wget https://. TODO: remove when the build is hermetic
384 ca-certificates
385
386 # when spec tests use 'time', dash falls back on 'time' command
387 'time'
388
389 # TODO: probably can remove C++ compiler now that re2c is a wedge
390 gcc
391 g++
392
393 # for cmark
394 cmake
395 # to build Python-2.7.13 (could be a wedge)
396 make
397
398 xz-utils # extract e.g. zsh/yash tarballs
399 bzip2 # extract e.g. busybox tarball
400
401 # for syscall measurements
402 strace
403
404 # used by test/spec-runner.sh
405 gawk
406 )
407
408 apt-install "${packages[@]}"
409}
410
411app-tests() {
412 local -a packages=(
413 # build/py.sh all
414 libreadline-dev
415 python2-dev
416
417 # retrieving spec-bin -- TODO: move to build time
418 wget
419 # for wget https://. TODO: remove when the build is hermetic
420 ca-certificates
421
422 curl # wait for cpp-tarball
423
424 gcc
425 g++ # for C++ tarball
426
427 # to build ble.sh
428 make
429 # used by ble.sh
430 gawk
431 procps
432
433 # for ble.sh contra
434 libx11-dev
435 libxft-dev
436 libncursesw5-dev
437 )
438
439 apt-install "${packages[@]}"
440}
441
442if test $(basename $0) = 'from-apt.sh'; then
443 "$@"
444fi