OILS / devtools / release.sh View on Github | oilshell.org

818 lines, 390 significant
1#!/usr/bin/env bash
2#
3# The big Oils release process.
4#
5# Usage:
6# devtools/release.sh <function name>
7#
8# Steps:
9# edit oil-version.txt, build/doc.sh update-src-versions, bump devtools/release-note.sh
10# $0 make-release-branch
11# $0 two-tarballs # CPython, then oils-for-unix, which is INSTALLED
12# demo/osh-debug.sh osh-for-release: Start a shell to dogfood
13# build/cpython-defs.sh {oil-py-names,filter-methods}
14# (regenerate C source)
15#
16# Run on each machine:
17# $0 auto-machine1
18# $0 auto-machine2 ($0 dep-benchmarks first)
19#
20# In between:
21# [switch benchmarks-data repo] commit src/oil-for-unix-* and push to flanders.
22# TODO: Make sure benchmark-data directory is clean!
23#
24# Resume manual work
25#
26# Commit files to oilshell/benchmark-data repo and sync.
27# benchmarks/report.sh all
28# $0 deploy-tar # needed to publish tarball checksum in HTML
29# build/doc.sh run-for-release
30# $0 compress
31# devtools/release-version.sh git-changelog-$VERSION
32# devtools/release-version.sh announcement-$VERSION
33# MAYBE: ./local.sh test-release-tree if you want to preview it
34# $0 deploy-doc (makes releases.html)
35#
36# demo/osh-debug.sh analyze # see what you ran
37#
38# - Go to oilshell.org repo and do:
39# ./deploy.sh site # copy release
40# ./deploy.sh bump-index-version
41# make
42# ./deploy.sh site # copy new index
43# ./deploy.sh bump-release-version
44# - Go to oilshell.org__deploy and "git add release/$VERSION".
45# - git commit -a
46
47set -o nounset
48set -o pipefail
49set -o errexit
50
51shopt -s strict:all 2>/dev/null || true # dogfood for OSH
52
53REPO_ROOT=$(cd $(dirname $0)/.. ; pwd)
54OIL_VERSION=$(head -n 1 oil-version.txt)
55
56source devtools/common.sh # banner
57source benchmarks/common.sh # BENCHMARK_DATA_OILS, OSH_CPP_BENCHMARK_DATA
58 # redefines OIL_VERSION as readonly
59
60readonly OSH_RELEASE_BINARY=$REPO_ROOT/_tmp/oil-tar-test/oil-$OIL_VERSION/_bin/osh
61readonly YSH_RELEASE_BINARY=$REPO_ROOT/_tmp/oil-tar-test/oil-$OIL_VERSION/_bin/ysh
62
63log() {
64 echo "$@" 1>&2
65}
66
67make-release-branch() {
68 git checkout master
69 local name=release/$OIL_VERSION
70 git checkout -b $name
71 git push -u origin $name
72}
73
74ensure-smooth-build() {
75 # Stray files can mess up the unit tests
76 devtools/git.sh error-if-untracked
77
78 build/clean.sh
79
80 sudo -k; sudo true # clear and re-cache credentials
81
82 # Install with root privileges
83 _install
84}
85
86# For redoing a release. This is everything until you have to 'git pull' the
87# benchmark-data repo to make reports.
88#
89# PRECONDITION: $0 two-tarballs was run manually, which runs
90# ensure-smooth-build.
91auto-machine1() {
92 local resume=${1:-} # workaround for spec test flakiness bug
93 local resume2=${2:-} # skip past spec sanity check
94 local resume3=${3:-} # skip past metrics and wild tests
95 local resume4=${4:-} # skip past full spec tests
96
97 if test -z "$resume"; then
98 $0 build-and-test
99 fi
100
101 if test -z "$resume2"; then
102 _spec-sanity-check # just run a few spec tests
103 fi
104
105 if test -z "$resume3"; then
106 $0 metrics # this can catch bugs
107 test/wild.sh all
108 fi
109
110 if test -z "$resume4"; then
111 $0 spec-all # full spec test run
112 fi
113
114 $0 benchmark-run do_machine1
115}
116
117# Note: needs dep-benchmarks to run
118auto-machine2() {
119 ensure-smooth-build
120
121 # Note: this can't be done until we sync the oils-for-unix source from
122 # machine 1.
123 $0 benchmark-build
124 $0 benchmark-run
125}
126
127# TODO:
128# - enforce that there is a release/$VERSION branch?
129
130# oilshell.org__deploy/
131# releases.html
132# release/
133# $VERSION/
134# index.html # release page, from doc/release-index.md
135# oil-version.txt
136# release-date.txt
137# announcement.html # HTML redirect
138# changelog.html
139# doc/
140# index.html
141# ...
142# test/ # results
143# spec.wwz/
144# machine-lisa/
145# wild.wwz/
146# unit.wwz/
147# other.wwz/
148# gold.txt
149# parse-errors.txt
150# runtime-errors.txt
151# tools-deps.txt
152# osh-usage.txt
153# lossless.txt
154# tarball/ # log of building and running the tarball?
155# asan/ # spec tests or other?
156# # or it can be put under test/{spec,wild}
157# metrics.wwz/ # static metrics on source code?
158# line-counts/
159# nativedeps.txt (build/stats.sh line counts)
160# bytecode size, number of PyCodeObject
161# number of functions, classes, etc.?
162# bytecode/bundle size (binary size on x86_64 is in ovm-build.sh)
163# tarball size?
164# coverage.wwz/
165# unified/ # clang-coverage
166# benchmarks.wwz/
167# compute
168# osh-parser/
169# osh-runtime/
170# vm-baseline/
171# ...
172# startup/
173# download/ # What about native binaries?
174# 0.0.0/
175# oil-0.0.0.tar.xz
176
177_test-tarball() {
178 local name=${1:-hello}
179 local version=${2:-0.0.0}
180 local install=${3:-}
181
182 local tmp=_tmp/${name}-tar-test
183 rm -r -f $tmp
184 mkdir -p $tmp
185
186 pushd $tmp
187 tar --extract -z < ../../_release/$name-$version.tar.gz
188
189 cd $name-$version
190 ./configure
191
192 # Build the fast one for a test.
193 # TODO: Maybe edit the Makefile to change the top target.
194 local bin=_bin/${name}.ovm # not dbg
195 time make $bin
196 $bin --version
197
198 if test -n "$install"; then
199 sudo ./install
200 fi
201 popd
202}
203
204test-oil-tar() {
205 local install=${1:-} # non-empty to install
206 _test-tarball oil $(head -n 1 oil-version.txt) "$install"
207}
208
209_release-build() {
210 # NOTE: deps/from-tar.sh {configure,build}-python is assumed
211
212 # Build the oil tar
213 $0 oil
214
215 test-oil-tar
216
217 # For _spec-sanity-check
218 ln -s -f --no-target-directory -v oil.ovm $OSH_RELEASE_BINARY
219 ln -s -f --no-target-directory -v oil.ovm $YSH_RELEASE_BINARY
220}
221
222readonly HAVE_ROOT=1
223
224readonly -a MORE_TESTS=(
225 process-table
226 gold
227 ysh-ify
228 parse-errors runtime-errors
229 ysh-runtime-errors
230 ysh-parse-errors
231 ysh-every-string
232 lossless
233 osh-usage tools-deps
234 syscall
235)
236
237run-more-tests() {
238 for name in "${MORE_TESTS[@]}"; do
239 case $name in
240 gold)
241 if test -n "${OILS_HIJACK_SHEBANG:-}"; then
242 cat >&2 <<'EOF'
243=====
244WARNING: Skipping gold tests because $OILS_HIJACK_SHEBANG is set.'
245Run them manually with:
246
247 test/gold.sh run-for-release
248=====
249EOF
250 continue
251 fi
252 ;;
253 *)
254 banner "Test suite: $name"
255 ;;
256 esac
257
258 test/$name.sh run-for-release
259 done
260
261 ysh/run.sh run-for-release
262
263 data_lang/j8-errors.sh run-for-release
264}
265
266_spec-sanity-check() {
267 # Quick early test for _bin/osh and _bin/ysh
268
269 # TODO: Use --ovm-bin-dir
270 # Note: MAX_PROCS=1 prevents [#oil-dev > Random Spec Test Stoppages]
271 # Still need to fix that bug
272 MAX_PROCS=1 NUM_SPEC_TASKS=2 OSH_LIST="$OSH_RELEASE_BINARY" test/spec-py.sh osh-all
273 MAX_PROCS=1 NUM_SPEC_TASKS=2 YSH_LIST="$YSH_RELEASE_BINARY" test/spec-py.sh ysh-all
274}
275
276spec-all() {
277 ### Run all spec tests
278
279 test/stateful.sh soil-run # Same as CI
280
281 # Create the tests we're running
282 test/smoosh.sh make-spec
283
284 # TODO: Use --ovm-bin-dir
285 export OSH_LIST="$REPO_ROOT/bin/osh $OSH_RELEASE_BINARY"
286 export YSH_LIST="$REPO_ROOT/bin/ysh $YSH_RELEASE_BINARY"
287 test/spec-py.sh all-and-smoosh
288
289 # Build $OSH_CPP_BENCHMARK_DATA
290 _build-oils-benchmark-data
291
292 # TODO: Use --oils-cpp-bin-dir
293 # Collect and publish stats about the C++ translation.
294 OSH_CC="$OSH_CPP_BENCHMARK_DATA" test/spec-cpp.sh osh-all
295 YSH_CC="$YSH_CPP_BENCHMARK_DATA" test/spec-cpp.sh ysh-all
296}
297
298spec-cpp() {
299 ### For repair
300
301 # TODO: Use --oils-cpp-bin-dir
302
303 # Quick
304 # NUM_SPEC_TASKS=2 OSH_CC="$OSH_CPP_BENCHMARK_DATA" test/spec-cpp.sh all
305 OSH_CC="$OSH_CPP_BENCHMARK_DATA" test/spec-cpp.sh all
306}
307
308# For quickly debugging failures that don't happen in dev mode.
309spec-one() {
310 export OSH_LIST="$REPO_ROOT/bin/osh $OSH_RELEASE_BINARY"
311 export YSH_LIST="$REPO_ROOT/bin/ysh $YSH_RELEASE_BINARY"
312 test/spec.sh "$@"
313}
314
315build-and-test() {
316 ### Build tarballs and test them. And preliminaries like unit tests.
317
318 # TODO: Log this whole thing? Include logs with the /release/ page?
319
320 # Before doing anything
321 test/lint.sh soil-run
322
323 test/unit.sh run-for-release # Python unit tests
324
325 test/coverage.sh run-for-release # C++ unit tests
326
327 # App bundle
328 _release-build
329
330 # TODO: test oils-for-unix in Alpine chroot too.
331 # NOTE: Need test/alpine.sh download;extract;setup-dns,add-oil-build-deps,
332 # etc.
333 if test -n "$HAVE_ROOT"; then
334 # TODO: Factor out test/alpine.sh to test/chroot.sh
335 test/alpine.sh copy-tar '' oil
336 test/alpine.sh test-tar '' oil
337 fi
338
339 test/spec.sh smoke # Initial smoke test, slightly redundant.
340
341 run-more-tests
342}
343
344_install() {
345 test/spec-bin.sh install-shells-with-apt
346
347 # A subset of build/py.sh ubuntu-deps. (Do we need build-essential?)
348 #sudo apt install python-dev
349}
350
351_build-oils-benchmark-data() {
352 pushd $BENCHMARK_DATA_OILS
353 ./configure
354 for variant in dbg opt; do
355 # DWARF version 4 is a hack for bloaty, which doesn't support version 5.
356 # I don't think this should affect benchmarks besides
357 # metrics/native-code.sh, so we don't bother building a separate binary.
358 # The Soil CI runs without this flag.
359 CXXFLAGS=-gdwarf-4 _build/oils.sh '' $variant SKIP_REBUILD
360 done
361 popd
362}
363
364benchmark-build() {
365 ### Build function on machine 2.
366
367 build/clean.sh
368 if test -n "$HAVE_ROOT"; then
369 _install
370 fi
371 build/py.sh all
372 _release-build
373}
374
375# Run benchmarks with the binary built out of the tarball.
376benchmark-run() {
377 local do_machine1=${1:-}
378
379 _build-oils-benchmark-data
380 OSH_OVM=$OSH_RELEASE_BINARY benchmarks/auto.sh all "$do_machine1"
381}
382
383_compressed-tarball() {
384 local name=${1:-hello}
385 local version=${2:-0.0.0}
386
387 local in=_release/$name.tar
388 local out=_release/$name-$version.tar.gz
389
390 # Overwrite it to cause rebuild of oil.tar
391 build/stamp.sh write-release-date
392
393 #make -d -r $in # To debug
394 make $in
395 time gzip -c $in > $out
396 ls -l $out
397
398 # xz version is considerably smaller. 1.15 MB vs. 1.59 MB.
399 local out2=_release/$name-$version.tar.xz
400 time xz -c $in > $out2
401 ls -l $out2
402}
403
404oil() {
405 _compressed-tarball oil $OIL_VERSION
406}
407
408hello() {
409 _compressed-tarball hello $(head -n 1 build/testdata/hello-version.txt)
410}
411
412
413_link() {
414 ln -s -f -v --no-target-directory "$@"
415}
416
417compress() {
418 local root=$PWD/_release/VERSION/
419
420 log '--- more-tests'
421 local out="$root/more-tests.wwz"
422 pushd _tmp
423 time zip -r -q $out suite-logs unit syscall process-table
424 popd
425
426 # This has HTML reports, .profraw files, and logs of stdout, e.g.
427 # mycpp-unit/gc_heap_test.log
428 # About 1.5 MB
429 log "--- coverage"
430 local out="$root/test/coverage.wwz"
431 pushd _test/clang-coverage
432 # This also saves the logs
433 time zip -r -q $out .
434 popd
435
436 log "--- test/spec"
437 local out="$root/test/spec.wwz"
438 pushd _tmp/spec
439 time zip -r -q $out . # recursive, quiet
440 popd
441
442 log "--- test/wild"
443 local out="$root/test/wild.wwz"
444 pushd _tmp/wild-www
445 time zip -r -q $out . # recursive, quiet
446 popd
447
448 # NOTE: must be /pub/metrics.wwz so that relative URLs like
449 # ../../../web/line-counts.css work. The Soil UI also relies on such
450 # relative URLs.
451 log "--- metrics"
452 local out="$root/pub/metrics.wwz"
453 pushd _tmp/metrics
454 time zip -r -q $out . # recursive, quiet
455 popd
456
457 # Ditto: pub/src-tree.wwz lines up with URLs in Soil
458 log "--- src-tree"
459 local out="$root/pub/src-tree.wwz"
460 pushd _tmp/src-tree-www
461 time zip -r -q $out . # recursive, quiet
462 popd
463
464 compress-benchmarks
465
466 tree _release/VERSION
467}
468
469compress-benchmarks() {
470 local root=$PWD/_release/VERSION/
471 mkdir -p $root
472
473 log "--- benchmarks"
474
475 local out="$root/benchmarks.wwz"
476
477 # - For benchmarks that run on multiple machines, technically we only need
478 # index.html, but include stage1 and stage2.
479 # - For those that run on single machines, we also archive the raw/ dir.
480 # - Although benchmarks/compute is saved in oilshell/benchmark-data
481 # - Note: _tmp/uftrace/{raw,stage1} are big (hundreds of MB), so leave them
482 # out
483
484 pushd _tmp
485 find \
486 osh-parser/{stage1,stage2,index.html} \
487 osh-runtime/{stage1,stage2,index.html} \
488 vm-baseline/{stage1,stage2,index.html} \
489 ovm-build/{stage1,stage2,index.html} \
490 compute/{raw,stage1,stage2,index.html} \
491 gc/{raw,stage2,index.html} \
492 gc-cachegrind/{raw,stage2,index.html} \
493 mycpp-examples/{raw,stage2,index.html} \
494 uftrace/{stage2,index.html} \
495 -type f \
496 | xargs --verbose -- zip -q $out
497 popd
498}
499
500line-counts() {
501 local out_dir=$1 # should be an absolute path
502 mkdir -p $out_dir
503
504 # Counting directly from the build.
505 metrics/tarball.sh linecount-pydeps > $out_dir/pydeps.txt
506 metrics/tarball.sh linecount-nativedeps > $out_dir/nativedeps.txt
507 metrics/tarball.sh linecount-oils-cpp > $out_dir/oils-cpp.txt
508
509 metrics/source-code.sh write-reports $out_dir # for-translation and overview
510 metrics/source-code.sh cloc-report > $out_dir/cloc-report.txt
511
512 # goes to _tmp/metrics/preprocessed
513 metrics/source-code.sh preprocessed
514}
515
516metrics() {
517 local out=_tmp/metrics
518 mkdir -p $out
519
520 line-counts $PWD/$out/line-counts
521
522 # For another .wwz file
523 doctools/src-tree.sh soil-run
524
525 metrics/bytecode.sh run-for-release
526 metrics/native-code.sh run-for-release
527 build/cpython-defs.sh run-for-release
528
529 tree $out
530}
531
532deploy-doc() {
533 local deploy_repo='../oilshell.org__deploy'
534 local release_root_dir="$deploy_repo/release"
535 local release_dir="$release_root_dir/$OIL_VERSION"
536
537 cp -v -r --force --no-target-directory \
538 _release/VERSION/ $release_dir/
539
540 # Generate release index.
541 html-index $release_root_dir _tmp/releases.html
542 cp -v _tmp/releases.html $deploy_repo
543
544 tree -L 3 $release_root_dir
545
546 ls -l $deploy_repo/releases.html
547}
548
549readonly DOWNLOAD_DIR='../oilshell.org__deploy/download/'
550
551# Generating releases.html requires the old tarballs!
552sync-old-tar() {
553 local user=$1 # required username
554 rsync --archive --verbose \
555 $user@oilshell.org:oilshell.org/download/ $DOWNLOAD_DIR
556}
557
558# I think these aren't checked into git? They can just be managed separately?
559# Or should you check in the sha checksums? Those will be in releases.html,
560# but a CSV might be nice.
561deploy-tar() {
562 mkdir -p $DOWNLOAD_DIR
563
564 cp -v \
565 _release/oil-$OIL_VERSION.tar.* _release/oils-for-unix-$OIL_VERSION.tar.* \
566 $DOWNLOAD_DIR
567
568 ls -l $DOWNLOAD_DIR
569}
570
571#
572# Generate releases.html.
573#
574
575# Examples of similar release HTML pages:
576# - https://golang.org/dl/ - "Older versions", sha1 / sha256.
577# - Python has all point releases in chronological order, and then a separate
578# page for each changelog. There is too much boilerplate maybe?
579# - It has release notes before the downloads. Not sure I like that.
580# - node.js: https://nodejs.org/en/
581# - user agent detection for the right binary -- meh I don't want that
582# - Ruby: https://www.ruby-lang.org/en/downloads/releases/
583# - https://www.lua.org/download.html
584
585# Columns: Date / Version / Docs / / Files
586# Changelog .xz
587# Install
588# Docs/
589#
590# The files could be a separate page and separate table? I could provide
591# pre-built versions eventually? Linux static versions?
592
593# TODO: Each of these would be a good candidate for a data frame! Data vs.
594# presentation.
595
596# Simple UI:
597# - home page shows latest version (source release for now, binary release later?)
598# - link to Changelog, INSTALL, doc index
599# - or see all releases
600# - Grey out older releases?
601
602# TODO: Should be sorted by date? How to do that, with bash array? Or Awk?
603# $timestamp $version $timestamp file? And then sort -n I guess? Change
604# the release date format. It will use Unix timestamp (OK until 2038!)
605
606_html-index() {
607 local release_root_dir=$1 # the directory we want to make an index of
608
609 for entry in $release_root_dir/*; do
610 if ! test -d $entry; then
611 continue
612 fi
613 local dir=$entry
614
615 local version
616 version=$(head -n 1 $dir/oil-version.txt)
617 local release_date
618 release_date=$(head -n 1 $dir/release-date.txt)
619
620 log "-- $dir"
621 log "Version: $version"
622 log "Release Date: $release_date"
623 log ""
624
625 echo "$release_date $version"
626 done > _tmp/release-meta.txt
627
628 # Reverse sort by release date
629 sort -r _tmp/release-meta.txt > _tmp/sorted-releases.txt
630
631 while read date _ version; do
632 log "Release Date: $date"
633 log "Version: $version"
634
635 # anchor
636 cat <<EOF
637<tr>
638 <td>
639 <span class="date">$date</span>
640 </td>
641 <td>
642 <a name="$version"></a>
643 <span class="version-number">$version</span>
644 </td>
645 <td>
646 <p> <a href="release/$version/announcement.html">Announcement</a>
647 &nbsp; | &nbsp; <a href="release/$version/">Docs and Details</a>
648 </p>
649 </td>
650</tr>
651EOF
652
653 build/doc.sh tarball-links-row-html $version
654
655 cat <<EOF
656<tr>
657 <td colspan="3">
658 <div style="padding: 1em;" >
659 </div>
660 </td>
661</tr>
662
663EOF
664
665 done < _tmp/sorted-releases.txt
666}
667
668_releases-html-header() {
669 # TODO: use html-head here, and publish web/*.css somewhere outside of
670 # /release/$VERSION/? The list of all releases isn't versioned for obvious
671 # reasons. Other docs are in the oilshell.org repo using the all-2020.css
672 # bundle.
673
674 cat <<EOF
675<!DOCTYPE html>
676<html>
677 <head>
678 <meta name="viewport" content="width=device-width, initial-scale=1">
679 <title>Oils Releases</title>
680 <style>
681EOF
682
683 cat web/base.css
684 cat web/release-index.css
685
686cat <<EOF
687 h1 {
688 text-align: center;
689 }
690 </style>
691 </head>
692 <body class="width50">
693 <p id="home-link">
694 <a href="/">oilshell.org</a>
695 </p>
696 <h1>Oils Releases</h1>
697
698 <table class="release-table">
699EOF
700}
701
702html-index() {
703 local release_root_dir=$1
704 local out=${2:-_tmp/releases.html}
705
706 { _releases-html-header
707 _html-index $release_root_dir
708
709 cat <<EOF
710 </table>
711 </body>
712</html>
713EOF
714
715 } > $out
716
717 ls -l $out
718}
719
720# For quickly iterating on tarball size reductions.
721tarball-size() {
722 make clean-repo
723 make _bin/oil.ovm-dbg # faster way to build bytecode
724 oil # make tarball
725 test-oil-tar # Ctrl-C this, then run metrics/tarball.sh
726}
727
728dep-smoosh() {
729 local repo=~/git/languages/smoosh
730 if ! test -d $repo; then
731 local base_dir=$(dirname $repo)
732 mkdir -p $base_dir
733 pushd $base_dir
734 git clone git@github.com:mgree/smoosh.git
735 popd
736 fi
737}
738
739dep-benchmarks() {
740 ### Before auto-machine2
741
742 # 2023-07: Also need deps/from-tar.sh {configure,build}-cpython
743
744 benchmarks/osh-runtime.sh download
745 benchmarks/osh-runtime.sh extract
746
747 benchmarks/ovm-build.sh download
748 benchmarks/ovm-build.sh extract-other
749
750 # For ovm-build benchmark.
751 deps/from-binary.sh download-clang
752 deps/from-binary.sh extract-clang
753}
754
755more-release-deps() {
756 # List of deps that are NOT in soil/worker.sh here
757 # https://github.com/oilshell/oil/issues/926
758
759 # TODO: Make a container image for these.
760 if false; then
761 # TODO: Did this manually
762 # test/alpine.sh
763 # dep-alpine
764
765 # test/smoosh.sh
766 dep-smoosh
767
768 dep-benchmarks
769 fi
770}
771
772py-tarball() {
773 local in=_release/oil.tar
774 local out=_release/oil-$OIL_VERSION.tar.gz
775
776 make $in
777 time gzip -c $in > $out
778 ls -l $out
779
780 test-oil-tar
781}
782
783native-tarball() {
784 # oils-for-unix
785 devtools/release-native.sh make-tar
786 # Also install as root
787 devtools/release-native.sh extract-for-benchmarks INSTALL
788}
789
790two-tarballs() {
791 ### First step of release. Assume that CI passes
792
793 ensure-smooth-build
794
795 build/py.sh all
796 # "Base state" for repo scripts
797 ./NINJA-config.sh
798
799 py-tarball
800
801 native-tarball
802}
803
804upload-tmp() {
805 local tarball=$1
806 local user=$2
807
808 scp $tarball $user@oilshell.org:tmp/
809}
810
811sync-tmp() {
812 local user=$1
813 local dest=${2:-_tmp/candidates}
814 mkdir -p $dest
815 rsync --archive --verbose $user@oilshell.org:tmp/ $dest
816}
817
818"$@"