OILS / devtools / release.sh View on Github | oils.pub

835 lines, 395 significant
1#!/usr/bin/env bash
2#
3# The big Oils release process.
4#
5# Usage:
6# devtools/release.sh <function name>
7#
8# Steps:
9# edit oils-version.txt, build/doc.sh update-src-versions, and
10# bump devtools/release-note.sh
11# $0 make-release-branch
12# $0 two-tarballs # CPython, then oils-for-unix, which is INSTALLED
13# demo/osh-debug.sh osh-for-release: Start a shell to dogfood
14# build/cpython-defs.sh {oil-py-names,filter-methods}
15# (regenerate C source)
16#
17# Run on each machine:
18# $0 auto-machine1
19# $0 auto-machine2 ($0 dep-benchmarks first)
20#
21# In between:
22# [switch benchmarks-data repo] commit src/oil-for-unix-* and push to flanders.
23# TODO: Make sure benchmark-data directory is clean!
24#
25# Resume manual work
26#
27# Commit files to oilshell/benchmark-data repo and sync.
28# benchmarks/report.sh all
29# $0 deploy-tar # needed to publish tarball checksum in HTML
30# build/doc.sh run-for-release
31# $0 compress
32# devtools/release-version.sh git-changelog-$VERSION
33# devtools/release-version.sh announcement-$VERSION
34# MAYBE: ./local.sh test-release-tree if you want to preview it
35# $0 deploy-doc (makes releases.html)
36#
37# demo/osh-debug.sh analyze # see what you ran
38#
39# - Go to oils.pub repo and do:
40# ./deploy.sh site # copy release
41# ./deploy.sh bump-index-version
42# make
43# ./deploy.sh site # copy new index
44# ./deploy.sh bump-release-version
45# - Go to oils.pub__deploy and "git add release/$VERSION".
46# - git commit -a
47
48set -o nounset
49set -o pipefail
50set -o errexit
51
52shopt -s strict:all 2>/dev/null || true # dogfood for OSH
53
54REPO_ROOT=$(cd $(dirname $0)/.. ; pwd)
55OIL_VERSION=$(head -n 1 oils-version.txt)
56
57source devtools/common.sh # banner
58source benchmarks/common.sh # BENCHMARK_DATA_OILS, OSH_CPP_BENCHMARK_DATA
59 # redefines OIL_VERSION as readonly
60
61readonly OSH_RELEASE_BINARY=$REPO_ROOT/_tmp/oil-tar-test/oil-$OIL_VERSION/_bin/osh
62readonly YSH_RELEASE_BINARY=$REPO_ROOT/_tmp/oil-tar-test/oil-$OIL_VERSION/_bin/ysh
63
64log() {
65 echo "$@" 1>&2
66}
67
68make-release-branch() {
69 git checkout master
70 local name=release/$OIL_VERSION
71 git checkout -b $name
72 git push -u origin $name
73}
74
75ensure-smooth-build() {
76 # Stray files can mess up the unit tests
77 devtools/git.sh error-if-untracked
78
79 build/clean.sh
80
81 sudo -k; sudo true # clear and re-cache credentials
82
83 # Install with root privileges
84 _install
85}
86
87# For redoing a release. This is everything until you have to 'git pull' the
88# benchmark-data repo to make reports.
89#
90# PRECONDITION: $0 two-tarballs was run manually, which runs
91# ensure-smooth-build.
92auto-machine1() {
93 local resume=${1:-} # workaround for spec test flakiness bug
94 local resume2=${2:-} # skip past spec sanity check
95 local resume3=${3:-} # skip past metrics and wild tests
96 local resume4=${4:-} # skip past full spec tests
97
98 if test -z "$resume"; then
99 $0 build-and-test
100 fi
101
102 if test -z "$resume2"; then
103 _spec-sanity-check # just run a few spec tests
104 fi
105
106 if test -z "$resume3"; then
107 $0 metrics # this can catch bugs
108 test/wild.sh all
109 fi
110
111 if test -z "$resume4"; then
112 $0 spec-all # full spec test run
113 fi
114
115 $0 benchmark-run do_machine1
116}
117
118# Note: needs dep-benchmarks to run
119auto-machine2() {
120 ensure-smooth-build
121
122 # Note: this can't be done until we sync the oils-for-unix source from
123 # machine 1.
124 $0 benchmark-build
125 $0 benchmark-run
126}
127
128# TODO:
129# - enforce that there is a release/$VERSION branch?
130
131# oils.pub__deploy/
132# releases.html
133# release/
134# $VERSION/
135# index.html # release page, from doc/release-index.md
136# oils-version.txt
137# release-date.txt
138# announcement.html # HTML redirect
139# changelog.html
140# doc/
141# index.html
142# ...
143# test/ # results
144# spec.wwz/
145# machine-lisa/
146# wild.wwz/
147# unit.wwz/
148# other.wwz/
149# gold.txt
150# parse-errors.txt
151# runtime-errors.txt
152# tools-deps.txt
153# osh-usage.txt
154# lossless.txt
155# tarball/ # log of building and running the tarball?
156# asan/ # spec tests or other?
157# # or it can be put under test/{spec,wild}
158# metrics.wwz/ # static metrics on source code?
159# line-counts/
160# nativedeps.txt (build/stats.sh line counts)
161# bytecode size, number of PyCodeObject
162# number of functions, classes, etc.?
163# bytecode/bundle size (binary size on x86_64 is in ovm-build.sh)
164# tarball size?
165# coverage.wwz/
166# unified/ # clang-coverage
167# benchmarks.wwz/
168# compute
169# osh-parser/
170# osh-runtime/
171# vm-baseline/
172# ...
173# startup/
174# download/ # What about native binaries?
175# 0.0.0/
176# oil-0.0.0.tar.xz
177
178_test-tarball() {
179 local name=${1:-hello}
180 local version=${2:-0.0.0}
181 local install=${3:-}
182
183 local tmp=_tmp/${name}-tar-test
184 rm -r -f $tmp
185 mkdir -p $tmp
186
187 pushd $tmp
188 tar --extract -z < ../../_release/$name-$version.tar.gz
189
190 cd $name-$version
191 ./configure
192
193 # Build the fast one for a test.
194 # TODO: Maybe edit the Makefile to change the top target.
195 local bin=_bin/${name}.ovm # not dbg
196 time make $bin
197 $bin --version
198
199 if test -n "$install"; then
200 sudo ./install
201 fi
202 popd
203}
204
205test-oil-tar() {
206 local install=${1:-} # non-empty to install
207 _test-tarball oil $(head -n 1 oils-version.txt) "$install"
208}
209
210_release-build() {
211 # NOTE: deps/from-tar.sh {configure,build}-python is assumed
212
213 # Build the oil tar
214 $0 oil
215
216 test-oil-tar
217
218 # For _spec-sanity-check
219 ln -s -f --no-target-directory -v oil.ovm $OSH_RELEASE_BINARY
220 ln -s -f --no-target-directory -v oil.ovm $YSH_RELEASE_BINARY
221}
222
223readonly HAVE_ROOT=1
224
225readonly -a MORE_TESTS=(
226 process-table
227 gold
228 ysh-ify
229 parse-errors runtime-errors
230 ysh-runtime-errors
231 ysh-parse-errors
232 ysh-every-string
233 lossless
234 osh-usage tools-deps
235 syscall
236)
237# TODO: Unify with CI, and clean up
238# doc/error-catalog.sh
239# data_lang/j8-errors.sh
240# ysh/run.sh
241
242run-more-tests() {
243 for name in "${MORE_TESTS[@]}"; do
244 case $name in
245 gold)
246 if test -n "${OILS_HIJACK_SHEBANG:-}"; then
247 cat >&2 <<'EOF'
248=====
249WARNING: Skipping gold tests because $OILS_HIJACK_SHEBANG is set.'
250Run them manually with:
251
252 test/gold.sh run-for-release
253=====
254EOF
255 continue
256 fi
257 ;;
258 *)
259 banner "Test suite: $name"
260 ;;
261 esac
262
263 test/$name.sh run-for-release
264 done
265
266 ysh/run.sh run-for-release
267
268 data_lang/j8-errors.sh run-for-release
269}
270
271_spec-sanity-check() {
272 # Quick early test for _bin/osh and _bin/ysh
273
274 # TODO: Use --ovm-bin-dir
275 # Note: MAX_PROCS=1 prevents [#oil-dev > Random Spec Test Stoppages]
276 # Still need to fix that bug
277 MAX_PROCS=1 NUM_SPEC_TASKS=2 OSH_LIST="$OSH_RELEASE_BINARY" test/spec-py.sh osh-all
278 MAX_PROCS=1 NUM_SPEC_TASKS=2 YSH_LIST="$YSH_RELEASE_BINARY" test/spec-py.sh ysh-all
279}
280
281spec-all() {
282 ### Run all spec tests
283
284 test/stateful.sh soil-run # Same as CI
285
286 # Create the tests we're running
287 test/smoosh.sh make-spec
288
289 # TODO: Use --ovm-bin-dir
290 export OSH_LIST="$REPO_ROOT/bin/osh $OSH_RELEASE_BINARY"
291 export YSH_LIST="$REPO_ROOT/bin/ysh $YSH_RELEASE_BINARY"
292 test/spec-py.sh all-and-smoosh
293
294 # Build $OSH_CPP_BENCHMARK_DATA
295 _build-oils-benchmark-data
296
297 # TODO: Use --oils-cpp-bin-dir
298 # Collect and publish stats about the C++ translation.
299 OSH_CC="$OSH_CPP_BENCHMARK_DATA" test/spec-cpp.sh osh-all
300 YSH_CC="$YSH_CPP_BENCHMARK_DATA" test/spec-cpp.sh ysh-all
301}
302
303spec-cpp() {
304 ### For repair
305
306 # TODO: Use --oils-cpp-bin-dir
307
308 # Quick
309 # NUM_SPEC_TASKS=2 OSH_CC="$OSH_CPP_BENCHMARK_DATA" test/spec-cpp.sh all
310 OSH_CC="$OSH_CPP_BENCHMARK_DATA" test/spec-cpp.sh all
311}
312
313# For quickly debugging failures that don't happen in dev mode.
314spec-one() {
315 export OSH_LIST="$REPO_ROOT/bin/osh $OSH_RELEASE_BINARY"
316 export YSH_LIST="$REPO_ROOT/bin/ysh $YSH_RELEASE_BINARY"
317 test/spec.sh "$@"
318}
319
320build-and-test() {
321 ### Build tarballs and test them. And preliminaries like unit tests.
322
323 # TODO: Log this whole thing? Include logs with the /release/ page?
324
325 # Before doing anything
326 test/lint.sh soil-run
327
328 test/unit.sh run-for-release # Python unit tests
329
330 test/coverage.sh run-for-release # C++ unit tests
331
332 # App bundle
333 _release-build
334
335 # TODO: test oils-for-unix in Alpine chroot too.
336 # NOTE: Need test/alpine.sh download;extract;setup-dns,add-oil-build-deps,
337 # etc.
338 if test -n "$HAVE_ROOT"; then
339 # TODO: Factor out test/alpine.sh to test/chroot.sh
340 test/alpine.sh copy-tar '' oil
341 test/alpine.sh test-tar '' oil
342 fi
343
344 test/spec.sh smoke # Initial smoke test, slightly redundant.
345
346 run-more-tests
347}
348
349_install() {
350 test/spec-bin.sh install-shells-with-apt
351
352 # A subset of build/py.sh ubuntu-deps. (Do we need build-essential?)
353 #sudo apt install python-dev
354}
355
356_build-oils-benchmark-data() {
357 pushd $BENCHMARK_DATA_OILS
358 ./configure
359 for variant in dbg opt; do
360 # DWARF version 4 is a hack for bloaty, which doesn't support version 5.
361 # I don't think this should affect benchmarks besides
362 # metrics/native-code.sh, so we don't bother building a separate binary.
363 # The Soil CI runs without this flag.
364 CXXFLAGS=-gdwarf-4 _build/oils.sh --variant "$variant" --skip-rebuild
365 done
366 popd
367}
368
369benchmark-build() {
370 ### Build function on machine 2.
371
372 build/clean.sh
373 if test -n "$HAVE_ROOT"; then
374 _install
375 fi
376
377 build/py.sh all # runs configure-for-dev
378 configure-for-release
379
380 _release-build
381}
382
383# Run benchmarks with the binary built out of the tarball.
384benchmark-run() {
385 local do_machine1=${1:-}
386
387 _build-oils-benchmark-data
388 OSH_OVM=$OSH_RELEASE_BINARY benchmarks/auto.sh all "$do_machine1"
389}
390
391_compressed-tarball() {
392 local name=${1:-hello}
393 local version=${2:-0.0.0}
394
395 local in=_release/$name.tar
396 local out=_release/$name-$version.tar.gz
397
398 # Overwrite it to cause rebuild of oil.tar
399 build/stamp.sh write-release-date
400
401 #make -d -r $in # To debug
402 make $in
403 time gzip -c $in > $out
404 ls -l $out
405
406 # xz version is considerably smaller. 1.15 MB vs. 1.59 MB.
407 local out2=_release/$name-$version.tar.xz
408 time xz -c $in > $out2
409 ls -l $out2
410}
411
412oil() {
413 _compressed-tarball oil $OIL_VERSION
414}
415
416hello() {
417 _compressed-tarball hello $(head -n 1 build/testdata/hello-version.txt)
418}
419
420
421_link() {
422 ln -s -f -v --no-target-directory "$@"
423}
424
425compress() {
426 local root=$PWD/_release/VERSION/
427
428 log '--- more-tests'
429 local out="$root/more-tests.wwz"
430 pushd _tmp
431 time zip -r -q $out suite-logs unit syscall process-table
432 popd
433
434 # This has HTML reports, .profraw files, and logs of stdout, e.g.
435 # mycpp-unit/gc_heap_test.log
436 # About 1.5 MB
437 log "--- coverage"
438 local out="$root/test/coverage.wwz"
439 pushd _test/clang-coverage
440 # This also saves the logs
441 time zip -r -q $out .
442 popd
443
444 log "--- test/spec"
445 local out="$root/test/spec.wwz"
446 pushd _tmp/spec
447 time zip -r -q $out . # recursive, quiet
448 popd
449
450 log "--- test/wild"
451 local out="$root/test/wild.wwz"
452 pushd _tmp/wild-www
453 time zip -r -q $out . # recursive, quiet
454 popd
455
456 # NOTE: must be /pub/metrics.wwz so that relative URLs like
457 # ../../../web/line-counts.css work. The Soil UI also relies on such
458 # relative URLs.
459 log "--- metrics"
460 local out="$root/pub/metrics.wwz"
461 pushd _tmp/metrics
462 time zip -r -q $out . # recursive, quiet
463 popd
464
465 # Ditto: pub/src-tree.wwz lines up with URLs in Soil
466 log "--- src-tree"
467 local out="$root/pub/src-tree.wwz"
468 pushd _tmp/src-tree-www
469 time zip -r -q $out . # recursive, quiet
470 popd
471
472 compress-benchmarks
473
474 tree _release/VERSION
475}
476
477compress-benchmarks() {
478 local root=$PWD/_release/VERSION/
479 mkdir -p $root
480
481 log "--- benchmarks"
482
483 local out="$root/benchmarks.wwz"
484
485 # - For benchmarks that run on multiple machines, technically we only need
486 # index.html, but include stage1 and stage2.
487 # - For those that run on single machines, we also archive the raw/ dir.
488 # - Although benchmarks/compute is saved in oilshell/benchmark-data
489 # - Note: _tmp/uftrace/{raw,stage1} are big (hundreds of MB), so leave them
490 # out
491
492 pushd _tmp
493 find \
494 osh-parser/{stage1,stage2,index.html} \
495 osh-runtime/{stage1,stage2,index.html} \
496 vm-baseline/{stage1,stage2,index.html} \
497 ovm-build/{stage1,stage2,index.html} \
498 compute/{raw,stage1,stage2,index.html} \
499 gc/{raw,stage2,index.html} \
500 gc-cachegrind/{raw,stage2,index.html} \
501 mycpp-examples/{raw,stage2,index.html} \
502 uftrace/{stage2,index.html} \
503 -type f \
504 | xargs --verbose -- zip -q $out
505 popd
506}
507
508line-counts() {
509 local out_dir=$1 # should be an absolute path
510 mkdir -p $out_dir
511
512 # Counting directly from the build.
513 metrics/tarball.sh linecount-pydeps > $out_dir/pydeps.txt
514 metrics/tarball.sh linecount-nativedeps > $out_dir/nativedeps.txt
515 metrics/tarball.sh linecount-oils-cpp > $out_dir/oils-cpp.txt
516
517 metrics/source-code.sh write-reports $out_dir # for-translation and overview
518 metrics/source-code.sh cloc-report > $out_dir/cloc-report.txt
519
520 # goes to _tmp/metrics/preprocessed
521 metrics/source-code.sh preprocessed
522}
523
524metrics() {
525 local out=_tmp/metrics
526 mkdir -p $out
527
528 line-counts $PWD/$out/line-counts
529
530 # For another .wwz file
531 doctools/src-tree.sh soil-run
532
533 metrics/bytecode.sh run-for-release
534 metrics/native-code.sh run-for-release
535 # Disabled 2024-12
536 # build/cpython-defs.sh run-for-release
537
538 tree $out
539}
540
541deploy-doc() {
542 local deploy_repo='../oils.pub__deploy'
543 local release_root_dir="$deploy_repo/release"
544 local release_dir="$release_root_dir/$OIL_VERSION"
545
546 mkdir -p $release_dir
547 cp -v -r --force --no-target-directory \
548 _release/VERSION/ $release_dir/
549
550 # Generate release index.
551 html-index $release_root_dir _tmp/releases.html
552 cp -v _tmp/releases.html $deploy_repo
553
554 tree -L 3 $release_root_dir
555
556 ls -l $deploy_repo/releases.html
557}
558
559readonly DOWNLOAD_DIR='../oils.pub__deploy/download/'
560
561# Generating releases.html requires the old tarballs!
562sync-old-tar() {
563 local user=$1 # required username
564 rsync --archive --verbose \
565 $user@oilshell.org:oilshell.org/download/ $DOWNLOAD_DIR
566}
567
568# I think these aren't checked into git? They can just be managed separately?
569# Or should you check in the sha checksums? Those will be in releases.html,
570# but a CSV might be nice.
571deploy-tar() {
572 mkdir -p $DOWNLOAD_DIR
573
574 cp -v \
575 _release/oil-$OIL_VERSION.tar.* _release/oils-for-unix-$OIL_VERSION.tar.* \
576 $DOWNLOAD_DIR
577
578 ls -l $DOWNLOAD_DIR
579}
580
581#
582# Generate releases.html.
583#
584
585# Examples of similar release HTML pages:
586# - https://golang.org/dl/ - "Older versions", sha1 / sha256.
587# - Python has all point releases in chronological order, and then a separate
588# page for each changelog. There is too much boilerplate maybe?
589# - It has release notes before the downloads. Not sure I like that.
590# - node.js: https://nodejs.org/en/
591# - user agent detection for the right binary -- meh I don't want that
592# - Ruby: https://www.ruby-lang.org/en/downloads/releases/
593# - https://www.lua.org/download.html
594
595# Columns: Date / Version / Docs / / Files
596# Changelog .xz
597# Install
598# Docs/
599#
600# The files could be a separate page and separate table? I could provide
601# pre-built versions eventually? Linux static versions?
602
603# TODO: Each of these would be a good candidate for a data frame! Data vs.
604# presentation.
605
606# Simple UI:
607# - home page shows latest version (source release for now, binary release later?)
608# - link to Changelog, INSTALL, doc index
609# - or see all releases
610# - Grey out older releases?
611
612# TODO: Should be sorted by date? How to do that, with bash array? Or Awk?
613# $timestamp $version $timestamp file? And then sort -n I guess? Change
614# the release date format. It will use Unix timestamp (OK until 2038!)
615
616_html-index() {
617 local release_root_dir=$1 # the directory we want to make an index of
618
619 for entry in $release_root_dir/*; do
620 if ! test -d $entry; then
621 continue
622 fi
623 local dir=$entry
624
625 local version
626 version=$(head -n 1 $dir/oils-version.txt)
627 local release_date
628 release_date=$(head -n 1 $dir/release-date.txt)
629
630 log "-- $dir"
631 log "Version: $version"
632 log "Release Date: $release_date"
633 log ""
634
635 echo "$release_date $version"
636 done > _tmp/release-meta.txt
637
638 # Reverse sort by release date
639 sort -r _tmp/release-meta.txt > _tmp/sorted-releases.txt
640
641 while read date _ version; do
642 log "Release Date: $date"
643 log "Version: $version"
644
645 # anchor
646 cat <<EOF
647<tr>
648 <td>
649 <span class="date">$date</span>
650 </td>
651 <td>
652 <a name="$version"></a>
653 <span class="version-number">$version</span>
654 </td>
655 <td>
656 <p> <a href="release/$version/announcement.html">Announcement</a>
657 &nbsp; | &nbsp; <a href="release/$version/">Docs and Details</a>
658 </p>
659 </td>
660</tr>
661EOF
662
663 build/doc.sh tarball-links-row-html $version
664
665 cat <<EOF
666<tr>
667 <td colspan="3">
668 <div style="padding: 1em;" >
669 </div>
670 </td>
671</tr>
672
673EOF
674
675 done < _tmp/sorted-releases.txt
676}
677
678_releases-html-header() {
679 # TODO: use html-head here, and publish web/*.css somewhere outside of
680 # /release/$VERSION/? The list of all releases isn't versioned for obvious
681 # reasons. Other docs are in the oilshell.org repo using the all-2020.css
682 # bundle.
683
684 cat <<EOF
685<!DOCTYPE html>
686<html>
687 <head>
688 <meta name="viewport" content="width=device-width, initial-scale=1">
689 <title>Oils Releases</title>
690 <style>
691EOF
692
693 cat web/base.css
694 cat web/release-index.css
695
696cat <<EOF
697 h1 {
698 text-align: center;
699 }
700 </style>
701 </head>
702 <body class="width50">
703 <p id="home-link">
704 <a href="/">oils.pub</a>
705 </p>
706 <h1>Oils Releases</h1>
707
708 <table class="release-table">
709EOF
710}
711
712html-index() {
713 local release_root_dir=$1
714 local out=${2:-_tmp/releases.html}
715
716 { _releases-html-header
717 _html-index $release_root_dir
718
719 cat <<EOF
720 </table>
721 </body>
722</html>
723EOF
724
725 } > $out
726
727 ls -l $out
728}
729
730# For quickly iterating on tarball size reductions.
731tarball-size() {
732 make clean-repo
733 make _bin/oil.ovm-dbg # faster way to build bytecode
734 oil # make tarball
735 test-oil-tar # Ctrl-C this, then run metrics/tarball.sh
736}
737
738dep-smoosh() {
739 local repo=~/git/languages/smoosh
740 if ! test -d $repo; then
741 local base_dir=$(dirname $repo)
742 mkdir -p $base_dir
743 pushd $base_dir
744 git clone git@github.com:mgree/smoosh.git
745 popd
746 fi
747}
748
749dep-benchmarks() {
750 ### Before auto-machine2
751
752 # 2023-07: Also need deps/from-tar.sh {configure,build}-cpython
753
754 benchmarks/osh-runtime.sh download
755 benchmarks/osh-runtime.sh extract
756
757 benchmarks/ovm-build.sh download
758 benchmarks/ovm-build.sh extract-other
759
760 # For ovm-build benchmark.
761 deps/from-binary.sh download-clang
762 deps/from-binary.sh extract-clang
763}
764
765more-release-deps() {
766 # List of deps that are NOT in soil/worker.sh here
767 # https://github.com/oilshell/oil/issues/926
768
769 # TODO: Make a container image for these.
770 if false; then
771 # TODO: Did this manually
772 # test/alpine.sh
773 # dep-alpine
774
775 # test/smoosh.sh
776 dep-smoosh
777
778 dep-benchmarks
779 fi
780}
781
782py-tarball() {
783 local in=_release/oil.tar
784 local out=_release/oil-$OIL_VERSION.tar.gz
785
786 make $in
787 time gzip -c $in > $out
788 ls -l $out
789
790 test-oil-tar
791}
792
793native-tarball() {
794 # oils-for-unix
795 devtools/release-native.sh make-tar
796 # Also install as root
797 devtools/release-native.sh extract-for-benchmarks INSTALL
798}
799
800configure-for-release() {
801 # Run the slower configure, not configure-for-dev
802 ./configure
803}
804
805two-tarballs() {
806 ### First step of release. Assume that CI passes
807
808 ensure-smooth-build
809
810 build/py.sh all # runs ./configure-for-dev
811 configure-for-release
812
813 # "Base state" for repo scripts
814 ./NINJA-config.sh
815
816 py-tarball
817
818 native-tarball
819}
820
821upload-tmp() {
822 local tarball=$1
823 local user=$2
824
825 scp $tarball $user@oilshell.org:tmp/
826}
827
828sync-tmp() {
829 local user=$1
830 local dest=${2:-_tmp/candidates}
831 mkdir -p $dest
832 rsync --archive --verbose $user@oilshell.org:tmp/ $dest
833}
834
835"$@"