OILS / build / doc.sh View on Github | oils.pub

751 lines, 380 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# build/doc.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10# https://oilshell.org/release/$VERSION/
11# doc/
12# index.html
13# INSTALL.html
14# INSTALL-old.html
15
16readonly OIL_VERSION=$(head -n 1 oils-version.txt)
17export OIL_VERSION # for quick_ref.py
18
19THIS_DIR=$(readlink -f $(dirname $0))
20readonly THIS_DIR
21REPO_ROOT=$(cd $THIS_DIR/.. && pwd)
22readonly REPO_ROOT
23
24
25readonly HTML_BASE_DIR=_release/VERSION
26
27
28log() {
29 echo "$@" 1>&2
30}
31
32#
33# Deps (similar to doctools/cmark.sh and build/codegen.sh)
34#
35
36readonly MANDOC_DIR='_deps/mdocml-1.14.1'
37
38download-mandoc() {
39 mkdir -p _deps
40 wget --no-clobber --directory _deps \
41 https://mandoc.bsd.lv/snapshots/mdocml-1.14.1.tar.gz
42}
43
44build-mandoc() {
45 cd $MANDOC_DIR
46 ./configure
47 make
48}
49
50mandoc() {
51 $MANDOC_DIR/mandoc "$@"
52}
53
54# Places version is used
55#
56# - in --version
57# - in URL for every page? inside the binary
58# - in titles for index, install, osh-quick-ref TOC, etc.
59# - in deployment script
60
61# Run with environment variable
62help-gen() {
63 PYTHONPATH=.:vendor doctools/help_gen.py "$@"
64}
65
66cmark() {
67 # h2 and h3 are shown in TOC. The blog uses "legacy" h3 and h4.
68 PYTHONPATH=.:vendor doctools/cmark.py --toc-tag h2 --toc-tag h3 --toc-pretty-href "$@"
69}
70
71readonly MARKDOWN_DOCS=(
72 published
73
74 # polished
75 getting-started
76 portability
77 known-differences
78 ysh-error
79 error-handling
80 error-catalog
81 json
82 hay
83 simple-word-eval
84 quirks
85 warts
86
87 eggex
88 ysh-regex-api
89 upgrade-breakage
90 ysh-tour
91
92 style-guide
93 novelties
94
95 proc-func
96 block-literals
97 objects
98 types
99
100 # Data language
101 qsn
102 qtt
103 j8-notation
104 # Protocol
105 pretty-printing
106 stream-table-process
107 byo
108 ysh-doc-processing
109
110 lib-osh
111
112 doc-toolchain
113 doc-plugins
114 ul-table
115 ul-table-compare
116 idioms
117 shell-idioms
118 ysh-faq
119
120 language-influences
121 ysh-vs-python
122 ysh-vs-shell
123
124 syntactic-concepts
125 syntax-feelings
126 command-vs-expression-mode
127
128 # needs polish
129 # Note: docs about the YSH are prefixed 'ysh-'.
130 # data-model and command-vs-expression-mode span both OSH and YSH
131
132 index
133 faq-doc
134
135 options
136
137 old/index
138 old/project-tour
139 old/legacy-array
140 old/ysh-keywords
141 old/modules
142 old/expression-language
143 old/word-language
144 old/errors
145 old/ysh-builtins
146
147 io-builtins
148 unicode
149 framing
150 xtrace
151 headless
152 completion
153 strings
154 variables
155
156 # Internal stuff
157 interpreter-state
158 process-model
159 architecture-notes
160 parser-architecture
161)
162
163# Bug fix: Plain $(date) can output unicode characters (e.g. in Japanese
164# locale), which is loaded by Python into say u'\u5e74'. But the default
165# encoding in Python 2 is still 'ascii', which means that '%s' % u_str may
166# fail.
167#
168# I believe --rfc-e-mail should never output a Unicode character.
169#
170# A better fix would be to implement json_utf8.load(f), which doesn't decode
171# into unicode instances. This would remove useless conversions.
172
173readonly TIMESTAMP=$(date --rfc-email)
174
175split-and-render() {
176 local src=${1:-doc/known-differences.md}
177
178 local rel_path=${src%'.md'} # doc/known-differences
179 local tmp_prefix=_tmp/$rel_path # temp dir for splitting
180
181 local out=${2:-$HTML_BASE_DIR/$rel_path.html}
182 local web_url=${3:-'../web'}
183
184 mkdir -v -p $(dirname $out) $tmp_prefix
185
186 # Also add could add css_files. The one in the file takes precedence always?
187
188 # css_files: a space-separated list
189 # all_docs_url: so we link from doc/foo.html -> doc/
190
191 local css_files="$web_url/base.css $web_url/manual.css $web_url/toc.css $web_url/language.css $web_url/code.css"
192
193 doctools/split_doc.py \
194 -v build_timestamp="$TIMESTAMP" \
195 -v oil_version="$OIL_VERSION" \
196 -v css_files="$css_files" \
197 -v all_docs_url='.' \
198 -v repo_url="$src" \
199 $src $tmp_prefix
200
201 #ls -l _tmp/doc
202 #head _tmp/doc/*
203 #return
204
205 # for ysh-tour code blocks
206 local code_out=_tmp/code-blocks/$rel_path.txt
207 mkdir -v -p $(dirname $code_out)
208
209 cmark \
210 --code-block-output $code_out \
211 ${tmp_prefix}_meta.json ${tmp_prefix}_content.md > $out
212
213 log "$tmp_prefix -> (doctools/cmark) -> $out"
214}
215
216render-from-kate() {
217 ### Make it easier to configure Kate editor
218
219 # It want to pass an absolute path
220 # TODO: I can't figure out how to run this from Kate?
221
222 local full_path=$1
223
224 case $full_path in
225 $REPO_ROOT/*)
226 rel_path=${full_path#"$REPO_ROOT/"}
227 echo "relative path = $rel_path"
228 ;;
229 *)
230 die "$full_path should start with repo root $REPO_ROOT"
231 ;;
232 esac
233
234 split-and-render $rel_path
235}
236
237# Special case for README
238# Do NOT split because we don't want front matter in the markdown source.
239render-only() {
240 local src=${1:-README.md}
241
242 local name
243 case $src in
244 *.md)
245 name=$(basename $src .md)
246 ;;
247 *.txt)
248 name=$(basename $src .txt)
249 ;;
250 *)
251 name=$(basename $src)
252 ;;
253 esac
254
255 local out=${2:-$HTML_BASE_DIR/doc/$name.html}
256 local css_files=${3:-'../web/manual.css ../web/toc.css'}
257 local title=${4:-'Oils Source Code'}
258
259 local prefix=_tmp/doc/$name
260
261 local meta=${prefix}_meta.json
262 cat >$meta <<EOF
263{ "title": "$title",
264 "repo_url": "$src",
265 "css_files": "$css_files",
266 "all_docs_url": ".",
267
268 "build_timestamp": "$TIMESTAMP",
269 "oil_version": "$OIL_VERSION"
270}
271EOF
272
273 cmark $meta $src > $out
274 log "Wrote $out"
275}
276
277special() {
278 # TODO: do all READMEs
279 split-and-render mycpp/README.md \
280 $HTML_BASE_DIR/doc/oils-repo/mycpp/README.html \
281 ../../../web
282
283 local web_dir='../../web'
284 render-only 'README.md' $HTML_BASE_DIR/doc/oils-repo/README.html \
285 "$web_dir/base.css $web_dir/manual.css $web_dir/toc.css" 'Oils Source Code'
286
287 local web_dir='../web'
288 render-only INSTALL.txt '' \
289 "$web_dir/base.css $web_dir/install.css" 'Installing Oils'
290
291 render-only INSTALL-old.txt '' \
292 "$web_dir/base.css $web_dir/install.css" 'Installing Oils - old CPython build'
293
294 # These pages aren't in doc/
295 split-and-render doc/release-index.md _tmp/release-index.html
296 split-and-render doc/release-quality.md _tmp/release-quality.html
297}
298
299all-markdown() {
300 make-dirs
301
302 # TODO: We can set repo_url here! Then we don't need it for most docs.
303 # split_doc.py can return {} if the doc doesn't start with ---
304
305 #for d in doc/index.md doc/known-differences.md doc/*-manual.md \
306 # doc/eggex.md doc/oil-options.md doc/oil-func-proc-block.md; do
307 for d in "${MARKDOWN_DOCS[@]}"; do
308 split-and-render doc/$d.md
309 done
310
311 special
312}
313
314redir-body() {
315 local to_url=$1 # WARNING: no escaping
316 cat <<EOF
317<head>
318 <meta http-equiv="Refresh" content="0; URL=$to_url" />
319</head>
320EOF
321}
322
323redirect-pairs() {
324 # we want want /release/latest/ URLs to still work
325 cat <<EOF
326oil-language-tour ysh-tour
327oil-language-faq ysh-faq
328oil-help ysh-help
329oil-help-topics ysh-help-topics
330ysh-help ref/toc-ysh
331ysh-help-topics ref/toc-ysh
332EOF
333}
334
335all-redirects() {
336 redirect-pairs | while read -r from_page to_page; do
337 redir-body "$to_page.html" | tee "_release/VERSION/doc/$from_page.html"
338 done
339}
340
341# TODO: This could use some CSS.
342man-page() {
343 local root_dir=${1:-_release/VERSION}
344 mandoc -T html doc/osh.1 > $root_dir/osh.1.html
345 ls -l $root_dir
346}
347
348# I want to ship the INSTALL file literally, so just mutate things
349_sed-ext() {
350 sed --regexp-extended -i "$@"
351}
352
353update-src-versions() {
354 # Update tarball names, etc.
355 _sed-ext \
356 "s/[0-9]+\.[0-9]+\.[a-z0-9]+/$OIL_VERSION/g" \
357 doc/release-*.md INSTALL.txt INSTALL-old.txt
358
359 # Update /release/0.8.4/ URL, etc.
360 _sed-ext \
361 "s;/release/[0-9]+\.[0-9]+\.[a-z0-9]+/;/release/$OIL_VERSION/;g" \
362 doc/osh.1
363}
364
365#
366# Test Tools
367#
368
369split-doc-demo() {
370 cat > _tmp/testdoc.md <<EOF
371---
372title: foo
373---
374
375Title
376=====
377
378hello
379
380EOF
381
382 doctools/split_doc.py _tmp/testdoc.md _tmp/testdoc
383
384 head _tmp/testdoc*
385}
386
387#
388# Help is both markdown and text
389#
390
391readonly TMP_DIR=_tmp/doc
392readonly CODE_BLOCK_DIR=_tmp/code-blocks
393readonly TEXT_DIR=_devbuild/help
394readonly HTML_DIR=_release/VERSION
395readonly CODE_DIR=_devbuild/gen
396
397cards-from-indices() {
398 ### Make help cards
399
400 for lang in osh ysh data; do
401 help-gen cards-from-index $lang $TEXT_DIR \
402 < $HTML_DIR/doc/ref/toc-$lang.html
403 done
404}
405
406cards-from-chapters() {
407 ### Turn h3 topics into cards
408
409 local py_out=$CODE_DIR/help_meta.py
410
411 mkdir -p _gen/frontend
412 local cc_prefix=_gen/frontend/help_meta
413
414 help-gen cards-from-chapters $TEXT_DIR $py_out $cc_prefix \
415 $HTML_DIR/doc/ref/chap-*.html
416}
417
418ref-check() {
419 help-gen ref-check \
420 doc/ref/toc-*.md \
421 _release/VERSION/doc/ref/chap-*.html
422}
423
424fmt-check() {
425 PYTHONPATH=.:vendor doctools/fmt_check.py _release/VERSION/doc/ref/*.html
426}
427
428
429write-metrics() {
430 ### Check indexes and chapters against each other
431
432 local out=_release/VERSION/doc/metrics.txt
433
434 # send stderr to the log file too
435 ref-check > $out 2>&1
436
437 echo "Wrote $out"
438}
439
440tour() {
441 ### Build the Tour of YSH, and execute code as validation
442 local name=${1:-ysh-tour}
443
444 split-and-render doc/$name.md
445
446 local work_dir=$REPO_ROOT/_tmp/code-blocks/doc
447
448 mkdir -p $work_dir/lib
449
450 # Files used by module example
451 touch $work_dir/{build,test}.sh
452
453 cat >$work_dir/lines.txt <<'EOF'
454 doc/hello.md
455 "doc/with spaces.md"
456b'doc/with byte \yff.md'
457EOF
458
459 cat >$work_dir/myargs.ysh <<EOF
460const __provide__ = :| proc1 p2 p3 |
461
462proc proc1 {
463 echo proc1
464}
465
466proc p2 {
467 echo p2
468}
469
470proc p3 {
471 echo p3
472}
473EOF
474
475 cat >$work_dir/demo.py <<EOF
476#!/usr/bin/env python
477
478print("hi")
479EOF
480 chmod +x $work_dir/demo.py
481
482 cat >$work_dir/lib/util.ysh <<EOF
483const __provide__ = :| log |
484
485proc log {
486 echo @ARGV >&2
487}
488EOF
489
490 pushd $work_dir
491
492 # Prepend extra code
493 cat >tour.ysh - $name.txt <<EOF
494func myMethod(self) {
495 echo 'myMethod'
496}
497
498func mutatingMethod(self) {
499 echo 'mutatingMethod'
500}
501
502func makeMyObject(x) {
503 var methods = Object(null, {myMethod, 'M/mutatingMethod': mutatingMethod})
504 return (Object(methods, {x}))
505}
506EOF
507
508 # Fix: don't supply stdin!
509 $REPO_ROOT/bin/ysh tour.ysh < /dev/null
510 popd
511
512 # My own dev tools
513 # if test -d ~/vm-shared; then
514 if false; then
515 local path=_release/VERSION/doc/$name.html
516 cp -v $path ~/vm-shared/$path
517 fi
518}
519
520one() {
521 ### Iterate on one doc quickly
522
523 local name=${1:-options}
524
525 split-and-render doc/$name.md
526
527 # Make sure the doc has valid YSH code?
528 # TODO: Maybe need an attribute for OSH or YSH
529 pushd _tmp/code-blocks/doc
530 $REPO_ROOT/bin/ysh $name.txt
531 popd
532
533 if test -d ~/vm-shared; then
534 local out="${name%.md}.html"
535 local path=_release/VERSION/$out
536 cp -v $path ~/vm-shared/$path
537 fi
538}
539
540make-dirs() {
541 mkdir -p $TMP_DIR $CODE_BLOCK_DIR $TEXT_DIR $HTML_DIR/doc
542}
543
544one-ref() {
545 local md=${1:-doc/ref/index.md}
546 split-and-render $md '' '../../web'
547}
548
549all-ref() {
550 ### Build doc/ref in text and HTML. Depends on libcmark.so
551
552 log "Removing $TEXT_DIR/*"
553 rm -f $TEXT_DIR/*
554 make-dirs
555
556 # Make the indexes and chapters
557 for d in doc/ref/*.md; do
558 split-and-render $d '' '../../web'
559 done
560
561 # Note: if we want a $ref-topic shortcut, we might want to use Ninja to
562 # extract topics from all chapters first, and then make help_meta.json, like
563 # we have _devbuild/gen/help_meta.py.
564
565 # Text cards
566 cards-from-indices
567 # A few text cards, and HELP_TOPICS dict for URLs, for flat namespace
568 cards-from-chapters
569
570 return
571 if command -v pysum; then
572 # 19 KB of embedded help, seems OK. Biggest card is 'ysh-option'. Could
573 # compress it.
574 echo 'Size of embedded help:'
575 ls -l $TEXT_DIR | tee /dev/stderr | awk '{print $5}' | pysum
576 fi
577
578 # Better sorting
579 #LANG=C ls -l $TEXT_DIR
580}
581
582_copy-path() {
583 local src=$1 dest=$2
584 mkdir -p $(dirname $dest)
585 cp -v $src $dest
586}
587
588copy-web() {
589 find web \
590 \( -name _tmp -a -prune \) -o \
591 \( -name '*.css' -o -name '*.js' \) -a -printf '%p _release/VERSION/%p\n' |
592 xargs -n 2 -- $0 _copy-path
593}
594
595pretty-size() {
596 local path=$1
597 stat --format '%s' "$path" | python -c '
598import sys
599num_bytes = int(sys.stdin.read())
600print "{:,}".format(num_bytes)
601'
602}
603
604# NOTE: It might be better to link to files like this in the /release/ tree.
605# Although I am not signing them.
606
607# https://nodejs.org/dist/v8.11.4/SHASUMS256.txt.asc
608
609tarball-links-row-html() {
610 local version=$1
611
612 cat <<EOF
613<tr class="file-table-heading">
614 <td></td>
615 <td>File / SHA256 checksum</td>
616 <td class="size">Size</td>
617 <td></td>
618</tr>
619EOF
620
621 # we switched to .gz for oils-for-unix
622 # note: legacy names for old releases
623 for name in \
624 oils-for-unix-$version.tar.{gz,xz} \
625 oil-$version.tar.{gz,xz} \
626 oil-native-$version.tar.xz; do
627
628 local url="/download/$name" # The server URL
629 local path="../oils.pub__deploy/download/$name"
630
631 # Don't show tarballs that don't exist
632 if [[ $name == oils-for-unix-* && ! -f $path ]]; then
633 continue
634 fi
635 if [[ $name == oil-native-* && ! -f $path ]]; then
636 continue
637 fi
638
639 local checksum
640 checksum=$(sha256sum $path | awk '{print $1}')
641 local size
642 size=$(pretty-size $path)
643
644 # TODO: Port this to oil with "commas" extension.
645
646 # Three columns: date, version, and links
647 cat <<EOF
648 <tr>
649 <td></td>
650 <td class="filename"><a href="$url">$name</a></td>
651 <td class="size">$size</td>
652 </tr>
653 <tr>
654 <td></td>
655 <td colspan=2 class="checksum">$checksum</td>
656 </tr>
657EOF
658 done
659}
660
661this-release-links() {
662 echo '<div class="file-table">'
663 echo '<table>'
664 tarball-links-row-html "$OIL_VERSION"
665 echo '</table>'
666 echo '</div>'
667}
668
669# Turn HTML comment into a download link
670add-date-and-links() {
671 local snippet
672 snippet=$(this-release-links)
673
674 awk -v date=$1 -v snippet="$snippet" '
675 /<!-- REPLACE_WITH_DOWNLOAD_LINKS -->/ {
676 print(snippet)
677 next
678 }
679
680 /<!-- REPLACE_WITH_DATE -->/ {
681 print(date)
682 next
683 }
684
685 # Everything else
686 { print }
687 '
688}
689
690patch-release-pages() {
691 local release_date
692 release_date=$(cat _build/release-date.txt)
693
694 local root=_release/VERSION
695
696 add-date-and-links $release_date < _tmp/release-index.html > $root/index.html
697 add-date-and-links $release_date < _tmp/release-quality.html > $root/quality.html
698}
699
700copy-release-pages() {
701 ### For testing without releasing
702
703 cat < _tmp/release-index.html > $root/index.html
704 cat < _tmp/release-quality.html > $root/quality.html
705}
706
707run-for-release() {
708 ### Build a tree. Requires _build/release-date.txt to exist
709
710 local root=_release/VERSION
711 mkdir -p $root/{doc,test,pub}
712
713 tour
714
715 # Metadata
716 cp -v _build/release-date.txt oils-version.txt $root
717
718 # Docs
719 # Writes _release/VERSION and _tmp/release-index.html
720 all-markdown
721 all-ref
722 all-redirects # backward compat
723
724 fmt-check # Needs to run *after* we build the HTML
725
726 patch-release-pages
727
728 write-metrics
729
730 # Problem: You can't preview it without .wwz!
731 # Maybe have local redirects VERSION/test/wild/ to
732 #
733 # Instead of linking, I should compress them all here.
734
735 copy-web
736
737 if command -v tree >/dev/null; then
738 tree $root
739 else
740 find $root
741 fi
742}
743
744soil-run() {
745 build/stamp.sh write-release-date
746
747 run-for-release
748}
749
750"$@"
751