OILS / build / doc.sh View on Github | oils.pub

753 lines, 381 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# build/doc.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10# https://oilshell.org/release/$VERSION/
11# doc/
12# index.html
13# INSTALL.html
14# INSTALL-old.html
15
16readonly OIL_VERSION=$(head -n 1 oils-version.txt)
17export OIL_VERSION # for quick_ref.py
18
19THIS_DIR=$(readlink -f $(dirname $0))
20readonly THIS_DIR
21REPO_ROOT=$(cd $THIS_DIR/.. && pwd)
22readonly REPO_ROOT
23
24
25readonly HTML_BASE_DIR=_release/VERSION
26
27
28log() {
29 echo "$@" 1>&2
30}
31
32#
33# Deps (similar to doctools/cmark.sh and build/codegen.sh)
34#
35
36readonly MANDOC_DIR='_deps/mdocml-1.14.1'
37
38download-mandoc() {
39 mkdir -p _deps
40 wget --no-clobber --directory _deps \
41 https://mandoc.bsd.lv/snapshots/mdocml-1.14.1.tar.gz
42}
43
44build-mandoc() {
45 cd $MANDOC_DIR
46 ./configure
47 make
48}
49
50mandoc() {
51 $MANDOC_DIR/mandoc "$@"
52}
53
54# Places version is used
55#
56# - in --version
57# - in URL for every page? inside the binary
58# - in titles for index, install, osh-quick-ref TOC, etc.
59# - in deployment script
60
61# Run with environment variable
62help-gen() {
63 PYTHONPATH=.:vendor doctools/help_gen.py "$@"
64}
65
66cmark() {
67 # h2 and h3 are shown in TOC. The blog uses "legacy" h3 and h4.
68 PYTHONPATH=.:vendor doctools/cmark.py --toc-tag h2 --toc-tag h3 --toc-pretty-href "$@"
69}
70
71readonly MARKDOWN_DOCS=(
72 published
73
74 # polished
75 getting-started
76 portability
77 known-differences
78 ysh-error
79 error-handling
80 error-catalog
81 json
82 hay
83 simple-word-eval
84 quirks
85 warts
86
87 eggex
88 ysh-regex-api
89 upgrade-breakage
90 ysh-tour
91
92 style-guide
93 novelties
94
95 proc-func
96 block-literals
97 objects
98 types
99
100 # Data language
101 qsn
102 qtt
103 j8-notation
104 # Protocol
105 pretty-printing
106 stream-table-process
107 byo
108 ysh-doc-processing
109
110 table-object-doc
111
112 lib-osh
113
114 doc-toolchain
115 doc-plugins
116 ul-table
117 ul-table-compare
118 idioms
119 shell-idioms
120 ysh-faq
121
122 language-influences
123 ysh-vs-python
124 ysh-vs-shell
125
126 syntactic-concepts
127 syntax-feelings
128 command-vs-expression-mode
129
130 # needs polish
131 # Note: docs about the YSH are prefixed 'ysh-'.
132 # data-model and command-vs-expression-mode span both OSH and YSH
133
134 index
135 faq-doc
136
137 options
138
139 old/index
140 old/project-tour
141 old/legacy-array
142 old/ysh-keywords
143 old/modules
144 old/expression-language
145 old/word-language
146 old/errors
147 old/ysh-builtins
148
149 io-builtins
150 unicode
151 framing
152 xtrace
153 headless
154 completion
155 strings
156 variables
157
158 # Internal stuff
159 interpreter-state
160 process-model
161 architecture-notes
162 parser-architecture
163)
164
165# Bug fix: Plain $(date) can output unicode characters (e.g. in Japanese
166# locale), which is loaded by Python into say u'\u5e74'. But the default
167# encoding in Python 2 is still 'ascii', which means that '%s' % u_str may
168# fail.
169#
170# I believe --rfc-e-mail should never output a Unicode character.
171#
172# A better fix would be to implement json_utf8.load(f), which doesn't decode
173# into unicode instances. This would remove useless conversions.
174
175readonly TIMESTAMP=$(date --rfc-email)
176
177split-and-render() {
178 local src=${1:-doc/known-differences.md}
179
180 local rel_path=${src%'.md'} # doc/known-differences
181 local tmp_prefix=_tmp/$rel_path # temp dir for splitting
182
183 local out=${2:-$HTML_BASE_DIR/$rel_path.html}
184 local web_url=${3:-'../web'}
185
186 mkdir -v -p $(dirname $out) $tmp_prefix
187
188 # Also add could add css_files. The one in the file takes precedence always?
189
190 # css_files: a space-separated list
191 # all_docs_url: so we link from doc/foo.html -> doc/
192
193 local css_files="$web_url/base.css $web_url/manual.css $web_url/toc.css $web_url/language.css $web_url/code.css"
194
195 doctools/split_doc.py \
196 -v build_timestamp="$TIMESTAMP" \
197 -v oil_version="$OIL_VERSION" \
198 -v css_files="$css_files" \
199 -v all_docs_url='.' \
200 -v repo_url="$src" \
201 $src $tmp_prefix
202
203 #ls -l _tmp/doc
204 #head _tmp/doc/*
205 #return
206
207 # for ysh-tour code blocks
208 local code_out=_tmp/code-blocks/$rel_path.txt
209 mkdir -v -p $(dirname $code_out)
210
211 cmark \
212 --code-block-output $code_out \
213 ${tmp_prefix}_meta.json ${tmp_prefix}_content.md > $out
214
215 log "$tmp_prefix -> (doctools/cmark) -> $out"
216}
217
218render-from-kate() {
219 ### Make it easier to configure Kate editor
220
221 # It want to pass an absolute path
222 # TODO: I can't figure out how to run this from Kate?
223
224 local full_path=$1
225
226 case $full_path in
227 $REPO_ROOT/*)
228 rel_path=${full_path#"$REPO_ROOT/"}
229 echo "relative path = $rel_path"
230 ;;
231 *)
232 die "$full_path should start with repo root $REPO_ROOT"
233 ;;
234 esac
235
236 split-and-render $rel_path
237}
238
239# Special case for README
240# Do NOT split because we don't want front matter in the markdown source.
241render-only() {
242 local src=${1:-README.md}
243
244 local name
245 case $src in
246 *.md)
247 name=$(basename $src .md)
248 ;;
249 *.txt)
250 name=$(basename $src .txt)
251 ;;
252 *)
253 name=$(basename $src)
254 ;;
255 esac
256
257 local out=${2:-$HTML_BASE_DIR/doc/$name.html}
258 local css_files=${3:-'../web/manual.css ../web/toc.css'}
259 local title=${4:-'Oils Source Code'}
260
261 local prefix=_tmp/doc/$name
262
263 local meta=${prefix}_meta.json
264 cat >$meta <<EOF
265{ "title": "$title",
266 "repo_url": "$src",
267 "css_files": "$css_files",
268 "all_docs_url": ".",
269
270 "build_timestamp": "$TIMESTAMP",
271 "oil_version": "$OIL_VERSION"
272}
273EOF
274
275 cmark $meta $src > $out
276 log "Wrote $out"
277}
278
279special() {
280 # TODO: do all READMEs
281 split-and-render mycpp/README.md \
282 $HTML_BASE_DIR/doc/oils-repo/mycpp/README.html \
283 ../../../web
284
285 local web_dir='../../web'
286 render-only 'README.md' $HTML_BASE_DIR/doc/oils-repo/README.html \
287 "$web_dir/base.css $web_dir/manual.css $web_dir/toc.css" 'Oils Source Code'
288
289 local web_dir='../web'
290 render-only INSTALL.txt '' \
291 "$web_dir/base.css $web_dir/install.css" 'Installing Oils'
292
293 render-only INSTALL-old.txt '' \
294 "$web_dir/base.css $web_dir/install.css" 'Installing Oils - old CPython build'
295
296 # These pages aren't in doc/
297 split-and-render doc/release-index.md _tmp/release-index.html
298 split-and-render doc/release-quality.md _tmp/release-quality.html
299}
300
301all-markdown() {
302 make-dirs
303
304 # TODO: We can set repo_url here! Then we don't need it for most docs.
305 # split_doc.py can return {} if the doc doesn't start with ---
306
307 #for d in doc/index.md doc/known-differences.md doc/*-manual.md \
308 # doc/eggex.md doc/oil-options.md doc/oil-func-proc-block.md; do
309 for d in "${MARKDOWN_DOCS[@]}"; do
310 split-and-render doc/$d.md
311 done
312
313 special
314}
315
316redir-body() {
317 local to_url=$1 # WARNING: no escaping
318 cat <<EOF
319<head>
320 <meta http-equiv="Refresh" content="0; URL=$to_url" />
321</head>
322EOF
323}
324
325redirect-pairs() {
326 # we want want /release/latest/ URLs to still work
327 cat <<EOF
328oil-language-tour ysh-tour
329oil-language-faq ysh-faq
330oil-help ysh-help
331oil-help-topics ysh-help-topics
332ysh-help ref/toc-ysh
333ysh-help-topics ref/toc-ysh
334EOF
335}
336
337all-redirects() {
338 redirect-pairs | while read -r from_page to_page; do
339 redir-body "$to_page.html" | tee "_release/VERSION/doc/$from_page.html"
340 done
341}
342
343# TODO: This could use some CSS.
344man-page() {
345 local root_dir=${1:-_release/VERSION}
346 mandoc -T html doc/osh.1 > $root_dir/osh.1.html
347 ls -l $root_dir
348}
349
350# I want to ship the INSTALL file literally, so just mutate things
351_sed-ext() {
352 sed --regexp-extended -i "$@"
353}
354
355update-src-versions() {
356 # Update tarball names, etc.
357 _sed-ext \
358 "s/[0-9]+\.[0-9]+\.[a-z0-9]+/$OIL_VERSION/g" \
359 doc/release-*.md INSTALL.txt INSTALL-old.txt
360
361 # Update /release/0.8.4/ URL, etc.
362 _sed-ext \
363 "s;/release/[0-9]+\.[0-9]+\.[a-z0-9]+/;/release/$OIL_VERSION/;g" \
364 doc/osh.1
365}
366
367#
368# Test Tools
369#
370
371split-doc-demo() {
372 cat > _tmp/testdoc.md <<EOF
373---
374title: foo
375---
376
377Title
378=====
379
380hello
381
382EOF
383
384 doctools/split_doc.py _tmp/testdoc.md _tmp/testdoc
385
386 head _tmp/testdoc*
387}
388
389#
390# Help is both markdown and text
391#
392
393readonly TMP_DIR=_tmp/doc
394readonly CODE_BLOCK_DIR=_tmp/code-blocks
395readonly TEXT_DIR=_devbuild/help
396readonly HTML_DIR=_release/VERSION
397readonly CODE_DIR=_devbuild/gen
398
399cards-from-indices() {
400 ### Make help cards
401
402 for lang in osh ysh data; do
403 help-gen cards-from-index $lang $TEXT_DIR \
404 < $HTML_DIR/doc/ref/toc-$lang.html
405 done
406}
407
408cards-from-chapters() {
409 ### Turn h3 topics into cards
410
411 local py_out=$CODE_DIR/help_meta.py
412
413 mkdir -p _gen/frontend
414 local cc_prefix=_gen/frontend/help_meta
415
416 help-gen cards-from-chapters $TEXT_DIR $py_out $cc_prefix \
417 $HTML_DIR/doc/ref/chap-*.html
418}
419
420ref-check() {
421 help-gen ref-check \
422 doc/ref/toc-*.md \
423 _release/VERSION/doc/ref/chap-*.html
424}
425
426fmt-check() {
427 PYTHONPATH=.:vendor doctools/fmt_check.py _release/VERSION/doc/ref/*.html
428}
429
430
431write-metrics() {
432 ### Check indexes and chapters against each other
433
434 local out=_release/VERSION/doc/metrics.txt
435
436 # send stderr to the log file too
437 ref-check > $out 2>&1
438
439 echo "Wrote $out"
440}
441
442tour() {
443 ### Build the Tour of YSH, and execute code as validation
444 local name=${1:-ysh-tour}
445
446 split-and-render doc/$name.md
447
448 local work_dir=$REPO_ROOT/_tmp/code-blocks/doc
449
450 mkdir -p $work_dir/lib
451
452 # Files used by module example
453 touch $work_dir/{build,test}.sh
454
455 cat >$work_dir/lines.txt <<'EOF'
456 doc/hello.md
457 "doc/with spaces.md"
458b'doc/with byte \yff.md'
459EOF
460
461 cat >$work_dir/myargs.ysh <<EOF
462const __provide__ = :| proc1 p2 p3 |
463
464proc proc1 {
465 echo proc1
466}
467
468proc p2 {
469 echo p2
470}
471
472proc p3 {
473 echo p3
474}
475EOF
476
477 cat >$work_dir/demo.py <<EOF
478#!/usr/bin/env python
479
480print("hi")
481EOF
482 chmod +x $work_dir/demo.py
483
484 cat >$work_dir/lib/util.ysh <<EOF
485const __provide__ = :| log |
486
487proc log {
488 echo @ARGV >&2
489}
490EOF
491
492 pushd $work_dir
493
494 # Prepend extra code
495 cat >tour.ysh - $name.txt <<EOF
496func myMethod(self) {
497 echo 'myMethod'
498}
499
500func mutatingMethod(self) {
501 echo 'mutatingMethod'
502}
503
504func makeMyObject(x) {
505 var methods = Object(null, {myMethod, 'M/mutatingMethod': mutatingMethod})
506 return (Object(methods, {x}))
507}
508EOF
509
510 # Fix: don't supply stdin!
511 $REPO_ROOT/bin/ysh tour.ysh < /dev/null
512 popd
513
514 # My own dev tools
515 # if test -d ~/vm-shared; then
516 if false; then
517 local path=_release/VERSION/doc/$name.html
518 cp -v $path ~/vm-shared/$path
519 fi
520}
521
522one() {
523 ### Iterate on one doc quickly
524
525 local name=${1:-options}
526
527 split-and-render doc/$name.md
528
529 # Make sure the doc has valid YSH code?
530 # TODO: Maybe need an attribute for OSH or YSH
531 pushd _tmp/code-blocks/doc
532 $REPO_ROOT/bin/ysh $name.txt
533 popd
534
535 if test -d ~/vm-shared; then
536 local out="${name%.md}.html"
537 local path=_release/VERSION/$out
538 cp -v $path ~/vm-shared/$path
539 fi
540}
541
542make-dirs() {
543 mkdir -p $TMP_DIR $CODE_BLOCK_DIR $TEXT_DIR $HTML_DIR/doc
544}
545
546one-ref() {
547 local md=${1:-doc/ref/index.md}
548 split-and-render $md '' '../../web'
549}
550
551all-ref() {
552 ### Build doc/ref in text and HTML. Depends on libcmark.so
553
554 log "Removing $TEXT_DIR/*"
555 rm -f $TEXT_DIR/*
556 make-dirs
557
558 # Make the indexes and chapters
559 for d in doc/ref/*.md; do
560 split-and-render $d '' '../../web'
561 done
562
563 # Note: if we want a $ref-topic shortcut, we might want to use Ninja to
564 # extract topics from all chapters first, and then make help_meta.json, like
565 # we have _devbuild/gen/help_meta.py.
566
567 # Text cards
568 cards-from-indices
569 # A few text cards, and HELP_TOPICS dict for URLs, for flat namespace
570 cards-from-chapters
571
572 return
573 if command -v pysum; then
574 # 19 KB of embedded help, seems OK. Biggest card is 'ysh-option'. Could
575 # compress it.
576 echo 'Size of embedded help:'
577 ls -l $TEXT_DIR | tee /dev/stderr | awk '{print $5}' | pysum
578 fi
579
580 # Better sorting
581 #LANG=C ls -l $TEXT_DIR
582}
583
584_copy-path() {
585 local src=$1 dest=$2
586 mkdir -p $(dirname $dest)
587 cp -v $src $dest
588}
589
590copy-web() {
591 find web \
592 \( -name _tmp -a -prune \) -o \
593 \( -name '*.css' -o -name '*.js' \) -a -printf '%p _release/VERSION/%p\n' |
594 xargs -n 2 -- $0 _copy-path
595}
596
597pretty-size() {
598 local path=$1
599 stat --format '%s' "$path" | python -c '
600import sys
601num_bytes = int(sys.stdin.read())
602print "{:,}".format(num_bytes)
603'
604}
605
606# NOTE: It might be better to link to files like this in the /release/ tree.
607# Although I am not signing them.
608
609# https://nodejs.org/dist/v8.11.4/SHASUMS256.txt.asc
610
611tarball-links-row-html() {
612 local version=$1
613
614 cat <<EOF
615<tr class="file-table-heading">
616 <td></td>
617 <td>File / SHA256 checksum</td>
618 <td class="size">Size</td>
619 <td></td>
620</tr>
621EOF
622
623 # we switched to .gz for oils-for-unix
624 # note: legacy names for old releases
625 for name in \
626 oils-for-unix-$version.tar.{gz,xz} \
627 oil-$version.tar.{gz,xz} \
628 oil-native-$version.tar.xz; do
629
630 local url="/download/$name" # The server URL
631 local path="../oils.pub__deploy/download/$name"
632
633 # Don't show tarballs that don't exist
634 if [[ $name == oils-for-unix-* && ! -f $path ]]; then
635 continue
636 fi
637 if [[ $name == oil-native-* && ! -f $path ]]; then
638 continue
639 fi
640
641 local checksum
642 checksum=$(sha256sum $path | awk '{print $1}')
643 local size
644 size=$(pretty-size $path)
645
646 # TODO: Port this to oil with "commas" extension.
647
648 # Three columns: date, version, and links
649 cat <<EOF
650 <tr>
651 <td></td>
652 <td class="filename"><a href="$url">$name</a></td>
653 <td class="size">$size</td>
654 </tr>
655 <tr>
656 <td></td>
657 <td colspan=2 class="checksum">$checksum</td>
658 </tr>
659EOF
660 done
661}
662
663this-release-links() {
664 echo '<div class="file-table">'
665 echo '<table>'
666 tarball-links-row-html "$OIL_VERSION"
667 echo '</table>'
668 echo '</div>'
669}
670
671# Turn HTML comment into a download link
672add-date-and-links() {
673 local snippet
674 snippet=$(this-release-links)
675
676 awk -v date=$1 -v snippet="$snippet" '
677 /<!-- REPLACE_WITH_DOWNLOAD_LINKS -->/ {
678 print(snippet)
679 next
680 }
681
682 /<!-- REPLACE_WITH_DATE -->/ {
683 print(date)
684 next
685 }
686
687 # Everything else
688 { print }
689 '
690}
691
692patch-release-pages() {
693 local release_date
694 release_date=$(cat _build/release-date.txt)
695
696 local root=_release/VERSION
697
698 add-date-and-links $release_date < _tmp/release-index.html > $root/index.html
699 add-date-and-links $release_date < _tmp/release-quality.html > $root/quality.html
700}
701
702copy-release-pages() {
703 ### For testing without releasing
704
705 cat < _tmp/release-index.html > $root/index.html
706 cat < _tmp/release-quality.html > $root/quality.html
707}
708
709run-for-release() {
710 ### Build a tree. Requires _build/release-date.txt to exist
711
712 local root=_release/VERSION
713 mkdir -p $root/{doc,test,pub}
714
715 tour
716
717 # Metadata
718 cp -v _build/release-date.txt oils-version.txt $root
719
720 # Docs
721 # Writes _release/VERSION and _tmp/release-index.html
722 all-markdown
723 all-ref
724 all-redirects # backward compat
725
726 fmt-check # Needs to run *after* we build the HTML
727
728 patch-release-pages
729
730 write-metrics
731
732 # Problem: You can't preview it without .wwz!
733 # Maybe have local redirects VERSION/test/wild/ to
734 #
735 # Instead of linking, I should compress them all here.
736
737 copy-web
738
739 if command -v tree >/dev/null; then
740 tree $root
741 else
742 find $root
743 fi
744}
745
746soil-run() {
747 build/stamp.sh write-release-date
748
749 run-for-release
750}
751
752"$@"
753