OILS / build / doc.sh View on Github | oilshell.org

696 lines, 370 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# build/doc.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10# https://oilshell.org/release/$VERSION/
11# doc/
12# index.html
13# INSTALL.html
14# INSTALL-old.html
15
16readonly OIL_VERSION=$(head -n 1 oil-version.txt)
17export OIL_VERSION # for quick_ref.py
18
19THIS_DIR=$(readlink -f $(dirname $0))
20readonly THIS_DIR
21REPO_ROOT=$(cd $THIS_DIR/.. && pwd)
22readonly REPO_ROOT
23
24
25readonly HTML_BASE_DIR=_release/VERSION
26
27
28log() {
29 echo "$@" 1>&2
30}
31
32#
33# Deps (similar to doctools/cmark.sh and build/codegen.sh)
34#
35
36readonly MANDOC_DIR='_deps/mdocml-1.14.1'
37
38download-mandoc() {
39 mkdir -p _deps
40 wget --no-clobber --directory _deps \
41 https://mandoc.bsd.lv/snapshots/mdocml-1.14.1.tar.gz
42}
43
44build-mandoc() {
45 cd $MANDOC_DIR
46 ./configure
47 make
48}
49
50mandoc() {
51 $MANDOC_DIR/mandoc "$@"
52}
53
54# Places version is used
55#
56# - in --version
57# - in URL for every page? inside the binary
58# - in titles for index, install, osh-quick-ref TOC, etc.
59# - in deployment script
60
61# Run with environment variable
62help-gen() {
63 PYTHONPATH=. doctools/help_gen.py "$@"
64}
65
66cmark() {
67 # h2 and h3 are shown in TOC. The blog uses "legacy" h3 and h4.
68 PYTHONPATH=. doctools/cmark.py --toc-tag h2 --toc-tag h3 --toc-pretty-href "$@"
69}
70
71readonly MARKDOWN_DOCS=(
72 published
73
74 # polished
75 getting-started
76 portability
77 known-differences
78 ysh-error
79 error-handling
80 error-catalog
81 json
82 hay
83 simple-word-eval
84 quirks
85 warts
86
87 eggex
88 ysh-regex-api
89 upgrade-breakage
90 ysh-tour
91
92 style-guide
93 novelties
94
95 proc-func
96 block-literals
97
98 # Data language
99 qsn
100 qtt
101 j8-notation
102 # Protocol
103 pretty-printing
104 stream-table-process
105 byo
106 ysh-doc-processing
107
108 lib-osh
109
110 doc-toolchain
111 doc-plugins
112 idioms
113 shell-idioms
114 ysh-faq
115
116 language-influences
117 ysh-vs-python
118 ysh-vs-shell
119
120 syntactic-concepts
121 syntax-feelings
122 command-vs-expression-mode
123
124 # needs polish
125 # Note: docs about the YSH are prefixed 'ysh-'.
126 # data-model and command-vs-expression-mode span both OSH and YSH
127
128 index
129 faq-doc
130
131 options
132
133 old/index
134 old/project-tour
135 old/legacy-array
136 old/ysh-keywords
137 old/modules
138 old/expression-language
139 old/word-language
140 old/errors
141 old/ysh-builtins
142
143 io-builtins
144 unicode
145 framing
146 xtrace
147 headless
148 completion
149 strings
150 variables
151
152 # Internal stuff
153 interpreter-state
154 process-model
155 architecture-notes
156 parser-architecture
157)
158
159# Bug fix: Plain $(date) can output unicode characters (e.g. in Japanese
160# locale), which is loaded by Python into say u'\u5e74'. But the default
161# encoding in Python 2 is still 'ascii', which means that '%s' % u_str may
162# fail.
163#
164# I believe --rfc-e-mail should never output a Unicode character.
165#
166# A better fix would be to implement json_utf8.load(f), which doesn't decode
167# into unicode instances. This would remove useless conversions.
168
169readonly TIMESTAMP=$(date --rfc-email)
170
171split-and-render() {
172 local src=${1:-doc/known-differences.md}
173
174 local rel_path=${src%'.md'} # doc/known-differences
175 local tmp_prefix=_tmp/$rel_path # temp dir for splitting
176
177 local out=${2:-$HTML_BASE_DIR/$rel_path.html}
178 local web_url=${3:-'../web'}
179
180 mkdir -v -p $(dirname $out) $tmp_prefix
181
182 # Also add could add css_files. The one in the file takes precedence always?
183
184 # css_files: a space-separated list
185 # all_docs_url: so we link from doc/foo.html -> doc/
186
187 local css_files="$web_url/base.css $web_url/manual.css $web_url/toc.css $web_url/language.css $web_url/code.css"
188
189 doctools/split_doc.py \
190 -v build_timestamp="$TIMESTAMP" \
191 -v oil_version="$OIL_VERSION" \
192 -v css_files="$css_files" \
193 -v all_docs_url='.' \
194 -v repo_url="$src" \
195 $src $tmp_prefix
196
197 #ls -l _tmp/doc
198 #head _tmp/doc/*
199 #return
200
201 # for ysh-tour code blocks
202 local code_out=_tmp/code-blocks/$rel_path.txt
203 mkdir -v -p $(dirname $code_out)
204
205 cmark \
206 --code-block-output $code_out \
207 ${tmp_prefix}_meta.json ${tmp_prefix}_content.md > $out
208
209 log "$tmp_prefix -> (doctools/cmark) -> $out"
210}
211
212render-from-kate() {
213 ### Make it easier to configure Kate editor
214
215 # It want to pass an absolute path
216 # TODO: I can't figure out how to run this from Kate?
217
218 local full_path=$1
219
220 case $full_path in
221 $REPO_ROOT/*)
222 rel_path=${full_path#"$REPO_ROOT/"}
223 echo "relative path = $rel_path"
224 ;;
225 *)
226 die "$full_path should start with repo root $REPO_ROOT"
227 ;;
228 esac
229
230 split-and-render $rel_path
231}
232
233# Special case for README
234# Do NOT split because we don't want front matter in the markdown source.
235render-only() {
236 local src=${1:-README.md}
237
238 local name
239 case $src in
240 *.md)
241 name=$(basename $src .md)
242 ;;
243 *.txt)
244 name=$(basename $src .txt)
245 ;;
246 *)
247 name=$(basename $src)
248 ;;
249 esac
250
251 local out=${2:-$HTML_BASE_DIR/doc/$name.html}
252 local css_files=${3:-'../web/manual.css ../web/toc.css'}
253 local title=${4:-'Oils Source Code'}
254
255 local prefix=_tmp/doc/$name
256
257 local meta=${prefix}_meta.json
258 cat >$meta <<EOF
259{ "title": "$title",
260 "repo_url": "$src",
261 "css_files": "$css_files",
262 "all_docs_url": ".",
263
264 "build_timestamp": "$TIMESTAMP",
265 "oil_version": "$OIL_VERSION"
266}
267EOF
268
269 cmark $meta $src > $out
270 log "Wrote $out"
271}
272
273special() {
274 # TODO: do all READMEs
275 split-and-render mycpp/README.md \
276 $HTML_BASE_DIR/doc/oils-repo/mycpp/README.html \
277 ../../../web
278
279 local web_dir='../../web'
280 render-only 'README.md' $HTML_BASE_DIR/doc/oils-repo/README.html \
281 "$web_dir/base.css $web_dir/manual.css $web_dir/toc.css" 'Oils Source Code'
282
283 local web_dir='../web'
284 render-only INSTALL.txt '' \
285 "$web_dir/base.css $web_dir/install.css" 'Installing Oils'
286
287 render-only INSTALL-old.txt '' \
288 "$web_dir/base.css $web_dir/install.css" 'Installing Oils - old CPython build'
289
290 # These pages aren't in doc/
291 split-and-render doc/release-index.md _tmp/release-index.html
292 split-and-render doc/release-quality.md _tmp/release-quality.html
293}
294
295all-markdown() {
296 make-dirs
297
298 # TODO: We can set repo_url here! Then we don't need it for most docs.
299 # split_doc.py can return {} if the doc doesn't start with ---
300
301 #for d in doc/index.md doc/known-differences.md doc/*-manual.md \
302 # doc/eggex.md doc/oil-options.md doc/oil-func-proc-block.md; do
303 for d in "${MARKDOWN_DOCS[@]}"; do
304 split-and-render doc/$d.md
305 done
306
307 special
308}
309
310redir-body() {
311 local to_url=$1 # WARNING: no escaping
312 cat <<EOF
313<head>
314 <meta http-equiv="Refresh" content="0; URL=$to_url" />
315</head>
316EOF
317}
318
319redirect-pairs() {
320 # we want want /release/latest/ URLs to still work
321 cat <<EOF
322oil-language-tour ysh-tour
323oil-language-faq ysh-faq
324oil-help ysh-help
325oil-help-topics ysh-help-topics
326ysh-help ref/toc-ysh
327ysh-help-topics ref/toc-ysh
328EOF
329}
330
331all-redirects() {
332 redirect-pairs | while read -r from_page to_page; do
333 redir-body "$to_page.html" | tee "_release/VERSION/doc/$from_page.html"
334 done
335}
336
337# TODO: This could use some CSS.
338man-page() {
339 local root_dir=${1:-_release/VERSION}
340 mandoc -T html doc/osh.1 > $root_dir/osh.1.html
341 ls -l $root_dir
342}
343
344# I want to ship the INSTALL file literally, so just mutate things
345_sed-ext() {
346 sed --regexp-extended -i "$@"
347}
348
349update-src-versions() {
350 # Update tarball names, etc.
351 _sed-ext \
352 "s/[0-9]+\.[0-9]+\.[a-z0-9]+/$OIL_VERSION/g" \
353 doc/release-*.md INSTALL.txt INSTALL-old.txt
354
355 # Update /release/0.8.4/ URL, etc.
356 _sed-ext \
357 "s;/release/[0-9]+\.[0-9]+\.[a-z0-9]+/;/release/$OIL_VERSION/;g" \
358 doc/osh.1
359}
360
361#
362# Test Tools
363#
364
365split-doc-demo() {
366 cat > _tmp/testdoc.md <<EOF
367---
368title: foo
369---
370
371Title
372=====
373
374hello
375
376EOF
377
378 doctools/split_doc.py _tmp/testdoc.md _tmp/testdoc
379
380 head _tmp/testdoc*
381}
382
383#
384# Help is both markdown and text
385#
386
387readonly TMP_DIR=_tmp/doc
388readonly CODE_BLOCK_DIR=_tmp/code-blocks
389readonly TEXT_DIR=_devbuild/help
390readonly HTML_DIR=_release/VERSION
391readonly CODE_DIR=_devbuild/gen
392
393cards-from-indices() {
394 ### Make help cards
395
396 for lang in osh ysh data; do
397 help-gen cards-from-index $lang $TEXT_DIR \
398 < $HTML_DIR/doc/ref/toc-$lang.html
399 done
400}
401
402cards-from-chapters() {
403 ### Turn h3 topics into cards
404
405 local py_out=$CODE_DIR/help_meta.py
406
407 mkdir -p _gen/frontend
408 local cc_prefix=_gen/frontend/help_meta
409
410 help-gen cards-from-chapters $TEXT_DIR $py_out $cc_prefix \
411 $HTML_DIR/doc/ref/chap-*.html
412}
413
414ref-check() {
415 help-gen ref-check \
416 doc/ref/toc-*.md \
417 _release/VERSION/doc/ref/chap-*.html
418}
419
420fmt-check() {
421 PYTHONPATH=. doctools/fmt_check.py _release/VERSION/doc/ref/*.html
422}
423
424
425write-metrics() {
426 ### Check indexes and chapters against each other
427
428 local out=_release/VERSION/doc/metrics.txt
429
430 # send stderr to the log file too
431 ref-check > $out 2>&1
432
433 echo "Wrote $out"
434}
435
436tour() {
437 ### Build the Tour of YSH, and execute code as validation
438 local name=${1:-ysh-tour}
439
440 split-and-render doc/$name.md
441
442 local work_dir=$REPO_ROOT/_tmp/code-blocks/doc
443
444 mkdir -p $work_dir/lib
445
446 # Files used by module example
447 touch $work_dir/{build,test}.sh
448
449 cat >$work_dir/lib/util.ysh <<EOF
450log() { echo "$@" 1>&2; }
451EOF
452
453 pushd $work_dir
454 # Fix: don't supply stdin!
455 $REPO_ROOT/bin/ysh $name.txt < /dev/null
456 popd
457
458 # My own dev tools
459 # if test -d ~/vm-shared; then
460 if false; then
461 local path=_release/VERSION/doc/$name.html
462 cp -v $path ~/vm-shared/$path
463 fi
464}
465
466one() {
467 ### Iterate on one doc quickly
468
469 local name=${1:-options}
470
471 split-and-render doc/$name.md
472
473 # Make sure the doc has valid YSH code?
474 # TODO: Maybe need an attribute for OSH or YSH
475 pushd _tmp/code-blocks/doc
476 $REPO_ROOT/bin/ysh $name.txt
477 popd
478
479 if test -d ~/vm-shared; then
480 local out="${name%.md}.html"
481 local path=_release/VERSION/$out
482 cp -v $path ~/vm-shared/$path
483 fi
484}
485
486make-dirs() {
487 mkdir -p $TMP_DIR $CODE_BLOCK_DIR $TEXT_DIR $HTML_DIR/doc
488}
489
490one-ref() {
491 local md=${1:-doc/ref/index.md}
492 split-and-render $md '' '../../web'
493}
494
495all-ref() {
496 ### Build doc/ref in text and HTML. Depends on libcmark.so
497
498 log "Removing $TEXT_DIR/*"
499 rm -f $TEXT_DIR/*
500 make-dirs
501
502 # Make the indexes and chapters
503 for d in doc/ref/*.md; do
504 split-and-render $d '' '../../web'
505 done
506
507 # Note: if we want a $ref-topic shortcut, we might want to use Ninja to
508 # extract topics from all chapters first, and then make help_meta.json, like
509 # we have _devbuild/gen/help_meta.py.
510
511 # Text cards
512 cards-from-indices
513 # A few text cards, and HELP_TOPICS dict for URLs, for flat namespace
514 cards-from-chapters
515
516 if command -v pysum; then
517 # 19 KB of embedded help, seems OK. Biggest card is 'ysh-option'. Could
518 # compress it.
519 echo 'Size of embedded help:'
520 ls -l $TEXT_DIR | tee /dev/stderr | awk '{print $5}' | pysum
521 fi
522
523 # Better sorting
524 #LANG=C ls -l $TEXT_DIR
525}
526
527_copy-path() {
528 local src=$1 dest=$2
529 mkdir -p $(dirname $dest)
530 cp -v $src $dest
531}
532
533copy-web() {
534 find web \
535 \( -name _tmp -a -prune \) -o \
536 \( -name '*.css' -o -name '*.js' \) -a -printf '%p _release/VERSION/%p\n' |
537 xargs -n 2 -- $0 _copy-path
538}
539
540pretty-size() {
541 local path=$1
542 stat --format '%s' "$path" | python -c '
543import sys
544num_bytes = int(sys.stdin.read())
545print "{:,}".format(num_bytes)
546'
547}
548
549# NOTE: It might be better to link to files like this in the /release/ tree.
550# Although I am not signing them.
551
552# https://nodejs.org/dist/v8.11.4/SHASUMS256.txt.asc
553
554tarball-links-row-html() {
555 local version=$1
556
557 cat <<EOF
558<tr class="file-table-heading">
559 <td></td>
560 <td>File / SHA256 checksum</td>
561 <td class="size">Size</td>
562 <td></td>
563</tr>
564EOF
565
566 # we switched to .gz for oils-for-unix
567 # note: legacy names for old releases
568 for name in \
569 oils-for-unix-$version.tar.{gz,xz} \
570 oil-$version.tar.{gz,xz} \
571 oil-native-$version.tar.xz; do
572
573 local url="/download/$name" # The server URL
574 local path="../oilshell.org__deploy/download/$name"
575
576 # Don't show tarballs that don't exist
577 if [[ $name == oils-for-unix-* && ! -f $path ]]; then
578 continue
579 fi
580 if [[ $name == oil-native-* && ! -f $path ]]; then
581 continue
582 fi
583
584 local checksum
585 checksum=$(sha256sum $path | awk '{print $1}')
586 local size
587 size=$(pretty-size $path)
588
589 # TODO: Port this to oil with "commas" extension.
590
591 # Three columns: date, version, and links
592 cat <<EOF
593 <tr>
594 <td></td>
595 <td class="filename"><a href="$url">$name</a></td>
596 <td class="size">$size</td>
597 </tr>
598 <tr>
599 <td></td>
600 <td colspan=2 class="checksum">$checksum</td>
601 </tr>
602EOF
603 done
604}
605
606this-release-links() {
607 echo '<div class="file-table">'
608 echo '<table>'
609 tarball-links-row-html "$OIL_VERSION"
610 echo '</table>'
611 echo '</div>'
612}
613
614# Turn HTML comment into a download link
615add-date-and-links() {
616 local snippet
617 snippet=$(this-release-links)
618
619 awk -v date=$1 -v snippet="$snippet" '
620 /<!-- REPLACE_WITH_DOWNLOAD_LINKS -->/ {
621 print(snippet)
622 next
623 }
624
625 /<!-- REPLACE_WITH_DATE -->/ {
626 print(date)
627 next
628 }
629
630 # Everything else
631 { print }
632 '
633}
634
635patch-release-pages() {
636 local release_date
637 release_date=$(cat _build/release-date.txt)
638
639 local root=_release/VERSION
640
641 add-date-and-links $release_date < _tmp/release-index.html > $root/index.html
642 add-date-and-links $release_date < _tmp/release-quality.html > $root/quality.html
643}
644
645copy-release-pages() {
646 ### For testing without releasing
647
648 cat < _tmp/release-index.html > $root/index.html
649 cat < _tmp/release-quality.html > $root/quality.html
650}
651
652run-for-release() {
653 ### Build a tree. Requires _build/release-date.txt to exist
654
655 local root=_release/VERSION
656 mkdir -p $root/{doc,test,pub}
657
658 tour
659
660 # Metadata
661 cp -v _build/release-date.txt oil-version.txt $root
662
663 # Docs
664 # Writes _release/VERSION and _tmp/release-index.html
665 all-markdown
666 all-ref
667 all-redirects # backward compat
668
669 fmt-check # Needs to run *after* we build the HTML
670
671 patch-release-pages
672
673 write-metrics
674
675 # Problem: You can't preview it without .wwz!
676 # Maybe have local redirects VERSION/test/wild/ to
677 #
678 # Instead of linking, I should compress them all here.
679
680 copy-web
681
682 if command -v tree >/dev/null; then
683 tree $root
684 else
685 find $root
686 fi
687}
688
689soil-run() {
690 build/stamp.sh write-release-date
691
692 run-for-release
693}
694
695"$@"
696