OILS / build / doc.sh View on Github | oilshell.org

748 lines, 377 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# build/doc.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10# https://oilshell.org/release/$VERSION/
11# doc/
12# index.html
13# INSTALL.html
14# INSTALL-old.html
15
16readonly OIL_VERSION=$(head -n 1 oil-version.txt)
17export OIL_VERSION # for quick_ref.py
18
19THIS_DIR=$(readlink -f $(dirname $0))
20readonly THIS_DIR
21REPO_ROOT=$(cd $THIS_DIR/.. && pwd)
22readonly REPO_ROOT
23
24
25readonly HTML_BASE_DIR=_release/VERSION
26
27
28log() {
29 echo "$@" 1>&2
30}
31
32#
33# Deps (similar to doctools/cmark.sh and build/codegen.sh)
34#
35
36readonly MANDOC_DIR='_deps/mdocml-1.14.1'
37
38download-mandoc() {
39 mkdir -p _deps
40 wget --no-clobber --directory _deps \
41 https://mandoc.bsd.lv/snapshots/mdocml-1.14.1.tar.gz
42}
43
44build-mandoc() {
45 cd $MANDOC_DIR
46 ./configure
47 make
48}
49
50mandoc() {
51 $MANDOC_DIR/mandoc "$@"
52}
53
54# Places version is used
55#
56# - in --version
57# - in URL for every page? inside the binary
58# - in titles for index, install, osh-quick-ref TOC, etc.
59# - in deployment script
60
61# Run with environment variable
62help-gen() {
63 PYTHONPATH=. doctools/help_gen.py "$@"
64}
65
66cmark() {
67 # h2 and h3 are shown in TOC. The blog uses "legacy" h3 and h4.
68 PYTHONPATH=. doctools/cmark.py --toc-tag h2 --toc-tag h3 --toc-pretty-href "$@"
69}
70
71readonly MARKDOWN_DOCS=(
72 published
73
74 # polished
75 getting-started
76 portability
77 known-differences
78 ysh-error
79 error-handling
80 error-catalog
81 json
82 hay
83 simple-word-eval
84 quirks
85 warts
86
87 eggex
88 ysh-regex-api
89 upgrade-breakage
90 ysh-tour
91
92 style-guide
93 novelties
94
95 proc-func
96 block-literals
97 objects
98 types
99
100 # Data language
101 qsn
102 qtt
103 j8-notation
104 # Protocol
105 pretty-printing
106 stream-table-process
107 byo
108 ysh-doc-processing
109
110 lib-osh
111
112 doc-toolchain
113 doc-plugins
114 idioms
115 shell-idioms
116 ysh-faq
117
118 language-influences
119 ysh-vs-python
120 ysh-vs-shell
121
122 syntactic-concepts
123 syntax-feelings
124 command-vs-expression-mode
125
126 # needs polish
127 # Note: docs about the YSH are prefixed 'ysh-'.
128 # data-model and command-vs-expression-mode span both OSH and YSH
129
130 index
131 faq-doc
132
133 options
134
135 old/index
136 old/project-tour
137 old/legacy-array
138 old/ysh-keywords
139 old/modules
140 old/expression-language
141 old/word-language
142 old/errors
143 old/ysh-builtins
144
145 io-builtins
146 unicode
147 framing
148 xtrace
149 headless
150 completion
151 strings
152 variables
153
154 # Internal stuff
155 interpreter-state
156 process-model
157 architecture-notes
158 parser-architecture
159)
160
161# Bug fix: Plain $(date) can output unicode characters (e.g. in Japanese
162# locale), which is loaded by Python into say u'\u5e74'. But the default
163# encoding in Python 2 is still 'ascii', which means that '%s' % u_str may
164# fail.
165#
166# I believe --rfc-e-mail should never output a Unicode character.
167#
168# A better fix would be to implement json_utf8.load(f), which doesn't decode
169# into unicode instances. This would remove useless conversions.
170
171readonly TIMESTAMP=$(date --rfc-email)
172
173split-and-render() {
174 local src=${1:-doc/known-differences.md}
175
176 local rel_path=${src%'.md'} # doc/known-differences
177 local tmp_prefix=_tmp/$rel_path # temp dir for splitting
178
179 local out=${2:-$HTML_BASE_DIR/$rel_path.html}
180 local web_url=${3:-'../web'}
181
182 mkdir -v -p $(dirname $out) $tmp_prefix
183
184 # Also add could add css_files. The one in the file takes precedence always?
185
186 # css_files: a space-separated list
187 # all_docs_url: so we link from doc/foo.html -> doc/
188
189 local css_files="$web_url/base.css $web_url/manual.css $web_url/toc.css $web_url/language.css $web_url/code.css"
190
191 doctools/split_doc.py \
192 -v build_timestamp="$TIMESTAMP" \
193 -v oil_version="$OIL_VERSION" \
194 -v css_files="$css_files" \
195 -v all_docs_url='.' \
196 -v repo_url="$src" \
197 $src $tmp_prefix
198
199 #ls -l _tmp/doc
200 #head _tmp/doc/*
201 #return
202
203 # for ysh-tour code blocks
204 local code_out=_tmp/code-blocks/$rel_path.txt
205 mkdir -v -p $(dirname $code_out)
206
207 cmark \
208 --code-block-output $code_out \
209 ${tmp_prefix}_meta.json ${tmp_prefix}_content.md > $out
210
211 log "$tmp_prefix -> (doctools/cmark) -> $out"
212}
213
214render-from-kate() {
215 ### Make it easier to configure Kate editor
216
217 # It want to pass an absolute path
218 # TODO: I can't figure out how to run this from Kate?
219
220 local full_path=$1
221
222 case $full_path in
223 $REPO_ROOT/*)
224 rel_path=${full_path#"$REPO_ROOT/"}
225 echo "relative path = $rel_path"
226 ;;
227 *)
228 die "$full_path should start with repo root $REPO_ROOT"
229 ;;
230 esac
231
232 split-and-render $rel_path
233}
234
235# Special case for README
236# Do NOT split because we don't want front matter in the markdown source.
237render-only() {
238 local src=${1:-README.md}
239
240 local name
241 case $src in
242 *.md)
243 name=$(basename $src .md)
244 ;;
245 *.txt)
246 name=$(basename $src .txt)
247 ;;
248 *)
249 name=$(basename $src)
250 ;;
251 esac
252
253 local out=${2:-$HTML_BASE_DIR/doc/$name.html}
254 local css_files=${3:-'../web/manual.css ../web/toc.css'}
255 local title=${4:-'Oils Source Code'}
256
257 local prefix=_tmp/doc/$name
258
259 local meta=${prefix}_meta.json
260 cat >$meta <<EOF
261{ "title": "$title",
262 "repo_url": "$src",
263 "css_files": "$css_files",
264 "all_docs_url": ".",
265
266 "build_timestamp": "$TIMESTAMP",
267 "oil_version": "$OIL_VERSION"
268}
269EOF
270
271 cmark $meta $src > $out
272 log "Wrote $out"
273}
274
275special() {
276 # TODO: do all READMEs
277 split-and-render mycpp/README.md \
278 $HTML_BASE_DIR/doc/oils-repo/mycpp/README.html \
279 ../../../web
280
281 local web_dir='../../web'
282 render-only 'README.md' $HTML_BASE_DIR/doc/oils-repo/README.html \
283 "$web_dir/base.css $web_dir/manual.css $web_dir/toc.css" 'Oils Source Code'
284
285 local web_dir='../web'
286 render-only INSTALL.txt '' \
287 "$web_dir/base.css $web_dir/install.css" 'Installing Oils'
288
289 render-only INSTALL-old.txt '' \
290 "$web_dir/base.css $web_dir/install.css" 'Installing Oils - old CPython build'
291
292 # These pages aren't in doc/
293 split-and-render doc/release-index.md _tmp/release-index.html
294 split-and-render doc/release-quality.md _tmp/release-quality.html
295}
296
297all-markdown() {
298 make-dirs
299
300 # TODO: We can set repo_url here! Then we don't need it for most docs.
301 # split_doc.py can return {} if the doc doesn't start with ---
302
303 #for d in doc/index.md doc/known-differences.md doc/*-manual.md \
304 # doc/eggex.md doc/oil-options.md doc/oil-func-proc-block.md; do
305 for d in "${MARKDOWN_DOCS[@]}"; do
306 split-and-render doc/$d.md
307 done
308
309 special
310}
311
312redir-body() {
313 local to_url=$1 # WARNING: no escaping
314 cat <<EOF
315<head>
316 <meta http-equiv="Refresh" content="0; URL=$to_url" />
317</head>
318EOF
319}
320
321redirect-pairs() {
322 # we want want /release/latest/ URLs to still work
323 cat <<EOF
324oil-language-tour ysh-tour
325oil-language-faq ysh-faq
326oil-help ysh-help
327oil-help-topics ysh-help-topics
328ysh-help ref/toc-ysh
329ysh-help-topics ref/toc-ysh
330EOF
331}
332
333all-redirects() {
334 redirect-pairs | while read -r from_page to_page; do
335 redir-body "$to_page.html" | tee "_release/VERSION/doc/$from_page.html"
336 done
337}
338
339# TODO: This could use some CSS.
340man-page() {
341 local root_dir=${1:-_release/VERSION}
342 mandoc -T html doc/osh.1 > $root_dir/osh.1.html
343 ls -l $root_dir
344}
345
346# I want to ship the INSTALL file literally, so just mutate things
347_sed-ext() {
348 sed --regexp-extended -i "$@"
349}
350
351update-src-versions() {
352 # Update tarball names, etc.
353 _sed-ext \
354 "s/[0-9]+\.[0-9]+\.[a-z0-9]+/$OIL_VERSION/g" \
355 doc/release-*.md INSTALL.txt INSTALL-old.txt
356
357 # Update /release/0.8.4/ URL, etc.
358 _sed-ext \
359 "s;/release/[0-9]+\.[0-9]+\.[a-z0-9]+/;/release/$OIL_VERSION/;g" \
360 doc/osh.1
361}
362
363#
364# Test Tools
365#
366
367split-doc-demo() {
368 cat > _tmp/testdoc.md <<EOF
369---
370title: foo
371---
372
373Title
374=====
375
376hello
377
378EOF
379
380 doctools/split_doc.py _tmp/testdoc.md _tmp/testdoc
381
382 head _tmp/testdoc*
383}
384
385#
386# Help is both markdown and text
387#
388
389readonly TMP_DIR=_tmp/doc
390readonly CODE_BLOCK_DIR=_tmp/code-blocks
391readonly TEXT_DIR=_devbuild/help
392readonly HTML_DIR=_release/VERSION
393readonly CODE_DIR=_devbuild/gen
394
395cards-from-indices() {
396 ### Make help cards
397
398 for lang in osh ysh data; do
399 help-gen cards-from-index $lang $TEXT_DIR \
400 < $HTML_DIR/doc/ref/toc-$lang.html
401 done
402}
403
404cards-from-chapters() {
405 ### Turn h3 topics into cards
406
407 local py_out=$CODE_DIR/help_meta.py
408
409 mkdir -p _gen/frontend
410 local cc_prefix=_gen/frontend/help_meta
411
412 help-gen cards-from-chapters $TEXT_DIR $py_out $cc_prefix \
413 $HTML_DIR/doc/ref/chap-*.html
414}
415
416ref-check() {
417 help-gen ref-check \
418 doc/ref/toc-*.md \
419 _release/VERSION/doc/ref/chap-*.html
420}
421
422fmt-check() {
423 PYTHONPATH=. doctools/fmt_check.py _release/VERSION/doc/ref/*.html
424}
425
426
427write-metrics() {
428 ### Check indexes and chapters against each other
429
430 local out=_release/VERSION/doc/metrics.txt
431
432 # send stderr to the log file too
433 ref-check > $out 2>&1
434
435 echo "Wrote $out"
436}
437
438tour() {
439 ### Build the Tour of YSH, and execute code as validation
440 local name=${1:-ysh-tour}
441
442 split-and-render doc/$name.md
443
444 local work_dir=$REPO_ROOT/_tmp/code-blocks/doc
445
446 mkdir -p $work_dir/lib
447
448 # Files used by module example
449 touch $work_dir/{build,test}.sh
450
451 cat >$work_dir/lines.txt <<'EOF'
452 doc/hello.md
453 "doc/with spaces.md"
454b'doc/with byte \yff.md'
455EOF
456
457 cat >$work_dir/myargs.ysh <<EOF
458const __provide__ = :| proc1 p2 p3 |
459
460proc proc1 {
461 echo proc1
462}
463
464proc p2 {
465 echo p2
466}
467
468proc p3 {
469 echo p3
470}
471EOF
472
473 cat >$work_dir/demo.py <<EOF
474#!/usr/bin/env python
475
476print("hi")
477EOF
478 chmod +x $work_dir/demo.py
479
480 cat >$work_dir/lib/util.ysh <<EOF
481const __provide__ = :| log |
482
483proc log {
484 echo @ARGV >&2
485}
486EOF
487
488 pushd $work_dir
489
490 # Prepend extra code
491 cat >tour.ysh - $name.txt <<EOF
492func myMethod(self) {
493 echo 'myMethod'
494}
495
496func mutatingMethod(self) {
497 echo 'mutatingMethod'
498}
499
500func makeMyObject(x) {
501 var methods = Object(null, {myMethod, 'M/mutatingMethod': mutatingMethod})
502 return (Object(methods, {x}))
503}
504EOF
505
506 # Fix: don't supply stdin!
507 $REPO_ROOT/bin/ysh tour.ysh < /dev/null
508 popd
509
510 # My own dev tools
511 # if test -d ~/vm-shared; then
512 if false; then
513 local path=_release/VERSION/doc/$name.html
514 cp -v $path ~/vm-shared/$path
515 fi
516}
517
518one() {
519 ### Iterate on one doc quickly
520
521 local name=${1:-options}
522
523 split-and-render doc/$name.md
524
525 # Make sure the doc has valid YSH code?
526 # TODO: Maybe need an attribute for OSH or YSH
527 pushd _tmp/code-blocks/doc
528 $REPO_ROOT/bin/ysh $name.txt
529 popd
530
531 if test -d ~/vm-shared; then
532 local out="${name%.md}.html"
533 local path=_release/VERSION/$out
534 cp -v $path ~/vm-shared/$path
535 fi
536}
537
538make-dirs() {
539 mkdir -p $TMP_DIR $CODE_BLOCK_DIR $TEXT_DIR $HTML_DIR/doc
540}
541
542one-ref() {
543 local md=${1:-doc/ref/index.md}
544 split-and-render $md '' '../../web'
545}
546
547all-ref() {
548 ### Build doc/ref in text and HTML. Depends on libcmark.so
549
550 log "Removing $TEXT_DIR/*"
551 rm -f $TEXT_DIR/*
552 make-dirs
553
554 # Make the indexes and chapters
555 for d in doc/ref/*.md; do
556 split-and-render $d '' '../../web'
557 done
558
559 # Note: if we want a $ref-topic shortcut, we might want to use Ninja to
560 # extract topics from all chapters first, and then make help_meta.json, like
561 # we have _devbuild/gen/help_meta.py.
562
563 # Text cards
564 cards-from-indices
565 # A few text cards, and HELP_TOPICS dict for URLs, for flat namespace
566 cards-from-chapters
567
568 if command -v pysum; then
569 # 19 KB of embedded help, seems OK. Biggest card is 'ysh-option'. Could
570 # compress it.
571 echo 'Size of embedded help:'
572 ls -l $TEXT_DIR | tee /dev/stderr | awk '{print $5}' | pysum
573 fi
574
575 # Better sorting
576 #LANG=C ls -l $TEXT_DIR
577}
578
579_copy-path() {
580 local src=$1 dest=$2
581 mkdir -p $(dirname $dest)
582 cp -v $src $dest
583}
584
585copy-web() {
586 find web \
587 \( -name _tmp -a -prune \) -o \
588 \( -name '*.css' -o -name '*.js' \) -a -printf '%p _release/VERSION/%p\n' |
589 xargs -n 2 -- $0 _copy-path
590}
591
592pretty-size() {
593 local path=$1
594 stat --format '%s' "$path" | python -c '
595import sys
596num_bytes = int(sys.stdin.read())
597print "{:,}".format(num_bytes)
598'
599}
600
601# NOTE: It might be better to link to files like this in the /release/ tree.
602# Although I am not signing them.
603
604# https://nodejs.org/dist/v8.11.4/SHASUMS256.txt.asc
605
606tarball-links-row-html() {
607 local version=$1
608
609 cat <<EOF
610<tr class="file-table-heading">
611 <td></td>
612 <td>File / SHA256 checksum</td>
613 <td class="size">Size</td>
614 <td></td>
615</tr>
616EOF
617
618 # we switched to .gz for oils-for-unix
619 # note: legacy names for old releases
620 for name in \
621 oils-for-unix-$version.tar.{gz,xz} \
622 oil-$version.tar.{gz,xz} \
623 oil-native-$version.tar.xz; do
624
625 local url="/download/$name" # The server URL
626 local path="../oilshell.org__deploy/download/$name"
627
628 # Don't show tarballs that don't exist
629 if [[ $name == oils-for-unix-* && ! -f $path ]]; then
630 continue
631 fi
632 if [[ $name == oil-native-* && ! -f $path ]]; then
633 continue
634 fi
635
636 local checksum
637 checksum=$(sha256sum $path | awk '{print $1}')
638 local size
639 size=$(pretty-size $path)
640
641 # TODO: Port this to oil with "commas" extension.
642
643 # Three columns: date, version, and links
644 cat <<EOF
645 <tr>
646 <td></td>
647 <td class="filename"><a href="$url">$name</a></td>
648 <td class="size">$size</td>
649 </tr>
650 <tr>
651 <td></td>
652 <td colspan=2 class="checksum">$checksum</td>
653 </tr>
654EOF
655 done
656}
657
658this-release-links() {
659 echo '<div class="file-table">'
660 echo '<table>'
661 tarball-links-row-html "$OIL_VERSION"
662 echo '</table>'
663 echo '</div>'
664}
665
666# Turn HTML comment into a download link
667add-date-and-links() {
668 local snippet
669 snippet=$(this-release-links)
670
671 awk -v date=$1 -v snippet="$snippet" '
672 /<!-- REPLACE_WITH_DOWNLOAD_LINKS -->/ {
673 print(snippet)
674 next
675 }
676
677 /<!-- REPLACE_WITH_DATE -->/ {
678 print(date)
679 next
680 }
681
682 # Everything else
683 { print }
684 '
685}
686
687patch-release-pages() {
688 local release_date
689 release_date=$(cat _build/release-date.txt)
690
691 local root=_release/VERSION
692
693 add-date-and-links $release_date < _tmp/release-index.html > $root/index.html
694 add-date-and-links $release_date < _tmp/release-quality.html > $root/quality.html
695}
696
697copy-release-pages() {
698 ### For testing without releasing
699
700 cat < _tmp/release-index.html > $root/index.html
701 cat < _tmp/release-quality.html > $root/quality.html
702}
703
704run-for-release() {
705 ### Build a tree. Requires _build/release-date.txt to exist
706
707 local root=_release/VERSION
708 mkdir -p $root/{doc,test,pub}
709
710 tour
711
712 # Metadata
713 cp -v _build/release-date.txt oil-version.txt $root
714
715 # Docs
716 # Writes _release/VERSION and _tmp/release-index.html
717 all-markdown
718 all-ref
719 all-redirects # backward compat
720
721 fmt-check # Needs to run *after* we build the HTML
722
723 patch-release-pages
724
725 write-metrics
726
727 # Problem: You can't preview it without .wwz!
728 # Maybe have local redirects VERSION/test/wild/ to
729 #
730 # Instead of linking, I should compress them all here.
731
732 copy-web
733
734 if command -v tree >/dev/null; then
735 tree $root
736 else
737 find $root
738 fi
739}
740
741soil-run() {
742 build/stamp.sh write-release-date
743
744 run-for-release
745}
746
747"$@"
748