OILS / build / doc.sh View on Github | oils.pub

754 lines, 382 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# build/doc.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10# https://oilshell.org/release/$VERSION/
11# doc/
12# index.html
13# INSTALL.html
14# INSTALL-old.html
15
16readonly OIL_VERSION=$(head -n 1 oils-version.txt)
17export OIL_VERSION # for quick_ref.py
18
19THIS_DIR=$(readlink -f $(dirname $0))
20readonly THIS_DIR
21REPO_ROOT=$(cd $THIS_DIR/.. && pwd)
22readonly REPO_ROOT
23
24
25readonly HTML_BASE_DIR=_release/VERSION
26
27
28log() {
29 echo "$@" 1>&2
30}
31
32#
33# Deps (similar to doctools/cmark.sh and build/codegen.sh)
34#
35
36readonly MANDOC_DIR='_deps/mdocml-1.14.1'
37
38download-mandoc() {
39 mkdir -p _deps
40 wget --no-clobber --directory _deps \
41 https://mandoc.bsd.lv/snapshots/mdocml-1.14.1.tar.gz
42}
43
44build-mandoc() {
45 cd $MANDOC_DIR
46 ./configure
47 make
48}
49
50mandoc() {
51 $MANDOC_DIR/mandoc "$@"
52}
53
54# Places version is used
55#
56# - in --version
57# - in URL for every page? inside the binary
58# - in titles for index, install, osh-quick-ref TOC, etc.
59# - in deployment script
60
61# Run with environment variable
62help-gen() {
63 PYTHONPATH=.:vendor doctools/help_gen.py "$@"
64}
65
66cmark() {
67 # h2 and h3 are shown in TOC. The blog uses "legacy" h3 and h4.
68 PYTHONPATH=.:vendor doctools/cmark.py --toc-tag h2 --toc-tag h3 --toc-pretty-href "$@"
69}
70
71readonly MARKDOWN_DOCS=(
72 published
73
74 # polished
75 getting-started
76 portability
77 known-differences
78 ysh-error
79 error-handling
80 error-catalog
81 json
82 hay
83 simple-word-eval
84 quirks
85 warts
86
87 eggex
88 ysh-regex-api
89 upgrade-breakage
90 ysh-tour
91
92 style-guide
93 novelties
94
95 proc-func
96 block-literals
97 objects
98 types
99
100 # Data language
101 qsn
102 qtt
103 j8-notation
104 htm8
105 # Protocol
106 pretty-printing
107 stream-table-process
108 byo
109 ysh-doc-processing
110
111 table-object-doc
112
113 lib-osh
114
115 doc-toolchain
116 doc-plugins
117 ul-table
118 ul-table-compare
119 idioms
120 shell-idioms
121 ysh-faq
122
123 language-influences
124 ysh-vs-python
125 ysh-vs-shell
126
127 syntactic-concepts
128 syntax-feelings
129 command-vs-expression-mode
130
131 # needs polish
132 # Note: docs about the YSH are prefixed 'ysh-'.
133 # data-model and command-vs-expression-mode span both OSH and YSH
134
135 index
136 faq-doc
137
138 options
139
140 old/index
141 old/project-tour
142 old/legacy-array
143 old/ysh-keywords
144 old/modules
145 old/expression-language
146 old/word-language
147 old/errors
148 old/ysh-builtins
149
150 io-builtins
151 unicode
152 framing
153 xtrace
154 headless
155 completion
156 strings
157 variables
158
159 # Internal stuff
160 interpreter-state
161 process-model
162 architecture-notes
163 parser-architecture
164)
165
166# Bug fix: Plain $(date) can output unicode characters (e.g. in Japanese
167# locale), which is loaded by Python into say u'\u5e74'. But the default
168# encoding in Python 2 is still 'ascii', which means that '%s' % u_str may
169# fail.
170#
171# I believe --rfc-e-mail should never output a Unicode character.
172#
173# A better fix would be to implement json_utf8.load(f), which doesn't decode
174# into unicode instances. This would remove useless conversions.
175
176readonly TIMESTAMP=$(date --rfc-email)
177
178split-and-render() {
179 local src=${1:-doc/known-differences.md}
180
181 local rel_path=${src%'.md'} # doc/known-differences
182 local tmp_prefix=_tmp/$rel_path # temp dir for splitting
183
184 local out=${2:-$HTML_BASE_DIR/$rel_path.html}
185 local web_url=${3:-'../web'}
186
187 mkdir -v -p $(dirname $out) $tmp_prefix
188
189 # Also add could add css_files. The one in the file takes precedence always?
190
191 # css_files: a space-separated list
192 # all_docs_url: so we link from doc/foo.html -> doc/
193
194 local css_files="$web_url/base.css $web_url/manual.css $web_url/toc.css $web_url/language.css $web_url/code.css"
195
196 PYTHONPATH='.:vendor' doctools/split_doc.py \
197 -v build_timestamp="$TIMESTAMP" \
198 -v oil_version="$OIL_VERSION" \
199 -v css_files="$css_files" \
200 -v all_docs_url='.' \
201 -v repo_url="$src" \
202 $src $tmp_prefix
203
204 #ls -l _tmp/doc
205 #head _tmp/doc/*
206 #return
207
208 # for ysh-tour code blocks
209 local code_out=_tmp/code-blocks/$rel_path.txt
210 mkdir -v -p $(dirname $code_out)
211
212 cmark \
213 --code-block-output $code_out \
214 ${tmp_prefix}_meta.json ${tmp_prefix}_content.md > $out
215
216 log "$tmp_prefix -> (doctools/cmark) -> $out"
217}
218
219render-from-kate() {
220 ### Make it easier to configure Kate editor
221
222 # It want to pass an absolute path
223 # TODO: I can't figure out how to run this from Kate?
224
225 local full_path=$1
226
227 case $full_path in
228 $REPO_ROOT/*)
229 rel_path=${full_path#"$REPO_ROOT/"}
230 echo "relative path = $rel_path"
231 ;;
232 *)
233 die "$full_path should start with repo root $REPO_ROOT"
234 ;;
235 esac
236
237 split-and-render $rel_path
238}
239
240# Special case for README
241# Do NOT split because we don't want front matter in the markdown source.
242render-only() {
243 local src=${1:-README.md}
244
245 local name
246 case $src in
247 *.md)
248 name=$(basename $src .md)
249 ;;
250 *.txt)
251 name=$(basename $src .txt)
252 ;;
253 *)
254 name=$(basename $src)
255 ;;
256 esac
257
258 local out=${2:-$HTML_BASE_DIR/doc/$name.html}
259 local css_files=${3:-'../web/manual.css ../web/toc.css'}
260 local title=${4:-'Oils Source Code'}
261
262 local prefix=_tmp/doc/$name
263
264 local meta=${prefix}_meta.json
265 cat >$meta <<EOF
266{ "title": "$title",
267 "repo_url": "$src",
268 "css_files": "$css_files",
269 "all_docs_url": ".",
270
271 "build_timestamp": "$TIMESTAMP",
272 "oil_version": "$OIL_VERSION"
273}
274EOF
275
276 cmark $meta $src > $out
277 log "Wrote $out"
278}
279
280special() {
281 # TODO: do all READMEs
282 split-and-render mycpp/README.md \
283 $HTML_BASE_DIR/doc/oils-repo/mycpp/README.html \
284 ../../../web
285
286 local web_dir='../../web'
287 render-only 'README.md' $HTML_BASE_DIR/doc/oils-repo/README.html \
288 "$web_dir/base.css $web_dir/manual.css $web_dir/toc.css" 'Oils Source Code'
289
290 local web_dir='../web'
291 render-only INSTALL.txt '' \
292 "$web_dir/base.css $web_dir/install.css" 'Installing Oils'
293
294 render-only INSTALL-old.txt '' \
295 "$web_dir/base.css $web_dir/install.css" 'Installing Oils - old CPython build'
296
297 # These pages aren't in doc/
298 split-and-render doc/release-index.md _tmp/release-index.html
299 split-and-render doc/release-quality.md _tmp/release-quality.html
300}
301
302all-markdown() {
303 make-dirs
304
305 # TODO: We can set repo_url here! Then we don't need it for most docs.
306 # split_doc.py can return {} if the doc doesn't start with ---
307
308 #for d in doc/index.md doc/known-differences.md doc/*-manual.md \
309 # doc/eggex.md doc/oil-options.md doc/oil-func-proc-block.md; do
310 for d in "${MARKDOWN_DOCS[@]}"; do
311 split-and-render doc/$d.md
312 done
313
314 special
315}
316
317redir-body() {
318 local to_url=$1 # WARNING: no escaping
319 cat <<EOF
320<head>
321 <meta http-equiv="Refresh" content="0; URL=$to_url" />
322</head>
323EOF
324}
325
326redirect-pairs() {
327 # we want want /release/latest/ URLs to still work
328 cat <<EOF
329oil-language-tour ysh-tour
330oil-language-faq ysh-faq
331oil-help ysh-help
332oil-help-topics ysh-help-topics
333ysh-help ref/toc-ysh
334ysh-help-topics ref/toc-ysh
335EOF
336}
337
338all-redirects() {
339 redirect-pairs | while read -r from_page to_page; do
340 redir-body "$to_page.html" | tee "_release/VERSION/doc/$from_page.html"
341 done
342}
343
344# TODO: This could use some CSS.
345man-page() {
346 local root_dir=${1:-_release/VERSION}
347 mandoc -T html doc/osh.1 > $root_dir/osh.1.html
348 ls -l $root_dir
349}
350
351# I want to ship the INSTALL file literally, so just mutate things
352_sed-ext() {
353 sed --regexp-extended -i "$@"
354}
355
356update-src-versions() {
357 # Update tarball names, etc.
358 _sed-ext \
359 "s/[0-9]+\.[0-9]+\.[a-z0-9]+/$OIL_VERSION/g" \
360 doc/release-*.md INSTALL.txt INSTALL-old.txt
361
362 # Update /release/0.8.4/ URL, etc.
363 _sed-ext \
364 "s;/release/[0-9]+\.[0-9]+\.[a-z0-9]+/;/release/$OIL_VERSION/;g" \
365 doc/osh.1
366}
367
368#
369# Test Tools
370#
371
372split-doc-demo() {
373 cat > _tmp/testdoc.md <<EOF
374---
375title: foo
376---
377
378Title
379=====
380
381hello
382
383EOF
384
385 doctools/split_doc.py _tmp/testdoc.md _tmp/testdoc
386
387 head _tmp/testdoc*
388}
389
390#
391# Help is both markdown and text
392#
393
394readonly TMP_DIR=_tmp/doc
395readonly CODE_BLOCK_DIR=_tmp/code-blocks
396readonly TEXT_DIR=_devbuild/help
397readonly HTML_DIR=_release/VERSION
398readonly CODE_DIR=_devbuild/gen
399
400cards-from-indices() {
401 ### Make help cards
402
403 for lang in osh ysh data; do
404 help-gen cards-from-index $lang $TEXT_DIR \
405 < $HTML_DIR/doc/ref/toc-$lang.html
406 done
407}
408
409cards-from-chapters() {
410 ### Turn h3 topics into cards
411
412 local py_out=$CODE_DIR/help_meta.py
413
414 mkdir -p _gen/frontend
415 local cc_prefix=_gen/frontend/help_meta
416
417 help-gen cards-from-chapters $TEXT_DIR $py_out $cc_prefix \
418 $HTML_DIR/doc/ref/chap-*.html
419}
420
421ref-check() {
422 help-gen ref-check \
423 doc/ref/toc-*.md \
424 _release/VERSION/doc/ref/chap-*.html
425}
426
427fmt-check() {
428 PYTHONPATH=.:vendor doctools/fmt_check.py _release/VERSION/doc/ref/*.html
429}
430
431
432write-metrics() {
433 ### Check indexes and chapters against each other
434
435 local out=_release/VERSION/doc/metrics.txt
436
437 # send stderr to the log file too
438 ref-check > $out 2>&1
439
440 echo "Wrote $out"
441}
442
443tour() {
444 ### Build the Tour of YSH, and execute code as validation
445 local name=${1:-ysh-tour}
446
447 split-and-render doc/$name.md
448
449 local work_dir=$REPO_ROOT/_tmp/code-blocks/doc
450
451 mkdir -p $work_dir/lib
452
453 # Files used by module example
454 touch $work_dir/{build,test}.sh
455
456 cat >$work_dir/lines.txt <<'EOF'
457 doc/hello.md
458 "doc/with spaces.md"
459b'doc/with byte \yff.md'
460EOF
461
462 cat >$work_dir/myargs.ysh <<EOF
463const __provide__ = :| proc1 p2 p3 |
464
465proc proc1 {
466 echo proc1
467}
468
469proc p2 {
470 echo p2
471}
472
473proc p3 {
474 echo p3
475}
476EOF
477
478 cat >$work_dir/demo.py <<EOF
479#!/usr/bin/env python
480
481print("hi")
482EOF
483 chmod +x $work_dir/demo.py
484
485 cat >$work_dir/lib/util.ysh <<EOF
486const __provide__ = :| log |
487
488proc log {
489 echo @ARGV >&2
490}
491EOF
492
493 pushd $work_dir
494
495 # Prepend extra code
496 cat >tour.ysh - $name.txt <<EOF
497func myMethod(self) {
498 echo 'myMethod'
499}
500
501func mutatingMethod(self) {
502 echo 'mutatingMethod'
503}
504
505func makeMyObject(x) {
506 var methods = Object(null, {myMethod, 'M/mutatingMethod': mutatingMethod})
507 return (Object(methods, {x}))
508}
509EOF
510
511 # Fix: don't supply stdin!
512 $REPO_ROOT/bin/ysh tour.ysh < /dev/null
513 popd
514
515 # My own dev tools
516 # if test -d ~/vm-shared; then
517 if false; then
518 local path=_release/VERSION/doc/$name.html
519 cp -v $path ~/vm-shared/$path
520 fi
521}
522
523one() {
524 ### Iterate on one doc quickly
525
526 local name=${1:-options}
527
528 split-and-render doc/$name.md
529
530 # Make sure the doc has valid YSH code?
531 # TODO: Maybe need an attribute for OSH or YSH
532 pushd _tmp/code-blocks/doc
533 $REPO_ROOT/bin/ysh $name.txt
534 popd
535
536 if test -d ~/vm-shared; then
537 local out="${name%.md}.html"
538 local path=_release/VERSION/$out
539 cp -v $path ~/vm-shared/$path
540 fi
541}
542
543make-dirs() {
544 mkdir -p $TMP_DIR $CODE_BLOCK_DIR $TEXT_DIR $HTML_DIR/doc
545}
546
547one-ref() {
548 local md=${1:-doc/ref/index.md}
549 split-and-render $md '' '../../web'
550}
551
552all-ref() {
553 ### Build doc/ref in text and HTML. Depends on libcmark.so
554
555 log "Removing $TEXT_DIR/*"
556 rm -f $TEXT_DIR/*
557 make-dirs
558
559 # Make the indexes and chapters
560 for d in doc/ref/*.md; do
561 split-and-render $d '' '../../web'
562 done
563
564 # Note: if we want a $ref-topic shortcut, we might want to use Ninja to
565 # extract topics from all chapters first, and then make help_meta.json, like
566 # we have _devbuild/gen/help_meta.py.
567
568 # Text cards
569 cards-from-indices
570 # A few text cards, and HELP_TOPICS dict for URLs, for flat namespace
571 cards-from-chapters
572
573 return
574 if command -v pysum; then
575 # 19 KB of embedded help, seems OK. Biggest card is 'ysh-option'. Could
576 # compress it.
577 echo 'Size of embedded help:'
578 ls -l $TEXT_DIR | tee /dev/stderr | awk '{print $5}' | pysum
579 fi
580
581 # Better sorting
582 #LANG=C ls -l $TEXT_DIR
583}
584
585_copy-path() {
586 local src=$1 dest=$2
587 mkdir -p $(dirname $dest)
588 cp -v $src $dest
589}
590
591copy-web() {
592 find web \
593 \( -name _tmp -a -prune \) -o \
594 \( -name '*.css' -o -name '*.js' \) -a -printf '%p _release/VERSION/%p\n' |
595 xargs -n 2 -- $0 _copy-path
596}
597
598pretty-size() {
599 local path=$1
600 stat --format '%s' "$path" | python -c '
601import sys
602num_bytes = int(sys.stdin.read())
603print "{:,}".format(num_bytes)
604'
605}
606
607# NOTE: It might be better to link to files like this in the /release/ tree.
608# Although I am not signing them.
609
610# https://nodejs.org/dist/v8.11.4/SHASUMS256.txt.asc
611
612tarball-links-row-html() {
613 local version=$1
614
615 cat <<EOF
616<tr class="file-table-heading">
617 <td></td>
618 <td>File / SHA256 checksum</td>
619 <td class="size">Size</td>
620 <td></td>
621</tr>
622EOF
623
624 # we switched to .gz for oils-for-unix
625 # note: legacy names for old releases
626 for name in \
627 oils-for-unix-$version.tar.{gz,xz} \
628 oil-$version.tar.{gz,xz} \
629 oil-native-$version.tar.xz; do
630
631 local url="/download/$name" # The server URL
632 local path="../oils.pub__deploy/download/$name"
633
634 # Don't show tarballs that don't exist
635 if [[ $name == oils-for-unix-* && ! -f $path ]]; then
636 continue
637 fi
638 if [[ $name == oil-native-* && ! -f $path ]]; then
639 continue
640 fi
641
642 local checksum
643 checksum=$(sha256sum $path | awk '{print $1}')
644 local size
645 size=$(pretty-size $path)
646
647 # TODO: Port this to oil with "commas" extension.
648
649 # Three columns: date, version, and links
650 cat <<EOF
651 <tr>
652 <td></td>
653 <td class="filename"><a href="$url">$name</a></td>
654 <td class="size">$size</td>
655 </tr>
656 <tr>
657 <td></td>
658 <td colspan=2 class="checksum">$checksum</td>
659 </tr>
660EOF
661 done
662}
663
664this-release-links() {
665 echo '<div class="file-table">'
666 echo '<table>'
667 tarball-links-row-html "$OIL_VERSION"
668 echo '</table>'
669 echo '</div>'
670}
671
672# Turn HTML comment into a download link
673add-date-and-links() {
674 local snippet
675 snippet=$(this-release-links)
676
677 awk -v date=$1 -v snippet="$snippet" '
678 /<!-- REPLACE_WITH_DOWNLOAD_LINKS -->/ {
679 print(snippet)
680 next
681 }
682
683 /<!-- REPLACE_WITH_DATE -->/ {
684 print(date)
685 next
686 }
687
688 # Everything else
689 { print }
690 '
691}
692
693patch-release-pages() {
694 local release_date
695 release_date=$(cat _build/release-date.txt)
696
697 local root=_release/VERSION
698
699 add-date-and-links $release_date < _tmp/release-index.html > $root/index.html
700 add-date-and-links $release_date < _tmp/release-quality.html > $root/quality.html
701}
702
703copy-release-pages() {
704 ### For testing without releasing
705
706 cat < _tmp/release-index.html > $root/index.html
707 cat < _tmp/release-quality.html > $root/quality.html
708}
709
710run-for-release() {
711 ### Build a tree. Requires _build/release-date.txt to exist
712
713 local root=_release/VERSION
714 mkdir -p $root/{doc,test,pub}
715
716 tour
717
718 # Metadata
719 cp -v _build/release-date.txt oils-version.txt $root
720
721 # Docs
722 # Writes _release/VERSION and _tmp/release-index.html
723 all-markdown
724 all-ref
725 all-redirects # backward compat
726
727 fmt-check # Needs to run *after* we build the HTML
728
729 patch-release-pages
730
731 write-metrics
732
733 # Problem: You can't preview it without .wwz!
734 # Maybe have local redirects VERSION/test/wild/ to
735 #
736 # Instead of linking, I should compress them all here.
737
738 copy-web
739
740 if command -v tree >/dev/null; then
741 tree $root
742 else
743 find $root
744 fi
745}
746
747soil-run() {
748 build/stamp.sh write-release-date
749
750 run-for-release
751}
752
753"$@"
754