OILS / build / doc.sh View on Github | oilshell.org

749 lines, 378 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# build/doc.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10# https://oilshell.org/release/$VERSION/
11# doc/
12# index.html
13# INSTALL.html
14# INSTALL-old.html
15
16readonly OIL_VERSION=$(head -n 1 oil-version.txt)
17export OIL_VERSION # for quick_ref.py
18
19THIS_DIR=$(readlink -f $(dirname $0))
20readonly THIS_DIR
21REPO_ROOT=$(cd $THIS_DIR/.. && pwd)
22readonly REPO_ROOT
23
24
25readonly HTML_BASE_DIR=_release/VERSION
26
27
28log() {
29 echo "$@" 1>&2
30}
31
32#
33# Deps (similar to doctools/cmark.sh and build/codegen.sh)
34#
35
36readonly MANDOC_DIR='_deps/mdocml-1.14.1'
37
38download-mandoc() {
39 mkdir -p _deps
40 wget --no-clobber --directory _deps \
41 https://mandoc.bsd.lv/snapshots/mdocml-1.14.1.tar.gz
42}
43
44build-mandoc() {
45 cd $MANDOC_DIR
46 ./configure
47 make
48}
49
50mandoc() {
51 $MANDOC_DIR/mandoc "$@"
52}
53
54# Places version is used
55#
56# - in --version
57# - in URL for every page? inside the binary
58# - in titles for index, install, osh-quick-ref TOC, etc.
59# - in deployment script
60
61# Run with environment variable
62help-gen() {
63 PYTHONPATH=.:vendor doctools/help_gen.py "$@"
64}
65
66cmark() {
67 # h2 and h3 are shown in TOC. The blog uses "legacy" h3 and h4.
68 PYTHONPATH=.:vendor doctools/cmark.py --toc-tag h2 --toc-tag h3 --toc-pretty-href "$@"
69}
70
71readonly MARKDOWN_DOCS=(
72 published
73
74 # polished
75 getting-started
76 portability
77 known-differences
78 ysh-error
79 error-handling
80 error-catalog
81 json
82 hay
83 simple-word-eval
84 quirks
85 warts
86
87 eggex
88 ysh-regex-api
89 upgrade-breakage
90 ysh-tour
91
92 style-guide
93 novelties
94
95 proc-func
96 block-literals
97 objects
98 types
99
100 # Data language
101 qsn
102 qtt
103 j8-notation
104 # Protocol
105 pretty-printing
106 stream-table-process
107 byo
108 ysh-doc-processing
109
110 lib-osh
111
112 doc-toolchain
113 doc-plugins
114 ul-table
115 idioms
116 shell-idioms
117 ysh-faq
118
119 language-influences
120 ysh-vs-python
121 ysh-vs-shell
122
123 syntactic-concepts
124 syntax-feelings
125 command-vs-expression-mode
126
127 # needs polish
128 # Note: docs about the YSH are prefixed 'ysh-'.
129 # data-model and command-vs-expression-mode span both OSH and YSH
130
131 index
132 faq-doc
133
134 options
135
136 old/index
137 old/project-tour
138 old/legacy-array
139 old/ysh-keywords
140 old/modules
141 old/expression-language
142 old/word-language
143 old/errors
144 old/ysh-builtins
145
146 io-builtins
147 unicode
148 framing
149 xtrace
150 headless
151 completion
152 strings
153 variables
154
155 # Internal stuff
156 interpreter-state
157 process-model
158 architecture-notes
159 parser-architecture
160)
161
162# Bug fix: Plain $(date) can output unicode characters (e.g. in Japanese
163# locale), which is loaded by Python into say u'\u5e74'. But the default
164# encoding in Python 2 is still 'ascii', which means that '%s' % u_str may
165# fail.
166#
167# I believe --rfc-e-mail should never output a Unicode character.
168#
169# A better fix would be to implement json_utf8.load(f), which doesn't decode
170# into unicode instances. This would remove useless conversions.
171
172readonly TIMESTAMP=$(date --rfc-email)
173
174split-and-render() {
175 local src=${1:-doc/known-differences.md}
176
177 local rel_path=${src%'.md'} # doc/known-differences
178 local tmp_prefix=_tmp/$rel_path # temp dir for splitting
179
180 local out=${2:-$HTML_BASE_DIR/$rel_path.html}
181 local web_url=${3:-'../web'}
182
183 mkdir -v -p $(dirname $out) $tmp_prefix
184
185 # Also add could add css_files. The one in the file takes precedence always?
186
187 # css_files: a space-separated list
188 # all_docs_url: so we link from doc/foo.html -> doc/
189
190 local css_files="$web_url/base.css $web_url/manual.css $web_url/toc.css $web_url/language.css $web_url/code.css"
191
192 doctools/split_doc.py \
193 -v build_timestamp="$TIMESTAMP" \
194 -v oil_version="$OIL_VERSION" \
195 -v css_files="$css_files" \
196 -v all_docs_url='.' \
197 -v repo_url="$src" \
198 $src $tmp_prefix
199
200 #ls -l _tmp/doc
201 #head _tmp/doc/*
202 #return
203
204 # for ysh-tour code blocks
205 local code_out=_tmp/code-blocks/$rel_path.txt
206 mkdir -v -p $(dirname $code_out)
207
208 cmark \
209 --code-block-output $code_out \
210 ${tmp_prefix}_meta.json ${tmp_prefix}_content.md > $out
211
212 log "$tmp_prefix -> (doctools/cmark) -> $out"
213}
214
215render-from-kate() {
216 ### Make it easier to configure Kate editor
217
218 # It want to pass an absolute path
219 # TODO: I can't figure out how to run this from Kate?
220
221 local full_path=$1
222
223 case $full_path in
224 $REPO_ROOT/*)
225 rel_path=${full_path#"$REPO_ROOT/"}
226 echo "relative path = $rel_path"
227 ;;
228 *)
229 die "$full_path should start with repo root $REPO_ROOT"
230 ;;
231 esac
232
233 split-and-render $rel_path
234}
235
236# Special case for README
237# Do NOT split because we don't want front matter in the markdown source.
238render-only() {
239 local src=${1:-README.md}
240
241 local name
242 case $src in
243 *.md)
244 name=$(basename $src .md)
245 ;;
246 *.txt)
247 name=$(basename $src .txt)
248 ;;
249 *)
250 name=$(basename $src)
251 ;;
252 esac
253
254 local out=${2:-$HTML_BASE_DIR/doc/$name.html}
255 local css_files=${3:-'../web/manual.css ../web/toc.css'}
256 local title=${4:-'Oils Source Code'}
257
258 local prefix=_tmp/doc/$name
259
260 local meta=${prefix}_meta.json
261 cat >$meta <<EOF
262{ "title": "$title",
263 "repo_url": "$src",
264 "css_files": "$css_files",
265 "all_docs_url": ".",
266
267 "build_timestamp": "$TIMESTAMP",
268 "oil_version": "$OIL_VERSION"
269}
270EOF
271
272 cmark $meta $src > $out
273 log "Wrote $out"
274}
275
276special() {
277 # TODO: do all READMEs
278 split-and-render mycpp/README.md \
279 $HTML_BASE_DIR/doc/oils-repo/mycpp/README.html \
280 ../../../web
281
282 local web_dir='../../web'
283 render-only 'README.md' $HTML_BASE_DIR/doc/oils-repo/README.html \
284 "$web_dir/base.css $web_dir/manual.css $web_dir/toc.css" 'Oils Source Code'
285
286 local web_dir='../web'
287 render-only INSTALL.txt '' \
288 "$web_dir/base.css $web_dir/install.css" 'Installing Oils'
289
290 render-only INSTALL-old.txt '' \
291 "$web_dir/base.css $web_dir/install.css" 'Installing Oils - old CPython build'
292
293 # These pages aren't in doc/
294 split-and-render doc/release-index.md _tmp/release-index.html
295 split-and-render doc/release-quality.md _tmp/release-quality.html
296}
297
298all-markdown() {
299 make-dirs
300
301 # TODO: We can set repo_url here! Then we don't need it for most docs.
302 # split_doc.py can return {} if the doc doesn't start with ---
303
304 #for d in doc/index.md doc/known-differences.md doc/*-manual.md \
305 # doc/eggex.md doc/oil-options.md doc/oil-func-proc-block.md; do
306 for d in "${MARKDOWN_DOCS[@]}"; do
307 split-and-render doc/$d.md
308 done
309
310 special
311}
312
313redir-body() {
314 local to_url=$1 # WARNING: no escaping
315 cat <<EOF
316<head>
317 <meta http-equiv="Refresh" content="0; URL=$to_url" />
318</head>
319EOF
320}
321
322redirect-pairs() {
323 # we want want /release/latest/ URLs to still work
324 cat <<EOF
325oil-language-tour ysh-tour
326oil-language-faq ysh-faq
327oil-help ysh-help
328oil-help-topics ysh-help-topics
329ysh-help ref/toc-ysh
330ysh-help-topics ref/toc-ysh
331EOF
332}
333
334all-redirects() {
335 redirect-pairs | while read -r from_page to_page; do
336 redir-body "$to_page.html" | tee "_release/VERSION/doc/$from_page.html"
337 done
338}
339
340# TODO: This could use some CSS.
341man-page() {
342 local root_dir=${1:-_release/VERSION}
343 mandoc -T html doc/osh.1 > $root_dir/osh.1.html
344 ls -l $root_dir
345}
346
347# I want to ship the INSTALL file literally, so just mutate things
348_sed-ext() {
349 sed --regexp-extended -i "$@"
350}
351
352update-src-versions() {
353 # Update tarball names, etc.
354 _sed-ext \
355 "s/[0-9]+\.[0-9]+\.[a-z0-9]+/$OIL_VERSION/g" \
356 doc/release-*.md INSTALL.txt INSTALL-old.txt
357
358 # Update /release/0.8.4/ URL, etc.
359 _sed-ext \
360 "s;/release/[0-9]+\.[0-9]+\.[a-z0-9]+/;/release/$OIL_VERSION/;g" \
361 doc/osh.1
362}
363
364#
365# Test Tools
366#
367
368split-doc-demo() {
369 cat > _tmp/testdoc.md <<EOF
370---
371title: foo
372---
373
374Title
375=====
376
377hello
378
379EOF
380
381 doctools/split_doc.py _tmp/testdoc.md _tmp/testdoc
382
383 head _tmp/testdoc*
384}
385
386#
387# Help is both markdown and text
388#
389
390readonly TMP_DIR=_tmp/doc
391readonly CODE_BLOCK_DIR=_tmp/code-blocks
392readonly TEXT_DIR=_devbuild/help
393readonly HTML_DIR=_release/VERSION
394readonly CODE_DIR=_devbuild/gen
395
396cards-from-indices() {
397 ### Make help cards
398
399 for lang in osh ysh data; do
400 help-gen cards-from-index $lang $TEXT_DIR \
401 < $HTML_DIR/doc/ref/toc-$lang.html
402 done
403}
404
405cards-from-chapters() {
406 ### Turn h3 topics into cards
407
408 local py_out=$CODE_DIR/help_meta.py
409
410 mkdir -p _gen/frontend
411 local cc_prefix=_gen/frontend/help_meta
412
413 help-gen cards-from-chapters $TEXT_DIR $py_out $cc_prefix \
414 $HTML_DIR/doc/ref/chap-*.html
415}
416
417ref-check() {
418 help-gen ref-check \
419 doc/ref/toc-*.md \
420 _release/VERSION/doc/ref/chap-*.html
421}
422
423fmt-check() {
424 PYTHONPATH=.:vendor doctools/fmt_check.py _release/VERSION/doc/ref/*.html
425}
426
427
428write-metrics() {
429 ### Check indexes and chapters against each other
430
431 local out=_release/VERSION/doc/metrics.txt
432
433 # send stderr to the log file too
434 ref-check > $out 2>&1
435
436 echo "Wrote $out"
437}
438
439tour() {
440 ### Build the Tour of YSH, and execute code as validation
441 local name=${1:-ysh-tour}
442
443 split-and-render doc/$name.md
444
445 local work_dir=$REPO_ROOT/_tmp/code-blocks/doc
446
447 mkdir -p $work_dir/lib
448
449 # Files used by module example
450 touch $work_dir/{build,test}.sh
451
452 cat >$work_dir/lines.txt <<'EOF'
453 doc/hello.md
454 "doc/with spaces.md"
455b'doc/with byte \yff.md'
456EOF
457
458 cat >$work_dir/myargs.ysh <<EOF
459const __provide__ = :| proc1 p2 p3 |
460
461proc proc1 {
462 echo proc1
463}
464
465proc p2 {
466 echo p2
467}
468
469proc p3 {
470 echo p3
471}
472EOF
473
474 cat >$work_dir/demo.py <<EOF
475#!/usr/bin/env python
476
477print("hi")
478EOF
479 chmod +x $work_dir/demo.py
480
481 cat >$work_dir/lib/util.ysh <<EOF
482const __provide__ = :| log |
483
484proc log {
485 echo @ARGV >&2
486}
487EOF
488
489 pushd $work_dir
490
491 # Prepend extra code
492 cat >tour.ysh - $name.txt <<EOF
493func myMethod(self) {
494 echo 'myMethod'
495}
496
497func mutatingMethod(self) {
498 echo 'mutatingMethod'
499}
500
501func makeMyObject(x) {
502 var methods = Object(null, {myMethod, 'M/mutatingMethod': mutatingMethod})
503 return (Object(methods, {x}))
504}
505EOF
506
507 # Fix: don't supply stdin!
508 $REPO_ROOT/bin/ysh tour.ysh < /dev/null
509 popd
510
511 # My own dev tools
512 # if test -d ~/vm-shared; then
513 if false; then
514 local path=_release/VERSION/doc/$name.html
515 cp -v $path ~/vm-shared/$path
516 fi
517}
518
519one() {
520 ### Iterate on one doc quickly
521
522 local name=${1:-options}
523
524 split-and-render doc/$name.md
525
526 # Make sure the doc has valid YSH code?
527 # TODO: Maybe need an attribute for OSH or YSH
528 pushd _tmp/code-blocks/doc
529 $REPO_ROOT/bin/ysh $name.txt
530 popd
531
532 if test -d ~/vm-shared; then
533 local out="${name%.md}.html"
534 local path=_release/VERSION/$out
535 cp -v $path ~/vm-shared/$path
536 fi
537}
538
539make-dirs() {
540 mkdir -p $TMP_DIR $CODE_BLOCK_DIR $TEXT_DIR $HTML_DIR/doc
541}
542
543one-ref() {
544 local md=${1:-doc/ref/index.md}
545 split-and-render $md '' '../../web'
546}
547
548all-ref() {
549 ### Build doc/ref in text and HTML. Depends on libcmark.so
550
551 log "Removing $TEXT_DIR/*"
552 rm -f $TEXT_DIR/*
553 make-dirs
554
555 # Make the indexes and chapters
556 for d in doc/ref/*.md; do
557 split-and-render $d '' '../../web'
558 done
559
560 # Note: if we want a $ref-topic shortcut, we might want to use Ninja to
561 # extract topics from all chapters first, and then make help_meta.json, like
562 # we have _devbuild/gen/help_meta.py.
563
564 # Text cards
565 cards-from-indices
566 # A few text cards, and HELP_TOPICS dict for URLs, for flat namespace
567 cards-from-chapters
568
569 if command -v pysum; then
570 # 19 KB of embedded help, seems OK. Biggest card is 'ysh-option'. Could
571 # compress it.
572 echo 'Size of embedded help:'
573 ls -l $TEXT_DIR | tee /dev/stderr | awk '{print $5}' | pysum
574 fi
575
576 # Better sorting
577 #LANG=C ls -l $TEXT_DIR
578}
579
580_copy-path() {
581 local src=$1 dest=$2
582 mkdir -p $(dirname $dest)
583 cp -v $src $dest
584}
585
586copy-web() {
587 find web \
588 \( -name _tmp -a -prune \) -o \
589 \( -name '*.css' -o -name '*.js' \) -a -printf '%p _release/VERSION/%p\n' |
590 xargs -n 2 -- $0 _copy-path
591}
592
593pretty-size() {
594 local path=$1
595 stat --format '%s' "$path" | python -c '
596import sys
597num_bytes = int(sys.stdin.read())
598print "{:,}".format(num_bytes)
599'
600}
601
602# NOTE: It might be better to link to files like this in the /release/ tree.
603# Although I am not signing them.
604
605# https://nodejs.org/dist/v8.11.4/SHASUMS256.txt.asc
606
607tarball-links-row-html() {
608 local version=$1
609
610 cat <<EOF
611<tr class="file-table-heading">
612 <td></td>
613 <td>File / SHA256 checksum</td>
614 <td class="size">Size</td>
615 <td></td>
616</tr>
617EOF
618
619 # we switched to .gz for oils-for-unix
620 # note: legacy names for old releases
621 for name in \
622 oils-for-unix-$version.tar.{gz,xz} \
623 oil-$version.tar.{gz,xz} \
624 oil-native-$version.tar.xz; do
625
626 local url="/download/$name" # The server URL
627 local path="../oilshell.org__deploy/download/$name"
628
629 # Don't show tarballs that don't exist
630 if [[ $name == oils-for-unix-* && ! -f $path ]]; then
631 continue
632 fi
633 if [[ $name == oil-native-* && ! -f $path ]]; then
634 continue
635 fi
636
637 local checksum
638 checksum=$(sha256sum $path | awk '{print $1}')
639 local size
640 size=$(pretty-size $path)
641
642 # TODO: Port this to oil with "commas" extension.
643
644 # Three columns: date, version, and links
645 cat <<EOF
646 <tr>
647 <td></td>
648 <td class="filename"><a href="$url">$name</a></td>
649 <td class="size">$size</td>
650 </tr>
651 <tr>
652 <td></td>
653 <td colspan=2 class="checksum">$checksum</td>
654 </tr>
655EOF
656 done
657}
658
659this-release-links() {
660 echo '<div class="file-table">'
661 echo '<table>'
662 tarball-links-row-html "$OIL_VERSION"
663 echo '</table>'
664 echo '</div>'
665}
666
667# Turn HTML comment into a download link
668add-date-and-links() {
669 local snippet
670 snippet=$(this-release-links)
671
672 awk -v date=$1 -v snippet="$snippet" '
673 /<!-- REPLACE_WITH_DOWNLOAD_LINKS -->/ {
674 print(snippet)
675 next
676 }
677
678 /<!-- REPLACE_WITH_DATE -->/ {
679 print(date)
680 next
681 }
682
683 # Everything else
684 { print }
685 '
686}
687
688patch-release-pages() {
689 local release_date
690 release_date=$(cat _build/release-date.txt)
691
692 local root=_release/VERSION
693
694 add-date-and-links $release_date < _tmp/release-index.html > $root/index.html
695 add-date-and-links $release_date < _tmp/release-quality.html > $root/quality.html
696}
697
698copy-release-pages() {
699 ### For testing without releasing
700
701 cat < _tmp/release-index.html > $root/index.html
702 cat < _tmp/release-quality.html > $root/quality.html
703}
704
705run-for-release() {
706 ### Build a tree. Requires _build/release-date.txt to exist
707
708 local root=_release/VERSION
709 mkdir -p $root/{doc,test,pub}
710
711 tour
712
713 # Metadata
714 cp -v _build/release-date.txt oil-version.txt $root
715
716 # Docs
717 # Writes _release/VERSION and _tmp/release-index.html
718 all-markdown
719 all-ref
720 all-redirects # backward compat
721
722 fmt-check # Needs to run *after* we build the HTML
723
724 patch-release-pages
725
726 write-metrics
727
728 # Problem: You can't preview it without .wwz!
729 # Maybe have local redirects VERSION/test/wild/ to
730 #
731 # Instead of linking, I should compress them all here.
732
733 copy-web
734
735 if command -v tree >/dev/null; then
736 tree $root
737 else
738 find $root
739 fi
740}
741
742soil-run() {
743 build/stamp.sh write-release-date
744
745 run-for-release
746}
747
748"$@"
749