1 | #!/usr/bin/env bash
|
2 | #
|
3 | # Usage:
|
4 | # build/doc.sh <function name>
|
5 |
|
6 | set -o nounset
|
7 | set -o pipefail
|
8 | set -o errexit
|
9 |
|
10 | # https://oilshell.org/release/$VERSION/
|
11 | # doc/
|
12 | # index.html
|
13 | # INSTALL.html
|
14 | # INSTALL-old.html
|
15 |
|
16 | readonly OIL_VERSION=$(head -n 1 oil-version.txt)
|
17 | export OIL_VERSION # for quick_ref.py
|
18 |
|
19 | THIS_DIR=$(readlink -f $(dirname $0))
|
20 | readonly THIS_DIR
|
21 | REPO_ROOT=$(cd $THIS_DIR/.. && pwd)
|
22 | readonly REPO_ROOT
|
23 |
|
24 |
|
25 | readonly HTML_BASE_DIR=_release/VERSION
|
26 |
|
27 |
|
28 | log() {
|
29 | echo "$@" 1>&2
|
30 | }
|
31 |
|
32 | #
|
33 | # Deps (similar to doctools/cmark.sh and build/codegen.sh)
|
34 | #
|
35 |
|
36 | readonly MANDOC_DIR='_deps/mdocml-1.14.1'
|
37 |
|
38 | download-mandoc() {
|
39 | mkdir -p _deps
|
40 | wget --no-clobber --directory _deps \
|
41 | https://mandoc.bsd.lv/snapshots/mdocml-1.14.1.tar.gz
|
42 | }
|
43 |
|
44 | build-mandoc() {
|
45 | cd $MANDOC_DIR
|
46 | ./configure
|
47 | make
|
48 | }
|
49 |
|
50 | mandoc() {
|
51 | $MANDOC_DIR/mandoc "$@"
|
52 | }
|
53 |
|
54 | # Places version is used
|
55 | #
|
56 | # - in --version
|
57 | # - in URL for every page? inside the binary
|
58 | # - in titles for index, install, osh-quick-ref TOC, etc.
|
59 | # - in deployment script
|
60 |
|
61 | # Run with environment variable
|
62 | help-gen() {
|
63 | PYTHONPATH=. doctools/help_gen.py "$@"
|
64 | }
|
65 |
|
66 | cmark() {
|
67 | # h2 and h3 are shown in TOC. The blog uses "legacy" h3 and h4.
|
68 | PYTHONPATH=. doctools/cmark.py --toc-tag h2 --toc-tag h3 --toc-pretty-href "$@"
|
69 | }
|
70 |
|
71 | readonly MARKDOWN_DOCS=(
|
72 | published
|
73 |
|
74 | # polished
|
75 | getting-started
|
76 | portability
|
77 | known-differences
|
78 | ysh-error
|
79 | error-handling
|
80 | error-catalog
|
81 | json
|
82 | hay
|
83 | simple-word-eval
|
84 | quirks
|
85 | warts
|
86 |
|
87 | eggex
|
88 | ysh-regex-api
|
89 | upgrade-breakage
|
90 | ysh-tour
|
91 |
|
92 | style-guide
|
93 | novelties
|
94 |
|
95 | proc-func
|
96 | block-literals
|
97 |
|
98 | # Data language
|
99 | qsn
|
100 | qtt
|
101 | j8-notation
|
102 | # Protocol
|
103 | pretty-printing
|
104 | stream-table-process
|
105 | byo
|
106 | ysh-doc-processing
|
107 |
|
108 | lib-osh
|
109 |
|
110 | doc-toolchain
|
111 | doc-plugins
|
112 | idioms
|
113 | shell-idioms
|
114 | ysh-faq
|
115 |
|
116 | language-influences
|
117 | ysh-vs-python
|
118 | ysh-vs-shell
|
119 |
|
120 | syntactic-concepts
|
121 | syntax-feelings
|
122 | command-vs-expression-mode
|
123 |
|
124 | # needs polish
|
125 | # Note: docs about the YSH are prefixed 'ysh-'.
|
126 | # data-model and command-vs-expression-mode span both OSH and YSH
|
127 |
|
128 | index
|
129 | faq-doc
|
130 |
|
131 | options
|
132 |
|
133 | old/index
|
134 | old/project-tour
|
135 | old/legacy-array
|
136 | old/ysh-keywords
|
137 | old/modules
|
138 | old/expression-language
|
139 | old/word-language
|
140 | old/errors
|
141 | old/ysh-builtins
|
142 |
|
143 | io-builtins
|
144 | unicode
|
145 | framing
|
146 | xtrace
|
147 | headless
|
148 | completion
|
149 | strings
|
150 | variables
|
151 |
|
152 | # Internal stuff
|
153 | interpreter-state
|
154 | process-model
|
155 | architecture-notes
|
156 | parser-architecture
|
157 | )
|
158 |
|
159 | # Bug fix: Plain $(date) can output unicode characters (e.g. in Japanese
|
160 | # locale), which is loaded by Python into say u'\u5e74'. But the default
|
161 | # encoding in Python 2 is still 'ascii', which means that '%s' % u_str may
|
162 | # fail.
|
163 | #
|
164 | # I believe --rfc-e-mail should never output a Unicode character.
|
165 | #
|
166 | # A better fix would be to implement json_utf8.load(f), which doesn't decode
|
167 | # into unicode instances. This would remove useless conversions.
|
168 |
|
169 | readonly TIMESTAMP=$(date --rfc-email)
|
170 |
|
171 | split-and-render() {
|
172 | local src=${1:-doc/known-differences.md}
|
173 |
|
174 | local rel_path=${src%'.md'} # doc/known-differences
|
175 | local tmp_prefix=_tmp/$rel_path # temp dir for splitting
|
176 |
|
177 | local out=${2:-$HTML_BASE_DIR/$rel_path.html}
|
178 | local web_url=${3:-'../web'}
|
179 |
|
180 | mkdir -v -p $(dirname $out) $tmp_prefix
|
181 |
|
182 | # Also add could add css_files. The one in the file takes precedence always?
|
183 |
|
184 | # css_files: a space-separated list
|
185 | # all_docs_url: so we link from doc/foo.html -> doc/
|
186 |
|
187 | local css_files="$web_url/base.css $web_url/manual.css $web_url/toc.css $web_url/language.css $web_url/code.css"
|
188 |
|
189 | doctools/split_doc.py \
|
190 | -v build_timestamp="$TIMESTAMP" \
|
191 | -v oil_version="$OIL_VERSION" \
|
192 | -v css_files="$css_files" \
|
193 | -v all_docs_url='.' \
|
194 | -v repo_url="$src" \
|
195 | $src $tmp_prefix
|
196 |
|
197 | #ls -l _tmp/doc
|
198 | #head _tmp/doc/*
|
199 | #return
|
200 |
|
201 | # for ysh-tour code blocks
|
202 | local code_out=_tmp/code-blocks/$rel_path.txt
|
203 | mkdir -v -p $(dirname $code_out)
|
204 |
|
205 | cmark \
|
206 | --code-block-output $code_out \
|
207 | ${tmp_prefix}_meta.json ${tmp_prefix}_content.md > $out
|
208 |
|
209 | log "$tmp_prefix -> (doctools/cmark) -> $out"
|
210 | }
|
211 |
|
212 | render-from-kate() {
|
213 | ### Make it easier to configure Kate editor
|
214 |
|
215 | # It want to pass an absolute path
|
216 | # TODO: I can't figure out how to run this from Kate?
|
217 |
|
218 | local full_path=$1
|
219 |
|
220 | case $full_path in
|
221 | $REPO_ROOT/*)
|
222 | rel_path=${full_path#"$REPO_ROOT/"}
|
223 | echo "relative path = $rel_path"
|
224 | ;;
|
225 | *)
|
226 | die "$full_path should start with repo root $REPO_ROOT"
|
227 | ;;
|
228 | esac
|
229 |
|
230 | split-and-render $rel_path
|
231 | }
|
232 |
|
233 | # Special case for README
|
234 | # Do NOT split because we don't want front matter in the markdown source.
|
235 | render-only() {
|
236 | local src=${1:-README.md}
|
237 |
|
238 | local name
|
239 | case $src in
|
240 | *.md)
|
241 | name=$(basename $src .md)
|
242 | ;;
|
243 | *.txt)
|
244 | name=$(basename $src .txt)
|
245 | ;;
|
246 | *)
|
247 | name=$(basename $src)
|
248 | ;;
|
249 | esac
|
250 |
|
251 | local out=${2:-$HTML_BASE_DIR/doc/$name.html}
|
252 | local css_files=${3:-'../web/manual.css ../web/toc.css'}
|
253 | local title=${4:-'Oils Source Code'}
|
254 |
|
255 | local prefix=_tmp/doc/$name
|
256 |
|
257 | local meta=${prefix}_meta.json
|
258 | cat >$meta <<EOF
|
259 | { "title": "$title",
|
260 | "repo_url": "$src",
|
261 | "css_files": "$css_files",
|
262 | "all_docs_url": ".",
|
263 |
|
264 | "build_timestamp": "$TIMESTAMP",
|
265 | "oil_version": "$OIL_VERSION"
|
266 | }
|
267 | EOF
|
268 |
|
269 | cmark $meta $src > $out
|
270 | log "Wrote $out"
|
271 | }
|
272 |
|
273 | special() {
|
274 | # TODO: do all READMEs
|
275 | split-and-render mycpp/README.md \
|
276 | $HTML_BASE_DIR/doc/oils-repo/mycpp/README.html \
|
277 | ../../../web
|
278 |
|
279 | local web_dir='../../web'
|
280 | render-only 'README.md' $HTML_BASE_DIR/doc/oils-repo/README.html \
|
281 | "$web_dir/base.css $web_dir/manual.css $web_dir/toc.css" 'Oils Source Code'
|
282 |
|
283 | local web_dir='../web'
|
284 | render-only INSTALL.txt '' \
|
285 | "$web_dir/base.css $web_dir/install.css" 'Installing Oils'
|
286 |
|
287 | render-only INSTALL-old.txt '' \
|
288 | "$web_dir/base.css $web_dir/install.css" 'Installing Oils - old CPython build'
|
289 |
|
290 | # These pages aren't in doc/
|
291 | split-and-render doc/release-index.md _tmp/release-index.html
|
292 | split-and-render doc/release-quality.md _tmp/release-quality.html
|
293 | }
|
294 |
|
295 | all-markdown() {
|
296 | make-dirs
|
297 |
|
298 | # TODO: We can set repo_url here! Then we don't need it for most docs.
|
299 | # split_doc.py can return {} if the doc doesn't start with ---
|
300 |
|
301 | #for d in doc/index.md doc/known-differences.md doc/*-manual.md \
|
302 | # doc/eggex.md doc/oil-options.md doc/oil-func-proc-block.md; do
|
303 | for d in "${MARKDOWN_DOCS[@]}"; do
|
304 | split-and-render doc/$d.md
|
305 | done
|
306 |
|
307 | special
|
308 | }
|
309 |
|
310 | redir-body() {
|
311 | local to_url=$1 # WARNING: no escaping
|
312 | cat <<EOF
|
313 | <head>
|
314 | <meta http-equiv="Refresh" content="0; URL=$to_url" />
|
315 | </head>
|
316 | EOF
|
317 | }
|
318 |
|
319 | redirect-pairs() {
|
320 | # we want want /release/latest/ URLs to still work
|
321 | cat <<EOF
|
322 | oil-language-tour ysh-tour
|
323 | oil-language-faq ysh-faq
|
324 | oil-help ysh-help
|
325 | oil-help-topics ysh-help-topics
|
326 | ysh-help ref/toc-ysh
|
327 | ysh-help-topics ref/toc-ysh
|
328 | EOF
|
329 | }
|
330 |
|
331 | all-redirects() {
|
332 | redirect-pairs | while read -r from_page to_page; do
|
333 | redir-body "$to_page.html" | tee "_release/VERSION/doc/$from_page.html"
|
334 | done
|
335 | }
|
336 |
|
337 | # TODO: This could use some CSS.
|
338 | man-page() {
|
339 | local root_dir=${1:-_release/VERSION}
|
340 | mandoc -T html doc/osh.1 > $root_dir/osh.1.html
|
341 | ls -l $root_dir
|
342 | }
|
343 |
|
344 | # I want to ship the INSTALL file literally, so just mutate things
|
345 | _sed-ext() {
|
346 | sed --regexp-extended -i "$@"
|
347 | }
|
348 |
|
349 | update-src-versions() {
|
350 | # Update tarball names, etc.
|
351 | _sed-ext \
|
352 | "s/[0-9]+\.[0-9]+\.[a-z0-9]+/$OIL_VERSION/g" \
|
353 | doc/release-*.md INSTALL.txt INSTALL-old.txt
|
354 |
|
355 | # Update /release/0.8.4/ URL, etc.
|
356 | _sed-ext \
|
357 | "s;/release/[0-9]+\.[0-9]+\.[a-z0-9]+/;/release/$OIL_VERSION/;g" \
|
358 | doc/osh.1
|
359 | }
|
360 |
|
361 | #
|
362 | # Test Tools
|
363 | #
|
364 |
|
365 | split-doc-demo() {
|
366 | cat > _tmp/testdoc.md <<EOF
|
367 | ---
|
368 | title: foo
|
369 | ---
|
370 |
|
371 | Title
|
372 | =====
|
373 |
|
374 | hello
|
375 |
|
376 | EOF
|
377 |
|
378 | doctools/split_doc.py _tmp/testdoc.md _tmp/testdoc
|
379 |
|
380 | head _tmp/testdoc*
|
381 | }
|
382 |
|
383 | #
|
384 | # Help is both markdown and text
|
385 | #
|
386 |
|
387 | readonly TMP_DIR=_tmp/doc
|
388 | readonly CODE_BLOCK_DIR=_tmp/code-blocks
|
389 | readonly TEXT_DIR=_devbuild/help
|
390 | readonly HTML_DIR=_release/VERSION
|
391 | readonly CODE_DIR=_devbuild/gen
|
392 |
|
393 | cards-from-indices() {
|
394 | ### Make help cards
|
395 |
|
396 | for lang in osh ysh data; do
|
397 | help-gen cards-from-index $lang $TEXT_DIR \
|
398 | < $HTML_DIR/doc/ref/toc-$lang.html
|
399 | done
|
400 | }
|
401 |
|
402 | cards-from-chapters() {
|
403 | ### Turn h3 topics into cards
|
404 |
|
405 | local py_out=$CODE_DIR/help_meta.py
|
406 |
|
407 | mkdir -p _gen/frontend
|
408 | local cc_prefix=_gen/frontend/help_meta
|
409 |
|
410 | help-gen cards-from-chapters $TEXT_DIR $py_out $cc_prefix \
|
411 | $HTML_DIR/doc/ref/chap-*.html
|
412 | }
|
413 |
|
414 | ref-check() {
|
415 | help-gen ref-check \
|
416 | doc/ref/toc-*.md \
|
417 | _release/VERSION/doc/ref/chap-*.html
|
418 | }
|
419 |
|
420 | fmt-check() {
|
421 | PYTHONPATH=. doctools/fmt_check.py _release/VERSION/doc/ref/*.html
|
422 | }
|
423 |
|
424 |
|
425 | write-metrics() {
|
426 | ### Check indexes and chapters against each other
|
427 |
|
428 | local out=_release/VERSION/doc/metrics.txt
|
429 |
|
430 | # send stderr to the log file too
|
431 | ref-check > $out 2>&1
|
432 |
|
433 | echo "Wrote $out"
|
434 | }
|
435 |
|
436 | tour() {
|
437 | ### Build the Tour of YSH, and execute code as validation
|
438 | local name=${1:-ysh-tour}
|
439 |
|
440 | split-and-render doc/$name.md
|
441 |
|
442 | local work_dir=$REPO_ROOT/_tmp/code-blocks/doc
|
443 |
|
444 | mkdir -p $work_dir/lib
|
445 |
|
446 | # Files used by module example
|
447 | touch $work_dir/{build,test}.sh
|
448 |
|
449 | cat >$work_dir/lib/util.ysh <<EOF
|
450 | log() { echo "$@" 1>&2; }
|
451 | EOF
|
452 |
|
453 | pushd $work_dir
|
454 | # Fix: don't supply stdin!
|
455 | $REPO_ROOT/bin/ysh $name.txt < /dev/null
|
456 | popd
|
457 |
|
458 | # My own dev tools
|
459 | # if test -d ~/vm-shared; then
|
460 | if false; then
|
461 | local path=_release/VERSION/doc/$name.html
|
462 | cp -v $path ~/vm-shared/$path
|
463 | fi
|
464 | }
|
465 |
|
466 | one() {
|
467 | ### Iterate on one doc quickly
|
468 |
|
469 | local name=${1:-options}
|
470 |
|
471 | split-and-render doc/$name.md
|
472 |
|
473 | # Make sure the doc has valid YSH code?
|
474 | # TODO: Maybe need an attribute for OSH or YSH
|
475 | pushd _tmp/code-blocks/doc
|
476 | $REPO_ROOT/bin/ysh $name.txt
|
477 | popd
|
478 |
|
479 | if test -d ~/vm-shared; then
|
480 | local out="${name%.md}.html"
|
481 | local path=_release/VERSION/$out
|
482 | cp -v $path ~/vm-shared/$path
|
483 | fi
|
484 | }
|
485 |
|
486 | make-dirs() {
|
487 | mkdir -p $TMP_DIR $CODE_BLOCK_DIR $TEXT_DIR $HTML_DIR/doc
|
488 | }
|
489 |
|
490 | one-ref() {
|
491 | local md=${1:-doc/ref/index.md}
|
492 | split-and-render $md '' '../../web'
|
493 | }
|
494 |
|
495 | all-ref() {
|
496 | ### Build doc/ref in text and HTML. Depends on libcmark.so
|
497 |
|
498 | log "Removing $TEXT_DIR/*"
|
499 | rm -f $TEXT_DIR/*
|
500 | make-dirs
|
501 |
|
502 | # Make the indexes and chapters
|
503 | for d in doc/ref/*.md; do
|
504 | split-and-render $d '' '../../web'
|
505 | done
|
506 |
|
507 | # Note: if we want a $ref-topic shortcut, we might want to use Ninja to
|
508 | # extract topics from all chapters first, and then make help_meta.json, like
|
509 | # we have _devbuild/gen/help_meta.py.
|
510 |
|
511 | # Text cards
|
512 | cards-from-indices
|
513 | # A few text cards, and HELP_TOPICS dict for URLs, for flat namespace
|
514 | cards-from-chapters
|
515 |
|
516 | if command -v pysum; then
|
517 | # 19 KB of embedded help, seems OK. Biggest card is 'ysh-option'. Could
|
518 | # compress it.
|
519 | echo 'Size of embedded help:'
|
520 | ls -l $TEXT_DIR | tee /dev/stderr | awk '{print $5}' | pysum
|
521 | fi
|
522 |
|
523 | # Better sorting
|
524 | #LANG=C ls -l $TEXT_DIR
|
525 | }
|
526 |
|
527 | _copy-path() {
|
528 | local src=$1 dest=$2
|
529 | mkdir -p $(dirname $dest)
|
530 | cp -v $src $dest
|
531 | }
|
532 |
|
533 | copy-web() {
|
534 | find web \
|
535 | \( -name _tmp -a -prune \) -o \
|
536 | \( -name '*.css' -o -name '*.js' \) -a -printf '%p _release/VERSION/%p\n' |
|
537 | xargs -n 2 -- $0 _copy-path
|
538 | }
|
539 |
|
540 | pretty-size() {
|
541 | local path=$1
|
542 | stat --format '%s' "$path" | python -c '
|
543 | import sys
|
544 | num_bytes = int(sys.stdin.read())
|
545 | print "{:,}".format(num_bytes)
|
546 | '
|
547 | }
|
548 |
|
549 | # NOTE: It might be better to link to files like this in the /release/ tree.
|
550 | # Although I am not signing them.
|
551 |
|
552 | # https://nodejs.org/dist/v8.11.4/SHASUMS256.txt.asc
|
553 |
|
554 | tarball-links-row-html() {
|
555 | local version=$1
|
556 |
|
557 | cat <<EOF
|
558 | <tr class="file-table-heading">
|
559 | <td></td>
|
560 | <td>File / SHA256 checksum</td>
|
561 | <td class="size">Size</td>
|
562 | <td></td>
|
563 | </tr>
|
564 | EOF
|
565 |
|
566 | # we switched to .gz for oils-for-unix
|
567 | # note: legacy names for old releases
|
568 | for name in \
|
569 | oils-for-unix-$version.tar.{gz,xz} \
|
570 | oil-$version.tar.{gz,xz} \
|
571 | oil-native-$version.tar.xz; do
|
572 |
|
573 | local url="/download/$name" # The server URL
|
574 | local path="../oilshell.org__deploy/download/$name"
|
575 |
|
576 | # Don't show tarballs that don't exist
|
577 | if [[ $name == oils-for-unix-* && ! -f $path ]]; then
|
578 | continue
|
579 | fi
|
580 | if [[ $name == oil-native-* && ! -f $path ]]; then
|
581 | continue
|
582 | fi
|
583 |
|
584 | local checksum
|
585 | checksum=$(sha256sum $path | awk '{print $1}')
|
586 | local size
|
587 | size=$(pretty-size $path)
|
588 |
|
589 | # TODO: Port this to oil with "commas" extension.
|
590 |
|
591 | # Three columns: date, version, and links
|
592 | cat <<EOF
|
593 | <tr>
|
594 | <td></td>
|
595 | <td class="filename"><a href="$url">$name</a></td>
|
596 | <td class="size">$size</td>
|
597 | </tr>
|
598 | <tr>
|
599 | <td></td>
|
600 | <td colspan=2 class="checksum">$checksum</td>
|
601 | </tr>
|
602 | EOF
|
603 | done
|
604 | }
|
605 |
|
606 | this-release-links() {
|
607 | echo '<div class="file-table">'
|
608 | echo '<table>'
|
609 | tarball-links-row-html "$OIL_VERSION"
|
610 | echo '</table>'
|
611 | echo '</div>'
|
612 | }
|
613 |
|
614 | # Turn HTML comment into a download link
|
615 | add-date-and-links() {
|
616 | local snippet
|
617 | snippet=$(this-release-links)
|
618 |
|
619 | awk -v date=$1 -v snippet="$snippet" '
|
620 | /<!-- REPLACE_WITH_DOWNLOAD_LINKS -->/ {
|
621 | print(snippet)
|
622 | next
|
623 | }
|
624 |
|
625 | /<!-- REPLACE_WITH_DATE -->/ {
|
626 | print(date)
|
627 | next
|
628 | }
|
629 |
|
630 | # Everything else
|
631 | { print }
|
632 | '
|
633 | }
|
634 |
|
635 | patch-release-pages() {
|
636 | local release_date
|
637 | release_date=$(cat _build/release-date.txt)
|
638 |
|
639 | local root=_release/VERSION
|
640 |
|
641 | add-date-and-links $release_date < _tmp/release-index.html > $root/index.html
|
642 | add-date-and-links $release_date < _tmp/release-quality.html > $root/quality.html
|
643 | }
|
644 |
|
645 | copy-release-pages() {
|
646 | ### For testing without releasing
|
647 |
|
648 | cat < _tmp/release-index.html > $root/index.html
|
649 | cat < _tmp/release-quality.html > $root/quality.html
|
650 | }
|
651 |
|
652 | run-for-release() {
|
653 | ### Build a tree. Requires _build/release-date.txt to exist
|
654 |
|
655 | local root=_release/VERSION
|
656 | mkdir -p $root/{doc,test,pub}
|
657 |
|
658 | tour
|
659 |
|
660 | # Metadata
|
661 | cp -v _build/release-date.txt oil-version.txt $root
|
662 |
|
663 | # Docs
|
664 | # Writes _release/VERSION and _tmp/release-index.html
|
665 | all-markdown
|
666 | all-ref
|
667 | all-redirects # backward compat
|
668 |
|
669 | fmt-check # Needs to run *after* we build the HTML
|
670 |
|
671 | patch-release-pages
|
672 |
|
673 | write-metrics
|
674 |
|
675 | # Problem: You can't preview it without .wwz!
|
676 | # Maybe have local redirects VERSION/test/wild/ to
|
677 | #
|
678 | # Instead of linking, I should compress them all here.
|
679 |
|
680 | copy-web
|
681 |
|
682 | if command -v tree >/dev/null; then
|
683 | tree $root
|
684 | else
|
685 | find $root
|
686 | fi
|
687 | }
|
688 |
|
689 | soil-run() {
|
690 | build/stamp.sh write-release-date
|
691 |
|
692 | run-for-release
|
693 | }
|
694 |
|
695 | "$@"
|
696 |
|