OILS / _gen / bin / text_files.cc View on Github | oils.pub

2269 lines, 167 significant
1
2#include "cpp/embedded_file.h"
3
4namespace embedded_file {
5GLOBAL_STR(gStr0, R"zZXx(Errors
6
7 [UTF8] err-utf8-encode err-utf8-decode
8 [J8 String] err-j8-str-encode err-j8-str-decode
9 [J8 Lines] err-j8-lines-encode err-j8-lines-decode
10 [JSON] err-json-encode err-json-decode
11 [JSON8] err-json8-encode err-json8-decode
12)zZXx");
13
14GLOBAL_STR(gStr1, R"zZXx(Front End
15
16 [Lexing] ascii-whitespace [ \t\r\n]
17 ascii-control-chars
18)zZXx");
19
20GLOBAL_STR(gStr2, R"zZXx(J8 Notation
21
22 [J8 Strings] json-string "hi"
23 json-escape \" \\ \u1234
24 surrogate-pair \ud83e\udd26
25 j8-escape \' \u{1f926} \yff
26 u-prefix u'hi'
27 b-prefix b'hi'
28 no-prefix 'hi'
29 [J8 Lines] unquoted-line
30 [JSON8] json8-num json8-str
31 X json8-list X json8-dict
32 json8-comment
33 [TSV8] column-attrs column-types
34)zZXx");
35
36GLOBAL_STR(gStr3, R"zZXx(Usage: help TOPIC?
37
38Examples:
39
40 help # this help
41 help echo # help on the 'echo' builtin
42 help command-sub # help on command sub $(date)
43
44 help oils-usage # identical to oils-for-unix --help
45 help osh-usage # osh --help
46 help ysh-usage # ysh --help
47)zZXx");
48
49GLOBAL_STR(gStr4, R"zZXx(bin/oils-for-unix is an executable that contains OSH, YSH, and more.
50
51Usage:
52 oils-for-unix MAIN_NAME ARG*
53 MAIN_NAME ARG*
54
55It behaves like busybox. The command name can be passed as the first argument:
56
57 oils-for-unix ysh -c 'echo hi'
58
59More commonly, it's invoked through a symlink like 'ysh', which causes it to
60behave like that command:
61
62 ysh -c 'echo hi'
63)zZXx");
64
65GLOBAL_STR(gStr5, R"zZXx(Builtin Commands
66
67 [I/O] read echo printf
68 readarray mapfile
69 [Run Code] source . cmd/eval trap
70 [Set Options] set shopt
71 [Working Dir] cd chdir pwd
72 pushd popd dirs
73 [Completion] complete compgen compopt compadjust compexport
74 [Shell Process] exec X logout
75 umask ulimit times
76 [Child Process] jobs wait
77 fg X bg X kill X disown
78 [External] test [ getopts
79 [Conditional] cmd/true cmd/false colon :
80 [Introspection] help hash cmd/type X caller
81 [Word Lookup] command builtin
82 [Interactive] alias unalias history fc X bind
83X [Unsupported] enable
84)zZXx");
85
86GLOBAL_STR(gStr6, R"zZXx(The reference is divided in to "chapters", each of which has its own table of
87contents. Type:
88
89 help osh-$CHAPTER
90
91Where $CHAPTER is one of:
92
93 type-method
94 builtin-cmd
95 stdlib
96 front-end
97 cmd-lang
98 osh-assign
99 word-lang
100 mini-lang
101 option
102 special-var
103 plugin
104
105Example:
106
107 help osh-word-lang
108)zZXx");
109
110GLOBAL_STR(gStr7, R"zZXx(Command Language
111
112 [Commands] simple-command command-lookup-order
113 prefix-binding semicolon ;
114 [Conditional] case if dbracket [[
115 bang ! and && or ||
116 [Iteration] while until for for-expr-sh ((
117 [Control Flow] break continue return exit
118 [Grouping] sh-func sh-block { subshell (
119 [Concurrency] pipe | X pipe-amp |& ampersand &
120 [Redirects] redir-file > >> >| < <> not impl: &>
121 redir-desc >& <&
122 here-doc << <<-
123 here-str <<<
124 [Other Command] dparen (( time X coproc X select
125)zZXx");
126
127GLOBAL_STR(gStr8, R"zZXx(Front End
128
129 [Usage] oils-usage osh-usage shell-flags
130 config startup line-editing
131 exit-codes
132 [Lexing] comment # line-continuation \ ascii-whitespace [ \t\r\n]
133)zZXx");
134
135GLOBAL_STR(gStr9, R"zZXx(Other Mini Languages
136
137 [Arithmetic] arith-context Where legacy arithmetic is allowed
138 sh-numbers 0xFF 0755 etc.
139 sh-arith 1 + 2*3 a *= 2
140 sh-logical !a && b
141 sh-bitwise ~a ^ b
142 [Boolean] bool-expr [[ ! $x && $y || $z ]]
143 test ! $x -a $y -o $z
144 bool-infix $a -nt $b $x == $y
145 bool-path -d /etc
146 bool-str -n foo -z ''
147 bool-other -o errexit -v name[index]
148 [Patterns] glob-pat *.py
149 extglob ,(*.py|*.sh)
150 regex [[ foo =~ [a-z]+ ]]
151 [Other Sublang] braces {alice,bob}@example.com
152 histsub !$ !! !n
153 char-escapes \t \c \x00 \u03bc
154)zZXx");
155
156GLOBAL_STR(gStr10, R"zZXx(Global Shell Options
157
158 [Errors] nounset -u errexit -e inherit_errexit pipefail
159 [Globbing] noglob -f nullglob failglob X dotglob
160 dashglob (true)
161 [Other Option] noclobber -C
162 [Debugging] errtrace -E extdebug X verbose xtrace -x
163 [Interactive] emacs vi
164 [Compat] eval_unsafe_arith ignore_flags_not_impl
165 ignore_shopt_not_impl
166 [Optimize] rewrite_extern ysh_rewrite_extern
167)zZXx");
168
169GLOBAL_STR(gStr11, R"zZXx(Assignments and Expressions
170
171 [Literals] sh-init-list a=(v1 [i]=v2 [k]+=v3) a+=(v1 [i]=v2 [k]+=v3)
172 sh-array array[123]=v "${array[@]}"
173 sh-assoc assoc['k']=v "${assoc[@]}" "${!assoc[@]}"
174 [Operators] sh-assign str='xyz'
175 sh-append str+='abc'
176 [Builtins] local readonly export unset shift
177 declare typeset X let
178)zZXx");
179
180GLOBAL_STR(gStr12, R"zZXx(Plugins and Hooks
181
182 [Signals] SIGTERM SIGINT SIGQUIT
183 SIGTTIN SIGTTOU SIGWINCH
184 [Traps] DEBUG ERR EXIT X RETURN
185 [Words] PS1 X PS2 X PS3 PS4
186 [Completion] complete
187 [Other Plugin] PROMPT_COMMAND X command_not_found
188)zZXx");
189
190GLOBAL_STR(gStr13, R"zZXx(Special Variables
191
192 [Oils VM] OILS_VERSION LIB_OSH
193 [libc locale] osh-locale
194 [Interactive] OILS_COMP_UI HISTFILE
195 [POSIX Special] $@ $* $# $? $- $$ $! $0 $9
196 [Shell Vars] IFS X LANG X GLOBIGNORE
197 [Shell Options] SHELLOPTS X BASHOPTS
198 [Other Env] HOME PATH
199 [Other Special] BASH_REMATCH @PIPESTATUS
200 [Platform] HOSTNAME OSTYPE
201 [Call Stack] @BASH_SOURCE @FUNCNAME @BASH_LINENO
202 X @BASH_ARGV X @BASH_ARGC
203 [Tracing] LINENO
204 [Process State] UID EUID PPID X BASHPID
205X [Process Stack] BASH_SUBSHELL SHLVL
206X [Shell State] BASH_CMDS @DIRSTACK
207 [Completion] @COMP_WORDS COMP_CWORD COMP_LINE COMP_POINT
208 COMP_WORDBREAKS @COMPREPLY X COMP_KEY
209 X COMP_TYPE COMP_ARGV
210 [cd] PWD OLDPWD X CDPATH
211 [getopts] OPTIND OPTARG X OPTERR
212 [read] REPLY
213 [Functions] X RANDOM SECONDS
214)zZXx");
215
216GLOBAL_STR(gStr14, R"zZXx(Standard Library
217
218 [two] log die
219 [no-quotes] nq-assert nq-run
220 nq-capture nq-capture-2
221 nq-redir nq-redir-2
222 [bash-strict]
223 [task-five]
224)zZXx");
225
226GLOBAL_STR(gStr15, R"zZXx(OSH Types
227
228 [OSH] BashArray BashAssoc
229)zZXx");
230
231GLOBAL_STR(gStr16, R"zZXx(bin/osh is compatible with POSIX shell, bash, and other shells.
232
233Usage:
234 osh FLAG* SCRIPT ARG*
235 osh FLAG* -c COMMAND ARG*
236 osh FLAG*
237
238Examples:
239 osh -c 'echo hi'
240 osh myscript.sh
241 echo 'echo hi' | osh
242)zZXx");
243
244GLOBAL_STR(gStr17, R"zZXx(Word Language
245
246 [Glob] osh-glob *.py
247 [Joining] osh-word-join 'single'\'$myvar"double $x"
248 [Quotes] osh-string 'abc' $'line\n' "$var"
249 [Substitutions] command-sub $(date) `date`
250 var-sub ${var} $0 $9
251 arith-sub $((1 + 2))
252 tilde-sub ~/src
253 proc-sub diff <(sort L.txt) <(sort R.txt)
254 [Var Ops] op-bracket ${a[i+1]}, ${a[*]}
255 op-indirect ${!x}
256 op-test ${x:-default}
257 op-strip ${x%%suffix} etc.
258 op-patsub ${x//y/z}
259 op-slice ${a[@]:0:1}
260 op-format ${x@P} ${x@Q} etc.
261)zZXx");
262
263GLOBAL_STR(gStr18, R"zZXx(osh and ysh accept standard POSIX shell flags, like:
264
265 bin/osh -o errexit -c 'false'
266 bin/ysh -n myfile.ysh
267 bin/ysh +o errexit -c 'false; echo ok'
268
269They also accept these flags:
270
271 --eval FILE
272 Evaluate the given file, similar to the 'source' builtin. Specify it
273 multiple times to run multiple files.
274
275 If the errexit option is on (e.g. in YSH), then the shell stops when $?
276 is non-zero after evaluating a file. The $0 value is set to FILE.
277
278 --eval-pure FILE
279 Like --eval, but disallow I/O (known as "pure mode").
280
281 --location-str
282 Use this string to display error messages.
283 See 'help sourceCode' for an example.
284
285 --location-start-line
286 Use this line number offset to display error messages.
287
288 --tool Run a tool instead of the shell (cat-em|syntax-tree)
289 -n Parse the program but don't execute it. Print the AST.
290 --ast-format FMT The format for the AST (text|text-abbrev)
291
292Examples:
293
294 ysh --eval one.ysh --eval two.ysh -c 'echo hi' # Run 2 files first
295 osh -n -c 'hello' # pretty-print the AST
296 ysh --ast-format text -n -c 'hello' # in unabridged format
297)zZXx");
298
299GLOBAL_STR(gStr19, R"zZXx(Builtin Commands
300
301 [Memory] cmd/append Add elements to end of array
302 pp value proc test_
303 asdl_ cell_ X gc-stats_
304 [Handle Errors] error error 'failed' (status=2)
305 try Run with errexit, set _error
306 failed Test if _error.code !== 0
307 boolstatus Enforce 0 or 1 exit status
308 assert assert [42 === f(x)]
309 [Shell State] ysh-cd ysh-shopt compatible, and takes a block
310 shvar Temporary modify global settings
311 ctx Share and update a temporary "context"
312 push-registers Save registers like $?, PIPESTATUS
313 [Word Lookup] invoke Run a command, and control name lookup
314 runproc Run a proc; use as main entry point
315 X extern Run an external command, with an ENV
316 [Modules]
317 source-guard guard against duplicate 'source'
318 is-main false when sourcing a file
319 use create a module Obj from a source file
320 [I/O] ysh-read flags --all, -0
321 ysh-echo no -e -n with simple_echo
322 ysh-test --file --true etc.
323 ysh-wait wait --all --verbose
324 write Like echo, with --, --sep, --end
325 fork forkwait Replace & and (), and takes a block
326 redir Run a block, with redirects
327 [Run Code] ysh-trap
328 [Private] cat rm POSIX-compatible
329 sleep
330 [Hay Config] hay haynode For DSLs and config files
331 [Completion] compadjust compexport
332 [Data Formats] json read write
333 json8 read write
334)zZXx");
335
336GLOBAL_STR(gStr20, R"zZXx(Builtin Functions
337
338 [Values] len() func/type()
339 [Conversions] bool() int() float()
340 str() list() dict()
341 X runes() X encodeRunes()
342 X bytes() X encodeBytes()
343 [Str] strcmp()
344 [List] join()
345 [Dict] keys() values() get()
346 [Float] floatsEqual() X isinf() X isnan()
347 [Obj] first() rest() get()
348 [Word] maybe() shSplit()
349 [Serialize] toJson() fromJson()
350 toJson8() fromJson8()
351 X toJ8Line() X fromJ8Line()
352 [Pattern] _group() _start() _end()
353 [Reflection] func/eval() func/evalExpr()
354 [Introspect] shvarGet() getVar() setVar()
355 parseCommand() X parseExpr() X bindFrame()
356 [Hay Config] parseHay() evalHay()
357X [Hashing] sha1dc() sha256()
358)zZXx");
359
360GLOBAL_STR(gStr21, R"zZXx(The reference is divided in to "chapters", each of which has its own table of
361contents. Type:
362
363 help ysh-$CHAPTER
364
365Where $CHAPTER is one of:
366
367 type-method
368 builtin-func
369 builtin-cmd
370 stdlib
371 front-end
372 cmd-lang
373 ysh-cmd
374 expr-lang
375 word-lang
376 option
377 special-var
378 plugin
379
380Example:
381
382 help ysh-expr-lang
383 help ysh-ysh-cmd # may change
384)zZXx");
385
386GLOBAL_STR(gStr22, R"zZXx(Command Language
387
388 [Commands] simple-command
389 command-lookup-order
390 ysh-prefix-binding
391 semicolon ;
392 [Redirects] ysh-here-str read <<< '''
393 [YSH Simple] typed-arg json write (x)
394 lazy-expr-arg assert [42 === x]
395 block-arg cd /tmp { echo $PWD }; cd /tmp (; ; blockexpr)
396 [YSH Cond] ysh-case case (x) { *.py { echo 'python' } }
397 ysh-if if (x > 0) { echo }
398 [YSH Iter] ysh-for for i, item in (mylist) { echo }
399 ysh-while while (x > 0) { echo }
400)zZXx");
401
402GLOBAL_STR(gStr23, R"zZXx(Expression Language and Assignments
403
404 [Assignment] assign =
405 aug-assign += -= *= /= **= //= %=
406 &= |= ^= <<= >>=
407 [Literals] atom-literal null true false
408 int-literal 42 65_536 0xFF 0o755 0b10
409 float-literal 3.14 1.5e-10
410 X num-suffix 42 K Ki M Mi G Gi T Ti / ms us
411 char-literal \\ \t \" \y00 \u{3bc}
412 ysh-string "x is $x" $"x is $x" r'[a-z]\n'
413 u'line\n' b'byte \yff'
414 triple-quoted """ $""" r''' u''' b'''
415 list-literal ['one', 'two', 3] :| unquoted words |
416 dict-literal {name: 'bob'} {a, b}
417 range 1 ..< n 1 ..= n
418 block-expr ^(echo $PWD)
419 expr-literal ^[1 + 2*3]
420 str-template ^"$a and $b" for Str.replace()
421 expr-sub $[myobj]
422 expr-splice @[myobj]
423 [Operators] op-precedence Like Python
424 concat s1 ++ s2, L1 ++ L2, dict1 ++ dict2
425 ysh-equals === !== ~== is, is not
426 ysh-in in, not in
427 ysh-compare < <= > >= (numbers only)
428 ysh-logical not and or
429 ysh-arith + - * / // % **
430 ysh-unary + -
431 ysh-bitwise ~ & | ^ << >>
432 ysh-ternary '+' if x >= 0 else '-'
433 ysh-index s[0] mylist[3] mydict['key']
434 ysh-attr mydict.key mystr.startsWith('x')
435 ysh-slice a[1:-1] s[1:-1]
436 ysh-func-call f(x, y, ...pos; n=1, ...named)
437 thin-arrow mylist->pop()
438 fat-arrow mylist => join() => upper()
439 match-ops ~ !~ ~~ !~~
440 [Eggex] re-literal / d+ ; re-flags ; ERE /
441 re-primitive %zero 'sq'
442 class-literal [c a-z 'abc' @str_var \\ \y01 \u{3bc}]
443 re-chars \y01 \u{3bc}
444 named-class dot digit space word d s w
445 re-repeat d? d* d+ d{3} d{2,4}
446 re-compound seq1 seq2 alt1|alt2 (expr1 expr2)
447 re-capture <capture d+ as name: int>
448 re-splice Subpattern @subpattern
449 re-flags reg_icase reg_newline
450 X re-multiline ///
451)zZXx");
452
453GLOBAL_STR(gStr24, R"zZXx(Front End
454
455 [Usage] oils-usage ysh-usage shell-flags
456 config startup line-editing
457 exit-codes
458 [Lexing] comment # line-continuation \ ascii-whitespace [ \t\r\n]
459 [Lexing] doc-comment ### multiline-command ...
460 [Tools] cat-em syntax-tree
461)zZXx");
462
463GLOBAL_STR(gStr25, R"zZXx(Other Mini Languages
464
465 [Patterns] glob-pat *.py
466 [Other Sublang] braces {alice,bob}@example.com
467)zZXx");
468
469GLOBAL_STR(gStr26, R"zZXx(Global Shell Options
470
471 [Optimize] rewrite_extern ysh_rewrite_extern
472 [Groups] strict:all ysh:upgrade ysh:all
473 [YSH Details] opts-redefine opts-internal
474)zZXx");
475
476GLOBAL_STR(gStr27, R"zZXx(Plugins and Hooks
477
478 [YSH] renderPrompt()
479)zZXx");
480
481GLOBAL_STR(gStr28, R"zZXx(Special Variables
482
483 [YSH Vars] ARGV ENV
484 __defaults__ __builtins__ X __sh_function__
485 _this_dir
486 [YSH Status] _error
487 _pipeline_status _process_sub_status
488 [YSH Tracing] SHX_indent SHX_punct SHX_pid_str
489 [YSH read] _reply
490 [Oils VM] OILS_VERSION LIB_YSH
491 OILS_GC_THRESHOLD OILS_GC_ON_EXIT
492 OILS_GC_STATS OILS_GC_STATS_FD
493 OILS_LOCALE_OK
494 [libc locale] ysh-locale
495 [Interactive] OILS_COMP_UI YSH_HISTFILE
496 [Float] NAN INFINITY
497 [Module] __provide__
498 [Other Env] HOME PATH
499)zZXx");
500
501GLOBAL_STR(gStr29, R"zZXx(Standard Library
502
503 [math] abs() max() min() X round()
504 sum()
505 [list] all() any() repeat()
506 [yblocks] yb-capture yb-capture-2
507 [args] parser flag arg rest
508 parseArgs()
509 [binascii] X toBase16() X fromBase16()
510 X toBase64() X fromBase64()
511)zZXx");
512
513GLOBAL_STR(gStr30, R"zZXx(Types and Methods
514
515 [Atoms] Null null
516 Bool expr/true expr/false
517 [Numbers] Int
518 Float
519 Range
520 [String] Str find() findLast()
521 contains() replace()
522 trim() trimStart() trimEnd()
523 startsWith() endsWith()
524 upper() lower()
525 search() leftMatch()
526 split() lines()
527 [Patterns] Eggex
528 Match group() start() end()
529 X groups() X groupDict()
530 [Containers] List List/append() pop() extend()
531 indexOf() lastIndexOf() X includes()
532 insert() remove()
533 reverse() List/clear()
534 Dict erase() Dict/clear() append()
535 update() inc()
536 Place setValue()
537 [Code Types] Func
538 BuiltinFunc
539 BoundFunc
540 Proc docComment()
541 BuiltinProc
542 [Objects] Obj __invoke__ new
543 X __call__ __index__ X __str__
544 [Reflection] Command sourceCode()
545 Expr
546 Frame
547 DebugFrame toString()
548 io stdin io/eval() io/evalExpr()
549 captureStdout() captureAll() promptVal()
550 X time() X strftime() glob()
551 libcGlob()
552 vm getFrame() getDebugStack() id()
553)zZXx");
554
555GLOBAL_STR(gStr31, R"zZXx(bin/ysh is the shell with data tYpes, influenced by pYthon, JavaScript, ...
556
557Usage:
558 ysh FLAG* SCRIPT ARG*
559 ysh FLAG* -c COMMAND ARG*
560 ysh FLAG*
561
562Examples:
563 ysh -c 'echo hi'
564 ysh myscript.ysh
565 echo 'echo hi' | ysh
566
567Note that bin/ysh is the same as bin/osh with the ysh:all option group set:
568 osh -o ysh:all -c 'echo hi' # Same as YSH
569)zZXx");
570
571GLOBAL_STR(gStr32, R"zZXx(Word Language
572
573 [Glob] ysh-glob *.py
574 [Joining] ysh-word-join --flag='value' PATH="value"
575 [Quotes] ysh-string "x is $x" $"x is $x" r'[a-z]\n'
576 u'line\n' b'byte \yff'
577 triple-quoted """ $""" r''' u''' b'''
578 X tagged-str "<span id=$x>"html
579 [Substitutions] expr-sub echo $[42 + a[i]]
580 expr-splice echo @[split(x)]
581 var-splice @myarray @ARGV
582 command-sub $(date)
583 command-splice @(cat my-j8-lines.txt)
584 [Formatting] X ysh-printf ${x %.3f}
585 X ysh-format ${x|html}
586)zZXx");
587
588GLOBAL_STR(gStr33, R"zZXx(YSH Command Language Keywords
589
590 [Assignment] const var Declare variables
591 setvar setvar a[i] = 42
592 setglobal setglobal d.key = 'foo'
593 [Expression] equal = = 1 + 2*3
594 call call mylist->append(42)
595 [Definitions] proc proc p (s, ...rest) {
596 typed proc p (; typed, ...rest; n=0; b) {
597 func func f(x; opt1, opt2) { return (x + 1) }
598 ysh-return return (myexpr)
599)zZXx");
600
601GLOBAL_STR(gStr34, R"zZXx(# Can we define methods in pure YSH?
602#
603# (mylist->find(42) !== -1)
604#
605# instead of
606#
607# ('42' in mylist)
608#
609# Because 'in' is for Dict
610
611func find (haystack List, needle) {
612 for i, x in (haystack) {
613 if (x === needle) {
614 return (i)
615 }
616 }
617 return (-1)
618}
619)zZXx");
620
621GLOBAL_STR(gStr35, R"zZXx(# Bash strict mode, updated for 2024
622
623set -o nounset
624set -o pipefail
625set -o errexit
626shopt -s inherit_errexit
627shopt -s strict:all 2>/dev/null || true # dogfood for OSH
628
629)zZXx");
630
631GLOBAL_STR(gStr36, R"zZXx(# Library to turn a shell file into a "BYO test server"
632#
633# Usage:
634#
635# # from both bash and OSH
636# if test -z "$LIB_OSH"; then LIB_OSH=stdlib/osh; fi
637# source $LIB_OSH/byo-server-lib.sh
638#
639# The client creates a clean process state and directory state for each tests.
640#
641# (This file requires compgen -A, and maybe declare -f, so it's not POSIX
642# shell.)
643
644: ${LIB_OSH:-stdlib/osh}
645source $LIB_OSH/two.sh
646
647# List all functions defined in this file (and not in sourced files).
648_bash-print-funcs() {
649 ### Print shell functions in this file that don't start with _ (bash reflection)
650
651 local funcs
652 funcs=($(compgen -A function))
653
654 # extdebug makes `declare -F` print the file path, but, annoyingly, only
655 # if you pass the function names as arguments.
656 shopt -s extdebug
657
658 # bash format:
659 # func1 1 path1
660 # func2 2 path2 # where 2 is the linen umber
661
662 #declare -F "${funcs[@]}"
663
664 # TODO: do we need to normalize the LHS and RHS of $3 == path?
665 declare -F "${funcs[@]}" | awk -v "path=$0" '$3 == path { print $1 }'
666
667 shopt -u extdebug
668}
669
670_gawk-print-funcs() {
671 ### Print shell functions in this file that don't start with _ (awk parsing)
672
673 # Using gawk because it has match()
674 # - doesn't start with _
675
676 # space = / ' '* /
677 # shfunc = / %begin
678 # <capture !['_' ' '] ![' ']*>
679 # '()' space '{' space
680 # %end /
681 # docstring = / %begin
682 # space '###' ' '+
683 # <capture dot*>
684 # %end /
685 gawk '
686 match($0, /^([^_ ][^ ]*)\(\)[ ]*{[ ]*$/, m) {
687 #print NR " shfunc " m[1]
688 print m[1]
689 #print m[0]
690 }
691
692 match($0, /^[ ]*###[ ]+(.*)$/, m) {
693 print NR " docstring " m[1]
694 }
695' $0
696}
697
698_print-funcs() {
699 _bash-print-funcs
700 return
701
702 # TODO: make gawk work, with docstrings
703 if command -v gawk > /dev/null; then
704 _gawk-print-funcs
705 else
706 _bash-print-funcs
707 fi
708}
709
710
711byo-maybe-run() {
712 local command=${BYO_COMMAND:-}
713
714 case $command in
715 '')
716 # Do nothing if it's not specified
717 return
718 ;;
719
720 detect)
721 # all the commands supported, except 'detect'
722 echo list-tests
723 echo run-test
724
725 exit 66 # ASCII code for 'B' - what the protocol specifies
726 ;;
727
728 list-tests)
729 # TODO: use _bash-print-funcs? This fixes the transitive test problem,
730 # which happened in soil/web-remote-test.sh
731 # But it should work with OSH, not just bash! We need shopt -s extdebug
732 compgen -A function | grep '^test-'
733 exit 0
734 ;;
735
736 run-test)
737 local test_name=${BYO_ARG:-}
738 if test -z "$test_name"; then
739 die "BYO run-test: Expected BYO_ARG"
740 fi
741
742 # Avoid issues polluting recursive calls!
743 unset BYO_COMMAND BYO_ARG
744
745 # Shell convention: we name functions test-*
746 "$test_name"
747
748 # Only run if not set -e. Either way it's equivalent
749 exit $?
750 ;;
751
752 *)
753 die "Invalid BYO command '$command'"
754 ;;
755 esac
756
757 # Do nothing if BYO_COMMAND is not set.
758 # The program continues to its "main".
759}
760
761byo-must-run() {
762 local command=${BYO_COMMAND:-}
763 if test -z "$command"; then
764 die "Expected BYO_COMMAND= in environment"
765 fi
766
767 byo-maybe-run
768}
769)zZXx");
770
771GLOBAL_STR(gStr37, R"zZXx(#!/usr/bin/env bash
772#
773# Testing library for bash and OSH.
774#
775# Capture status/stdout/stderr, and nq-assert those values.
776
777: ${LIB_OSH=stdlib/osh}
778source $LIB_OSH/two.sh
779
780nq-assert() {
781 ### Assertion with same syntax as shell 'test'
782
783 if ! test "$@"; then
784 die "line ${BASH_LINENO[0]}: nq-assert $(printf '%q ' "$@") failed"
785 fi
786}
787
788# Problem: we want to capture status and stdout at the same time
789#
790# We use:
791#
792# __stdout=$(set -o errexit; "$@")
793# __status=$?
794#
795# However, we lose the trailing \n, since that's how command subs work.
796
797# Here is another possibility:
798#
799# shopt -s lastpipe # need this too
800# ( set -o errexit; "$@" ) | read -r -d __stdout
801# __status=${PIPESTATUS[0]}
802# shopt -u lastpipe
803#
804# But this feels complex for just the \n issue, which can be easily worked
805# around.
806
807nq-run() {
808 ### capture status only
809
810 local -n out_status=$1
811 shift
812
813 local __status
814
815 # Tricky: turn errexit off so we can capture it, but turn it on against
816 set +o errexit
817 ( set -o errexit; "$@" )
818 __status=$?
819 set -o errexit
820
821 out_status=$__status
822}
823
824nq-capture() {
825 ### capture status and stdout
826
827 local -n out_status=$1
828 local -n out_stdout=$2
829 shift 2
830
831 local __status
832 local __stdout
833
834 # Tricky: turn errexit off so we can capture it, but turn it on against
835 set +o errexit
836 __stdout=$(set -o errexit; "$@")
837 __status=$?
838 set -o errexit
839
840 out_status=$__status
841 out_stdout=$__stdout
842}
843
844nq-capture-2() {
845 ### capture status and stderr
846
847 # This is almost identical to the above
848
849 local -n out_status=$1
850 local -n out_stderr=$2
851 shift 2
852
853 local __status
854 local __stderr
855
856 # Tricky: turn errexit off so we can capture it, but turn it on against
857 set +o errexit
858 __stderr=$(set -o errexit; "$@" 2>&1)
859 __status=$?
860 set -o errexit
861
862 out_status=$__status
863 out_stderr=$__stderr
864}
865
866# 'byo test' can set this?
867: ${NQ_TEST_TEMP=/tmp}
868
869nq-redir() {
870 ### capture status and stdout
871
872 local -n out_status=$1
873 local -n out_stdout_file=$2
874 shift 2
875
876 local __status
877 local __stdout_file=$NQ_TEST_TEMP/nq-redir-$$.txt
878
879 # Tricky: turn errexit off so we can capture it, but turn it on against
880 set +o errexit
881 ( set -o errexit; "$@" ) > $__stdout_file
882 __status=$?
883 set -o errexit
884
885 out_status=$__status
886 out_stdout_file=$__stdout_file
887}
888
889nq-redir-2() {
890 ### capture status and stdout
891
892 local -n out_status=$1
893 local -n out_stderr_file=$2
894 shift 2
895
896 local __status
897 local __stderr_file=$NQ_TEST_TEMP/nq-redir-$$.txt
898
899 # Tricky: turn errexit off so we can capture it, but turn it on against
900 set +o errexit
901 ( set -o errexit; "$@" ) 2> $__stderr_file
902 __status=$?
903 set -o errexit
904
905 out_status=$__status
906 out_stderr_file=$__stderr_file
907}
908)zZXx");
909
910GLOBAL_STR(gStr38, R"zZXx(#!/usr/bin/env bash
911#
912# Common shell functions for task scripts.
913#
914# Usage:
915# source $LIB_OSH/task-five.sh
916#
917# test-foo() { # define task functions
918# echo foo
919# }
920# task-five "$@"
921
922# Definition of a "task"
923#
924# - File invokes task-five "$@"
925# - or maybe you can look at its source
926# - It's a shell function
927# - Has ### docstring
928# - Doesn't start with _
929
930: ${LIB_OSH=stdlib/osh}
931source $LIB_OSH/byo-server.sh
932
933_show-help() {
934 # TODO:
935 # - Use awk to find comments at the top of the file?
936 # - Use OSH to extract docstrings
937 # - BYO_COMMAND=list-tasks will reuse that logic? It only applies to the
938 # current file, not anything in a different file?
939
940 echo "Usage: $0 TASK_NAME ARGS..."
941 echo
942 echo "To complete tasks, run:"
943 echo " source devtools/completion.bash"
944 echo
945 echo "Tasks:"
946
947 if command -v column >/dev/null; then
948 _print-funcs | column
949 else
950 _print-funcs
951 fi
952}
953
954task-five() {
955 # Respond to BYO_COMMAND=list-tasks, etc. All task files need this.
956 byo-maybe-run
957
958 case ${1:-} in
959 ''|--help|-h)
960 _show-help
961 exit 0
962 ;;
963 esac
964
965 if ! declare -f "$1" >/dev/null; then
966 echo "$0: '$1' isn't an action in this task file. Try '$0 --help'"
967 exit 1
968 fi
969
970 "$@"
971}
972)zZXx");
973
974GLOBAL_STR(gStr39, R"zZXx(# Two functions I actually use, all the time.
975#
976# To keep depenedencies small, this library will NEVER grow other functions
977# (and is named to imply that.)
978#
979# Usage:
980# source --builtin two.sh
981#
982# Examples:
983# log 'hi'
984# die 'expected a number'
985
986if command -v source-guard >/dev/null; then # include guard for YSH
987 source-guard two || return 0
988fi
989
990log() {
991 ### Write a message to stderr.
992 echo "$@" >&2
993}
994
995die() {
996 ### Write an error message with the script name, and exit with status 1.
997 log "$0: fatal: $@"
998 exit 1
999}
1000
1001)zZXx");
1002
1003GLOBAL_STR(gStr40, R"zZXx(# These were helpful while implementing args.ysh
1004# Maybe we will want to export them in a prelude so that others can use them too?
1005#
1006# Prior art: Rust has `todo!()` which is quite nice. Other languages allow
1007# users to `raise NotImplmentedError()`.
1008
1009# Andy comments:
1010# - 'pass' can be : or true in shell. It's a little obscure / confusing, but
1011# there is an argument for minimalism. Although I prefer words like 'true',
1012# and that already means something.
1013# - UPDATE: we once took 'pass' as a keyword, but users complained because
1014# there is a command 'pass'. So we probably can't have this by default.
1015# Need to discuss source --builtin.
1016
1017# - todo could be more static? Rust presumably does it at compile time
1018
1019proc todo () {
1020 ## Raises a not implemented error when run.
1021 error ("TODO: not implemented") # TODO: is error code 1 ok?
1022}
1023
1024proc pass () {
1025 ## Use when you want to temporarily leave a block empty.
1026 _ null
1027}
1028)zZXx");
1029
1030GLOBAL_STR(gStr41, R"zZXx(# args.ysh
1031#
1032# Usage:
1033# source --builtin args.sh
1034
1035const __provide__ = :| parser parseArgs |
1036
1037#
1038#
1039# parser (&spec) {
1040# flag -v --verbose (help="Verbosely") # default is Bool, false
1041#
1042# flag -P --max-procs (Int, default=-1, doc='''
1043# Run at most P processes at a time
1044# ''')
1045#
1046# flag -i --invert (Bool, default=true, doc='''
1047# Long multiline
1048# Description
1049# ''')
1050#
1051# arg src (help='Source')
1052# arg dest (help='Dest')
1053# arg times (help='Foo')
1054#
1055# rest files
1056# }
1057#
1058# var args = parseArgs(spec, ARGV)
1059#
1060# echo "Verbose $[args.verbose]"
1061
1062# TODO: See list
1063# - flag builtin:
1064# - handle only long flag or only short flag
1065# - flag aliases
1066# - assert that default value has the declared type
1067
1068proc parser (; place ; ; block_def) {
1069 ## Create an args spec which can be passed to parseArgs.
1070 ##
1071 ## Example:
1072 ##
1073 ## # NOTE: &spec will create a variable named spec
1074 ## parser (&spec) {
1075 ## flag -v --verbose (Bool)
1076 ## }
1077 ##
1078 ## var args = parseArgs(spec, ARGV)
1079
1080 var p = {flags: [], args: []}
1081 ctx push (p) {
1082 call io->eval(block_def, vars={flag, arg, rest})
1083 }
1084
1085 # Validate that p.rest = [name] or null and reduce p.rest into name or null.
1086 if ('rest' in p) {
1087 if (len(p.rest) > 1) {
1088 error '`rest` was called more than once' (code=3)
1089 } else {
1090 setvar p.rest = p.rest[0]
1091 }
1092 } else {
1093 setvar p.rest = null
1094 }
1095
1096 var names = {}
1097 for items in ([p.flags, p.args]) {
1098 for x in (items) {
1099 if (x.name in names) {
1100 error "Duplicate flag/arg name $[x.name] in spec" (code=3)
1101 }
1102
1103 setvar names[x.name] = null
1104 }
1105 }
1106
1107 # TODO: what about `flag --name` and then `arg name`?
1108
1109 call place->setValue(p)
1110}
1111
1112const kValidTypes = [Bool, Float, List[Float], Int, List[Int], Str, List[Str]]
1113const kValidTypeNames = []
1114for vt in (kValidTypes) {
1115 var name = vt.name if ('name' in propView(vt)) else vt.unique_id
1116 call kValidTypeNames->append(name)
1117}
1118
1119func isValidType (type) {
1120 for valid in (kValidTypes) {
1121 if (type is valid) {
1122 return (true)
1123 }
1124 }
1125 return (false)
1126}
1127
1128proc flag (short, long ; type=Bool ; default=null, help=null) {
1129 ## Declare a flag within an `arg-parse`.
1130 ##
1131 ## Examples:
1132 ##
1133 ## arg-parse (&spec) {
1134 ## flag -v --verbose
1135 ## flag -n --count (Int, default=1)
1136 ## flag -p --percent (Float, default=0.0)
1137 ## flag -f --file (Str, help="File to process")
1138 ## flag -e --exclude (List[Str], help="File to exclude")
1139 ## }
1140
1141 if (type !== null and not isValidType(type)) {
1142 var type_names = ([null] ++ kValidTypeNames) => join(', ')
1143 error "Expected flag type to be one of: $type_names" (code=2)
1144 }
1145
1146 # Bool has a default of false, not null
1147 if (type is Bool and default === null) {
1148 setvar default = false
1149 }
1150
1151 var name = long => trimStart('--')
1152
1153 ctx emit flags ({short, long, name, type, default, help})
1154}
1155
1156proc arg (name ; ; help=null) {
1157 ## Declare a positional argument within an `arg-parse`.
1158 ##
1159 ## Examples:
1160 ##
1161 ## arg-parse (&spec) {
1162 ## arg name
1163 ## arg config (help="config file path")
1164 ## }
1165
1166 ctx emit args ({name, help})
1167}
1168
1169proc rest (name) {
1170 ## Take the remaining positional arguments within an `arg-parse`.
1171 ##
1172 ## Examples:
1173 ##
1174 ## arg-parse (&grepSpec) {
1175 ## arg query
1176 ## rest files
1177 ## }
1178
1179 # We emit instead of set to detect multiple invocations of "rest"
1180 ctx emit rest (name)
1181}
1182
1183func parseArgs(spec, argv) {
1184 ## Given a spec created by `parser`. Parse an array of strings `argv` per
1185 ## that spec.
1186 ##
1187 ## See `parser` for examples of use.
1188
1189 var i = 0
1190 var positionalPos = 0
1191 var argc = len(argv)
1192 var args = {}
1193 var rest = []
1194
1195 var value
1196 var found
1197 var escape_remaining = false
1198 while (i < argc) {
1199 var arg = argv[i]
1200
1201 if (escape_remaining) {
1202 call rest->append(arg)
1203 } elif (arg === '--') {
1204 if (not spec.rest) {
1205 error "Unexpected '--' argument - extraneous positional arguments are prohibited" (code=2)
1206 }
1207
1208 setvar escape_remaining = true
1209 } elif (arg.startsWith('-')) {
1210 setvar found = false
1211
1212 for flag in (spec.flags) {
1213 if ( (flag.short and flag.short === arg) or
1214 (flag.long and flag.long === arg) ) {
1215 if (flag.type === null or flag.type is Bool) {
1216 setvar value = true
1217 } elif (flag.type is Int) {
1218 setvar i += 1
1219 if (i >= len(argv)) {
1220 error "Expected Int after '$arg'" (code=2)
1221 }
1222
1223 try { setvar value = int(argv[i]) }
1224 if failed {
1225 error "Expected Int after '$arg', got '$[argv[i]]'" (code=2)
1226 }
1227 } elif (flag.type is List[Int]) {
1228 setvar i += 1
1229 if (i >= len(argv)) {
1230 error "Expected Int after '$arg'" (code=2)
1231 }
1232
1233 setvar value = get(args, flag.name, [])
1234 try { call value->append(int(argv[i])) }
1235 if failed {
1236 error "Expected Int after '$arg', got '$[argv[i]]'" (code=2)
1237 }
1238 } elif (flag.type is Float) {
1239 setvar i += 1
1240 if (i >= len(argv)) {
1241 error "Expected Float after '$arg'" (code=2)
1242 }
1243
1244 try { setvar value = float(argv[i]) }
1245 if failed {
1246 error "Expected Float after '$arg', got '$[argv[i]]'" (code=2)
1247 }
1248 } elif (flag.type is List[Float]) {
1249 setvar i += 1
1250 if (i >= len(argv)) {
1251 error "Expected Float after '$arg'" (code=2)
1252 }
1253
1254 setvar value = get(args, flag.name, [])
1255 try { call value->append(float(argv[i])) }
1256 if failed {
1257 error "Expected Float after '$arg', got '$[argv[i]]'" (code=2)
1258 }
1259 } elif (flag.type is Str) {
1260 setvar i += 1
1261 if (i >= len(argv)) {
1262 error "Expected Str after '$arg'" (code=2)
1263 }
1264
1265 setvar value = argv[i]
1266 } elif (flag.type is List[Str]) {
1267 setvar i += 1
1268 if (i >= len(argv)) {
1269 error "Expected Str after '$arg'" (code=2)
1270 }
1271
1272 setvar value = get(args, flag.name, [])
1273 call value->append(argv[i])
1274 }
1275
1276 setvar args[flag.name] = value
1277 setvar found = true
1278 break
1279 }
1280 }
1281
1282 if (not found) {
1283 error "Unknown flag '$arg'" (code=2)
1284 }
1285 } elif (positionalPos >= len(spec.args)) {
1286 if (not spec.rest) {
1287 error "Too many arguments, unexpected '$arg'" (code=2)
1288 }
1289
1290 call rest->append(arg)
1291 } else {
1292 var pos = spec.args[positionalPos]
1293 setvar positionalPos += 1
1294 setvar value = arg
1295 setvar args[pos.name] = value
1296 }
1297
1298 setvar i += 1
1299 }
1300
1301 if (spec.rest) {
1302 setvar args[spec.rest] = rest
1303 }
1304
1305 # Set defaults for flags
1306 for flag in (spec.flags) {
1307 if (flag.name not in args) {
1308 setvar args[flag.name] = flag.default
1309 }
1310 }
1311
1312 # Raise error on missing args
1313 for arg in (spec.args) {
1314 if (arg.name not in args) {
1315 error "Usage Error: Missing required argument $[arg.name]" (code=2)
1316 }
1317 }
1318
1319 return (args)
1320}
1321)zZXx");
1322
1323GLOBAL_STR(gStr42, R"zZXx(const __provide__ = :| Dict |
1324
1325proc Dict ( ; out; ; block) {
1326 var d = io->evalToDict(block)
1327 call out->setValue(d)
1328}
1329
1330)zZXx");
1331
1332GLOBAL_STR(gStr43, R"zZXx(const __provide__ = :| any all repeat |
1333
1334func any(list) {
1335 ### Returns true if any value in the list is truthy.
1336 # Empty list: returns false
1337
1338 for item in (list) {
1339 if (item) {
1340 return (true)
1341 }
1342 }
1343 return (false)
1344}
1345
1346func all(list) {
1347 ### Returns true if all values in the list are truthy.
1348 # Empty list: returns true
1349
1350 for item in (list) {
1351 if (not item) {
1352 return (false)
1353 }
1354 }
1355 return (true)
1356}
1357
1358func repeat(x, n) {
1359 ### Repeats a given Str or List, returning another Str or List
1360
1361 # Like Python's 'foo'*3 or ['foo', 'bar']*3
1362 # negative numbers are like 0 in Python
1363
1364 var t = type(x)
1365 case (t) {
1366 Str {
1367 var parts = []
1368 for i in (0 ..< n) {
1369 call parts->append(x)
1370 }
1371 return (join(parts))
1372 }
1373 List {
1374 var result = []
1375 for i in (0 ..< n) {
1376 call result->extend(x)
1377 }
1378 return (result)
1379 }
1380 (else) {
1381 error "Expected Str or List, got $t"
1382 }
1383 }
1384}
1385)zZXx");
1386
1387GLOBAL_STR(gStr44, R"zZXx(const __provide__ = :| identity max min abs sum |
1388
1389func identity(x) {
1390 ### The identity function. Returns its argument.
1391
1392 return (x)
1393}
1394
1395func __math_select(list, cmp) {
1396 ## Internal helper for `max` and `min`.
1397 ##
1398 ## NOTE: If `list` is empty, then an error is thrown.
1399
1400 if (len(list) === 0) {
1401 error "Unexpected empty list" (code=3)
1402 }
1403
1404 if (len(list) === 1) {
1405 return (list[0])
1406 }
1407
1408 var match = list[0]
1409 for i in (1 ..< len(list)) {
1410 setvar match = cmp(list[i], match)
1411 }
1412 return (match)
1413}
1414
1415func max(...args) {
1416 ## Compute the maximum of 2 or more values.
1417 ##
1418 ## `max` takes two different signatures:
1419 ## - `max(a, b)` to return the maximum of `a`, `b`
1420 ## - `max(list)` to return the greatest item in the `list`
1421 ##
1422 ## So, for example:
1423 ##
1424 ## max(1, 2) # => 2
1425 ## max([1, 2, 3]) # => 3
1426
1427 case (len(args)) {
1428 (1) { return (__math_select(args[0], max)) }
1429 (2) {
1430 if (args[0] > args[1]) {
1431 return (args[0])
1432 } else {
1433 return (args[1])
1434 }
1435 }
1436 (else) { error "max expects 1 or 2 args" (code=3) }
1437 }
1438}
1439
1440func min(...args) {
1441 ## Compute the minimum of 2 or more values.
1442 ##
1443 ## `min` takes two different signatures:
1444 ## - `min(a, b)` to return the minimum of `a`, `b`
1445 ## - `min(list)` to return the least item in the `list`
1446 ##
1447 ## So, for example:
1448 ##
1449 ## min(2, 3) # => 2
1450 ## max([1, 2, 3]) # => 1
1451
1452 case (len(args)) {
1453 (1) { return (__math_select(args[0], min)) }
1454 (2) {
1455 if (args[0] < args[1]) {
1456 return (args[0])
1457 } else {
1458 return (args[1])
1459 }
1460 }
1461 (else) { error "min expects 1 or 2 args" (code=3) }
1462 }
1463}
1464
1465func abs(x) {
1466 ## Compute the absolute (positive) value of a number (float or int).
1467
1468 if (x < 0) {
1469 return (-x)
1470 } else {
1471 return (x)
1472 }
1473}
1474
1475func sum(list; start=0) {
1476 ### Returns the sum of all elements in the list.
1477 # Empty list: returns 0
1478
1479 var sum = start
1480 for item in (list) {
1481 setvar sum += item
1482 }
1483 return (sum)
1484}
1485)zZXx");
1486
1487GLOBAL_STR(gStr45, R"zZXx(const __provide__ = :| sh shell ninja make |
1488
1489# Issues for quoting:
1490#
1491# 1. What is the alphabet we're quoting to?
1492# - Output can be "all bytes", "all unicode strings", or "ASCII"
1493# 2. Start with a simple algorithm to quote everything
1494# - POSIX shell may take ' to ''\'''
1495# 3. Heuristic that may avoid quotes, to make it more readable
1496# - But really it should be \'
1497# - If the ' appears at the beginning or the end, we could have a different
1498# algorithm. Or we could strip leading and trailing ''
1499# 4. Are there any byte strings / unicode strings that can't be quoted?
1500# - e.g. NUL bytes?
1501# - for JSON/JS, binary strings? Only Unicode strings can be encoded.
1502#
1503# Builtins:
1504# toJson() toJson8()
1505# toJ8Line() - does the "maybe unquoted" logic
1506#
1507# Related functions:
1508# encode.base{16,32,64} - Crockford has a base32
1509# decode.base{16,32,64}
1510# Also base85, and maybe base58 base36
1511#
1512# In Python: bin() oct() hex() int(i, 9) and %o %x (there is no %b)
1513#
1514# Other:
1515# Punycode for Unicode domain names uses xn-- ?
1516# CSS has escapes with \
1517# HTTP Cookies have "" and \?
1518#
1519# Related:
1520# demo/url-search-params.ysh is the PARSER / unquoter for quote.urlParam()
1521
1522func sh(s) {
1523 ### Quote POSIX sh string
1524
1525 # replace ' with sequence ' \' '
1526 # Note: the \\ is hard to read - '' doesn't help
1527 return ("'" ++ s.replace("'", "'\\''") ++ "'")
1528}
1529
1530func shell(s) {
1531 ### Quote shell string, where bash and zsh style $'\n' is allowed
1532
1533 # TODO: Binding for j8_lite.MaybeShellEncode / ShellEncode
1534 return (toJson(s))
1535}
1536
1537func ysh(s) {
1538 ### Quote YSH string as b'\n'
1539
1540 # TODO: Binding for j8_lite.YshEncode(unquoted_ok)
1541 return (toJson(s))
1542}
1543
1544#
1545# Build Tools
1546#
1547
1548func make(s) {
1549 var out = []
1550 var n = len(s)
1551 for i in (0 ..< n) {
1552 var ch = s[i]
1553
1554 case (ch) {
1555 (\r) |
1556 (\n) {
1557 error "Can't quote newlines for Make?"
1558 }
1559 ('$') {
1560 call out->append('$')
1561 call out->append('$')
1562 }
1563 ('%') |
1564 (r'\') |
1565 # glob characters
1566 ('[') | (']') | ('*') | ('?') {
1567 call out->append(r'\')
1568 call out->append(ch)
1569 }
1570 (else) {
1571 call out->append(ch)
1572 }
1573 }
1574 }
1575 return (join(out, ''))
1576}
1577
1578# https://ninja-build.org/manual.html#ref_lexer
1579# $ escapes newline, space, : and $
1580# and then the rest is interpreted by /bin/sh
1581func ninja(s) {
1582 var out = []
1583 var n = len(s)
1584 for i in (0 ..< n) {
1585 var ch = s[i]
1586
1587 case (ch) {
1588 # Subtlety: Ninja allows $ to escape a newline, but it's only for
1589 # line continuations - for breaking long lists of files.
1590 # - A file itself should not have a newline.
1591 # - It strips literal newlines out of shell commands.
1592 # So disallow it
1593 (\r) |
1594 (\n) {
1595 error "Can't quote newlines for Ninja"
1596 }
1597 ('$') |
1598 (' ') |
1599 #('#') | # Ninja has no way to escape comments!
1600 (':') {
1601 call out->append('$')
1602 call out->append(ch)
1603 }
1604 (else) {
1605 call out->append(ch)
1606 }
1607 }
1608 }
1609 return (join(out, ''))
1610}
1611
1612# I don't know the rule here - uses \?
1613func cmake(s) {
1614 return (s)
1615}
1616
1617#
1618# Tables, Objects, Documents
1619#
1620
1621func csv(s) {
1622 # double up " etc.
1623 return (s)
1624}
1625
1626func sql(s) {
1627 # double up ' etc.
1628 return (s)
1629}
1630
1631# quote.json is just toJson()
1632
1633func html(s) {
1634 ### Quote shell string, where bash and zsh style $'\n' is allowed
1635
1636 # Binding for j8_lite.MaybeShellEncode / ShellEncode
1637 return (toJson(s))
1638}
1639
1640#
1641# Web
1642#
1643
1644func urlParam(s) {
1645 # urllib.quote
1646 # 'foo bar %' -> 'foo+bar %AB'
1647 return (toJson(s))
1648}
1649
1650#
1651# Programming Languages
1652#
1653
1654# Python 2 or 3
1655func py(s) {
1656 return (s)
1657}
1658
1659# C or C++
1660# Can it be Unicode?
1661func c(s) {
1662 return (s)
1663}
1664
1665# quote.js is just toJson() ?
1666# But it can't handle binary strings?
1667# We can make a table?
1668
1669#
1670# Windows
1671#
1672
1673# We want something that works everywhere -- it should never DOUBLE-ESCAPE, but
1674# it can "unnecessarily" ESCAPE certain characters.
1675
1676# C runtime
1677func win32_crt(s) {
1678 return (s)
1679}
1680
1681# win32 cmd.exe batch files?
1682func win32_cmd(s) {
1683 return (s)
1684}
1685
1686# batch different than cmd.exe?
1687func win32_batch(s) {
1688 return (s)
1689}
1690)zZXx");
1691
1692GLOBAL_STR(gStr46, R"zZXx(# stream.ysh
1693#
1694# Usage:
1695# source --builtin stream.ysh
1696#
1697# For reading lines, decoding, extracting, splitting
1698
1699shopt --unset no_osh_builtins # allow OSH 'local' for now
1700source $LIB_OSH/byo-server.sh # make this file a test server
1701
1702source $LIB_YSH/args.ysh
1703
1704proc slurp-by (; num_lines) {
1705 var buf = []
1706 for line in (io.stdin) {
1707 call buf->append(line)
1708 if (len(buf) === num_lines) {
1709 json write (buf, space=0)
1710
1711 # TODO:
1712 #call buf->clear()
1713 setvar buf = []
1714 }
1715 }
1716 if (buf) {
1717 json write (buf, space=0)
1718 }
1719}
1720
1721proc test-slurp-by {
1722 seq 8 | slurp-by (3)
1723}
1724
1725### Awk
1726
1727# Naming
1728#
1729# TEXT INPUT
1730# each-word # this doesn't go by lines, it does a global regex split or something?
1731#
1732# LINE INPUT
1733# each-line --j8 { echo "-- $_line" } # similar to @()
1734# each-line --j8 (^"-- $_line") # is this superfluous?
1735#
1736# each-split name1 name2
1737# (delim=' ')
1738# (ifs=' ')
1739# (pat=/d+/)
1740# # also assign names for each part?
1741#
1742# each-match # regex match
1743# must-match # assert that every line matches
1744#
1745# TABLE INPUT
1746# each-row # TSV and TSV8 input?
1747#
1748# They all take templates or blocks?
1749
1750proc each-line (...words; template=null; ; block=null) {
1751 # TODO:
1752 # parse --j8 --max-jobs flag
1753
1754 # parse template_str as string
1755 # TODO: this is dangerous though ... because you can execute code
1756 # I think you need a SAFE version
1757
1758 # evaluate template string expression - I guess that allows $(echo hi) and so
1759 # forth
1760
1761 # evaluate block with _line binding
1762 # block: execute in parallel with --max-jobs
1763
1764 for line in (stdin) {
1765 echo TODO
1766 }
1767}
1768
1769proc test-each-line {
1770 echo 'TODO: need basic test runner'
1771
1772 # ysh-tool test stream.ysh
1773 #
1774 # Col
1775}
1776
1777proc each-j8-line (; ; ; block) {
1778 for _line in (io.stdin) {
1779 # TODO: fromJ8Line() toJ8Line()
1780 # var _line = fromJson(_line)
1781 call io->eval(block, vars={_line})
1782 }
1783}
1784
1785proc test-each-j8-line {
1786 var lines = []
1787 var prefix = 'z'
1788
1789 # unquoted
1790 seq 3 | each-j8-line {
1791 call lines->append(prefix ++ _line)
1792 }
1793 pp test_ (lines)
1794
1795 # Note: no trailing new lines, since they aren't significant in Unix
1796 var expected = ['z1', 'z2', 'z3']
1797 assert [expected === lines]
1798}
1799
1800proc each-row (; ; block) {
1801 echo TODO
1802}
1803
1804proc split-by (; delim; ifs=null; block) {
1805
1806 # TODO: provide the option to bind names? Or is that a separate thing?
1807 # The output of this is "ragged"
1808
1809 for line in (io.stdin) {
1810 #pp (line)
1811 var parts = line.split(delim)
1812 pp (parts)
1813
1814 # variable number
1815 call io->eval(block, dollar0=line, pos_args=parts)
1816 }
1817}
1818
1819proc chop () {
1820 ### alias for split-by
1821 echo TODO
1822}
1823
1824proc test-split-by {
1825 var z = 'z' # test out scoping
1826 var count = 0 # test out mutation
1827
1828 # TODO: need split by space
1829 # Where the leading and trailing are split
1830 # if-split-by(' ') doesn't work well
1831
1832 line-data | split-by (/s+/) {
1833
1834 # how do we deal with nonexistent?
1835 # should we also bind _parts or _words?
1836
1837 echo "$z | $0 | $1 | $z"
1838
1839 setvar count += 1
1840 }
1841 echo "count = $count"
1842}
1843
1844proc must-split-by (; ; ifs=null; block) {
1845 ### like if-split-by
1846
1847 echo TODO
1848}
1849
1850# Naming: each-match, each-split?
1851
1852proc if-match (; pattern, template=null; ; block=null) {
1853 ### like 'grep' but with submatches
1854
1855 for line in (io.stdin) {
1856 var m = line.search(pattern)
1857 if (m) {
1858 #pp asdl_ (m)
1859 #var groups = m.groups()
1860
1861 # Should we also pass _line?
1862
1863 if (block) {
1864 call io->eval(block, dollar0=m.group(0))
1865 } elif (template) {
1866 echo TEMPLATE
1867 } else {
1868 echo TSV
1869 }
1870 }
1871 }
1872
1873 # always succeeds - I think must-match is the one that can fail
1874}
1875
1876proc must-match (; pattern; block) {
1877 ### like if-match
1878
1879 echo TODO
1880}
1881
1882proc line-data {
1883 # note: trailing ''' issue, I should probably get rid of the last line
1884
1885 write --end '' -- '''
1886 prefix 30 foo
1887 oils
1888 /// 42 bar
1889 '''
1890}
1891
1892const pat = /<capture d+> s+ <capture w+>/
1893
1894proc test-if-match {
1895 var z = 'z' # test out scoping
1896 var count = 0 # test out mutation
1897
1898 # Test cases should be like:
1899 # grep: print the matches, or just count them
1900 # sed: print a new line based on submatches
1901 # awk: re-arrange the cols, and also accumulate counters
1902
1903 line-data | if-match (pat) {
1904 echo "$z $0 $z"
1905 # TODO: need pos_args
1906
1907 #echo "-- $2 $1 --"
1908
1909 setvar count += 1
1910 }
1911 echo "count = $count"
1912}
1913
1914proc test-if-match-2 {
1915 # If there's no block or template, it should print out a TSV with:
1916 #
1917 # $0 ...
1918 # $1 $2
1919 # $_line maybe?
1920
1921 #line-data | if-match (pat)
1922
1923 var z = 'z' # scoping
1924 line-data | if-match (pat, ^"$z $0 $z")
1925 line-data | if-match (pat, ^"-- $0 --")
1926}
1927
1928# might be a nice way to write it, not sure if byo.sh can discover it
1929if false {
1930tests 'if-match' {
1931 proc case-block {
1932 echo TODO
1933 }
1934 proc case-template {
1935 echo TODO
1936 }
1937}
1938}
1939
1940# Protocol:
1941#
1942# - The file lists its tests the "actions"
1943# - Then the test harness runs them
1944# - But should it be ENV vars
1945#
1946# - BYO_LIST_TESTS=1
1947# - BYO_RUN_TEST=foo
1948# - $PWD is a CLEAN temp dir, the process doesn't have to do anything
1949
1950# - silent on success, but prints file on output
1951# - OK this makes sense
1952#
1953# The trivial test in Python:
1954#
1955# from test import byo
1956# byo.maybe_main()
1957#
1958# bash library:
1959# source --builtin byo-server.sh
1960#
1961# byo-maybe-main # reads env variables, and then exits
1962#
1963# source --builtin assertions.ysh
1964#
1965# assert-ok 'echo hi'
1966# assert-stdout 'hi' 'echo -n hi'
1967#
1968# "$@"
1969#
1970# Run all tests
1971# util/byo-client.sh run-tests $YSH stdlib/table.ysh
1972# util/byo-client.sh run-tests -f x $YSH stdlib/table.ysh
1973
1974# Clean process
1975# Clean working dir
1976
1977#
1978# Stream Protocol:
1979# #.byo - is this she-dot, that's for a file
1980# Do we need metadata?
1981#
1982
1983# The harness
1984#
1985# It's process based testing.
1986#
1987# Test runner process: bash or OSH (unlike sharness!)
1988# Tested process: any language - bash,
1989#
1990# Key point: you don't have to quote shell code?
1991
1992list-byo-tests() {
1993 echo TODO
1994}
1995
1996run-byo-tests() {
1997 # source it
1998 echo TODO
1999}
2000
2001byo-maybe-run
2002)zZXx");
2003
2004GLOBAL_STR(gStr47, R"zZXx(# table.ysh - Library for tables.
2005#
2006# Usage:
2007# source --builtin table.ysh
2008
2009shopt --unset no_osh_builtins # allow OSH 'local' for now
2010source $LIB_OSH/byo-server.sh # make this file a test server
2011
2012proc table (...words; place; ; block) {
2013 var n = len(words)
2014
2015 # TODO: parse flags
2016 #
2017 # --by-row
2018 # --by-col
2019 #
2020 # Place is optional
2021
2022 if (n === 0) {
2023 echo TODO
2024 return
2025 }
2026
2027 var action = words[0]
2028
2029 # textual operations
2030 case (action) {
2031 cat {
2032 echo todo
2033 }
2034 align {
2035 echo todo
2036 }
2037 tabify {
2038 echo todo
2039 }
2040 tabify {
2041 echo todo
2042 }
2043 header {
2044 echo todo
2045 }
2046 slice {
2047 # this has typed args
2048 # do we do some sort of splat?
2049 echo todo
2050 }
2051 to-tsv {
2052 echo todo
2053 }
2054 }
2055
2056 echo TODO
2057}
2058
2059proc test-table {
2060 return
2061
2062 table (&files1) {
2063 cols num_bytes path
2064 type Int Str
2065
2066 row 10 README.md
2067 row 12 main.py
2068
2069 row (12, 'lib.py')
2070 row (num_bytes=12, path='util.py')
2071 }
2072
2073 # 2 columns - The default is by column?
2074 assert ['Dict' === type(files1)]
2075 assert [2 === len(files1)]
2076
2077 # Same table
2078 table --by-row (&files2) {
2079 cols num_bytes path
2080 type Int Str
2081
2082 row 10 README.md
2083 row 12 main.py
2084
2085 row (12, 'lib.py')
2086 row (num_bytes=12, path='util.py')
2087 }
2088
2089 # 4 rows
2090 assert ['List' === type(files2)]
2091 assert [4 === len(files2)]
2092}
2093
2094# "subcommands" of the dialect
2095
2096proc cols (...names) {
2097 # cols name age
2098 echo TODO
2099}
2100
2101proc types (...types) {
2102 # types - Int? Str?
2103 echo TODO
2104}
2105
2106proc attr (name; ...values) {
2107 # attr units ('-', 'secs')
2108 echo TODO
2109}
2110
2111# is this allowed outside table {} blocks too?
2112proc row {
2113 echo TODO
2114}
2115
2116#
2117# dplyr names
2118#
2119
2120# TODO: can we parse select?
2121
2122proc where {
2123 echo
2124}
2125
2126# TODO: should be able to test argv[0] or something
2127# Or pass to _mutate-transmute
2128
2129proc mutate {
2130 echo TODO
2131}
2132
2133proc transmute {
2134 echo TODO
2135}
2136
2137proc rename {
2138 echo TODO
2139}
2140
2141proc group-by {
2142 echo TODO
2143}
2144
2145proc sort-by {
2146 echo TODO
2147}
2148
2149proc summary {
2150 echo TODO
2151}
2152
2153byo-maybe-run
2154)zZXx");
2155
2156GLOBAL_STR(gStr48, R"zZXx(#!/usr/bin/env bash
2157#
2158# Testing library for bash and OSH.
2159#
2160# Capture status/stdout/stderr, and nq-assert those values.
2161
2162const __provide__ = :| yb-capture yb-capture-2 |
2163
2164: ${LIB_OSH=stdlib/osh}
2165source $LIB_OSH/two.sh
2166
2167# There is no yb-run, because you can just use try { } and inspect _error.code
2168# There is no yb-redir, because you can just use try >$tmp { } and inspect _error.code
2169
2170proc yb-capture(; out; ; block) {
2171 ### capture status and stdout
2172
2173 var stdout = ''
2174 try {
2175 { call io->eval(block) } | read --all (&stdout)
2176
2177 # Note that this doesn't parse because of expression issue:
2178 # call io->eval(block) | read --all (&stdout)
2179 # used to be eval (block)
2180 }
2181 # TODO: if 'block' contains a pipeline, we lose this magic var
2182 var result = {status: _pipeline_status[0], stdout}
2183
2184 #echo 'result-1'
2185 #pp test_ (result)
2186
2187 call out->setValue(result)
2188}
2189
2190proc yb-capture-2(; out; ; block) {
2191 ### capture status and stderr
2192
2193 var stderr = ''
2194 try {
2195 redir 2>&1 { call io->eval(block); } | read --all (&stderr)
2196
2197 # Note that this doesn't parse because of expression issue:
2198 # call io->eval(block) 2>&1 | read --all (&stderr)
2199 # used to be eval (block) 2>&1
2200 }
2201 #pp test_ (_pipeline_status)
2202
2203 var result = {status: _pipeline_status[0], stderr}
2204 #echo 'result-2'
2205 #pp test_ (result)
2206
2207 call out->setValue(result)
2208}
2209)zZXx");
2210
2211
2212
2213TextFile array[] = {
2214 {.rel_path = "_devbuild/help/data-errors", .contents = gStr0},
2215 {.rel_path = "_devbuild/help/data-front-end", .contents = gStr1},
2216 {.rel_path = "_devbuild/help/data-j8-notation", .contents = gStr2},
2217 {.rel_path = "_devbuild/help/help", .contents = gStr3},
2218 {.rel_path = "_devbuild/help/oils-usage", .contents = gStr4},
2219 {.rel_path = "_devbuild/help/osh-builtin-cmd", .contents = gStr5},
2220 {.rel_path = "_devbuild/help/osh-chapters", .contents = gStr6},
2221 {.rel_path = "_devbuild/help/osh-cmd-lang", .contents = gStr7},
2222 {.rel_path = "_devbuild/help/osh-front-end", .contents = gStr8},
2223 {.rel_path = "_devbuild/help/osh-mini-lang", .contents = gStr9},
2224 {.rel_path = "_devbuild/help/osh-option", .contents = gStr10},
2225 {.rel_path = "_devbuild/help/osh-osh-assign", .contents = gStr11},
2226 {.rel_path = "_devbuild/help/osh-plugin", .contents = gStr12},
2227 {.rel_path = "_devbuild/help/osh-special-var", .contents = gStr13},
2228 {.rel_path = "_devbuild/help/osh-stdlib", .contents = gStr14},
2229 {.rel_path = "_devbuild/help/osh-type-method", .contents = gStr15},
2230 {.rel_path = "_devbuild/help/osh-usage", .contents = gStr16},
2231 {.rel_path = "_devbuild/help/osh-word-lang", .contents = gStr17},
2232 {.rel_path = "_devbuild/help/shell-flags", .contents = gStr18},
2233 {.rel_path = "_devbuild/help/ysh-builtin-cmd", .contents = gStr19},
2234 {.rel_path = "_devbuild/help/ysh-builtin-func", .contents = gStr20},
2235 {.rel_path = "_devbuild/help/ysh-chapters", .contents = gStr21},
2236 {.rel_path = "_devbuild/help/ysh-cmd-lang", .contents = gStr22},
2237 {.rel_path = "_devbuild/help/ysh-expr-lang", .contents = gStr23},
2238 {.rel_path = "_devbuild/help/ysh-front-end", .contents = gStr24},
2239 {.rel_path = "_devbuild/help/ysh-mini-lang", .contents = gStr25},
2240 {.rel_path = "_devbuild/help/ysh-option", .contents = gStr26},
2241 {.rel_path = "_devbuild/help/ysh-plugin", .contents = gStr27},
2242 {.rel_path = "_devbuild/help/ysh-special-var", .contents = gStr28},
2243 {.rel_path = "_devbuild/help/ysh-stdlib", .contents = gStr29},
2244 {.rel_path = "_devbuild/help/ysh-type-method", .contents = gStr30},
2245 {.rel_path = "_devbuild/help/ysh-usage", .contents = gStr31},
2246 {.rel_path = "_devbuild/help/ysh-word-lang", .contents = gStr32},
2247 {.rel_path = "_devbuild/help/ysh-ysh-cmd", .contents = gStr33},
2248 {.rel_path = "stdlib/methods.ysh", .contents = gStr34},
2249 {.rel_path = "stdlib/osh/bash-strict.sh", .contents = gStr35},
2250 {.rel_path = "stdlib/osh/byo-server.sh", .contents = gStr36},
2251 {.rel_path = "stdlib/osh/no-quotes.sh", .contents = gStr37},
2252 {.rel_path = "stdlib/osh/task-five.sh", .contents = gStr38},
2253 {.rel_path = "stdlib/osh/two.sh", .contents = gStr39},
2254 {.rel_path = "stdlib/prelude.ysh", .contents = gStr40},
2255 {.rel_path = "stdlib/ysh/args.ysh", .contents = gStr41},
2256 {.rel_path = "stdlib/ysh/def.ysh", .contents = gStr42},
2257 {.rel_path = "stdlib/ysh/list.ysh", .contents = gStr43},
2258 {.rel_path = "stdlib/ysh/math.ysh", .contents = gStr44},
2259 {.rel_path = "stdlib/ysh/quote.ysh", .contents = gStr45},
2260 {.rel_path = "stdlib/ysh/stream.ysh", .contents = gStr46},
2261 {.rel_path = "stdlib/ysh/table.ysh", .contents = gStr47},
2262 {.rel_path = "stdlib/ysh/yblocks.ysh", .contents = gStr48},
2263
2264 {.rel_path = nullptr, .contents = nullptr},
2265};
2266
2267} // namespace embedded_file
2268
2269TextFile* gEmbeddedFiles = embedded_file::array; // turn array into pointer