OILS / _gen / bin / text_files.cc View on Github | oils.pub

2268 lines, 167 significant
1
2#include "cpp/embedded_file.h"
3
4namespace embedded_file {
5GLOBAL_STR(gStr0, R"zZXx(Errors
6
7 [UTF8] err-utf8-encode err-utf8-decode
8 [J8 String] err-j8-str-encode err-j8-str-decode
9 [J8 Lines] err-j8-lines-encode err-j8-lines-decode
10 [JSON] err-json-encode err-json-decode
11 [JSON8] err-json8-encode err-json8-decode
12)zZXx");
13
14GLOBAL_STR(gStr1, R"zZXx(Front End
15
16 [Lexing] ascii-whitespace [ \t\r\n]
17 ascii-control-chars
18)zZXx");
19
20GLOBAL_STR(gStr2, R"zZXx(J8 Notation
21
22 [J8 Strings] json-string "hi"
23 json-escape \" \\ \u1234
24 surrogate-pair \ud83e\udd26
25 j8-escape \' \u{1f926} \yff
26 u-prefix u'hi'
27 b-prefix b'hi'
28 no-prefix 'hi'
29 [J8 Lines] unquoted-line
30 [JSON8] json8-num json8-str
31 X json8-list X json8-dict
32 json8-comment
33 [TSV8] column-attrs column-types
34)zZXx");
35
36GLOBAL_STR(gStr3, R"zZXx(Usage: help TOPIC?
37
38Examples:
39
40 help # this help
41 help echo # help on the 'echo' builtin
42 help command-sub # help on command sub $(date)
43
44 help oils-usage # identical to oils-for-unix --help
45 help osh-usage # osh --help
46 help ysh-usage # ysh --help
47)zZXx");
48
49GLOBAL_STR(gStr4, R"zZXx(bin/oils-for-unix is an executable that contains OSH, YSH, and more.
50
51Usage:
52 oils-for-unix MAIN_NAME ARG*
53 MAIN_NAME ARG*
54
55It behaves like busybox. The command name can be passed as the first argument:
56
57 oils-for-unix ysh -c 'echo hi'
58
59More commonly, it's invoked through a symlink like 'ysh', which causes it to
60behave like that command:
61
62 ysh -c 'echo hi'
63)zZXx");
64
65GLOBAL_STR(gStr5, R"zZXx(Builtin Commands
66
67 [I/O] read echo printf
68 readarray mapfile
69 [Run Code] source . cmd/eval trap
70 [Set Options] set shopt
71 [Working Dir] cd chdir pwd
72 pushd popd dirs
73 [Completion] complete compgen compopt compadjust compexport
74 [Shell Process] exec X logout
75 umask ulimit times
76 [Child Process] jobs wait
77 fg X bg X kill X disown
78 [External] test [ getopts
79 [Conditional] cmd/true cmd/false colon :
80 [Introspection] help hash cmd/type X caller
81 [Word Lookup] command builtin
82 [Interactive] alias unalias history fc X bind
83X [Unsupported] enable
84)zZXx");
85
86GLOBAL_STR(gStr6, R"zZXx(The reference is divided in to "chapters", each of which has its own table of
87contents. Type:
88
89 help osh-$CHAPTER
90
91Where $CHAPTER is one of:
92
93 type-method
94 builtin-cmd
95 stdlib
96 front-end
97 cmd-lang
98 osh-assign
99 word-lang
100 mini-lang
101 option
102 special-var
103 plugin
104
105Example:
106
107 help osh-word-lang
108)zZXx");
109
110GLOBAL_STR(gStr7, R"zZXx(Command Language
111
112 [Commands] simple-command command-lookup-order
113 prefix-binding semicolon ;
114 [Conditional] case if dbracket [[
115 bang ! and && or ||
116 [Iteration] while until for for-expr-sh ((
117 [Control Flow] break continue return exit
118 [Grouping] sh-func sh-block { subshell (
119 [Concurrency] pipe | X pipe-amp |& ampersand &
120 [Redirects] redir-file > >> >| < <> not impl: &>
121 redir-desc >& <&
122 here-doc << <<-
123 here-str <<<
124 [Other Command] dparen (( time X coproc X select
125)zZXx");
126
127GLOBAL_STR(gStr8, R"zZXx(Front End
128
129 [Usage] oils-usage osh-usage shell-flags
130 config startup line-editing
131 exit-codes
132 [Lexing] comment # line-continuation \ ascii-whitespace [ \t\r\n]
133)zZXx");
134
135GLOBAL_STR(gStr9, R"zZXx(Other Mini Languages
136
137 [Arithmetic] arith-context Where legacy arithmetic is allowed
138 sh-numbers 0xFF 0755 etc.
139 sh-arith 1 + 2*3 a *= 2
140 sh-logical !a && b
141 sh-bitwise ~a ^ b
142 [Boolean] bool-expr [[ ! $x && $y || $z ]]
143 test ! $x -a $y -o $z
144 bool-infix $a -nt $b $x == $y
145 bool-path -d /etc
146 bool-str -n foo -z ''
147 bool-other -o errexit -v name[index]
148 [Patterns] glob-pat *.py
149 extglob ,(*.py|*.sh)
150 regex [[ foo =~ [a-z]+ ]]
151 [Other Sublang] braces {alice,bob}@example.com
152 histsub !$ !! !n
153 char-escapes \t \c \x00 \u03bc
154)zZXx");
155
156GLOBAL_STR(gStr10, R"zZXx(Global Shell Options
157
158 [Errors] nounset -u errexit -e inherit_errexit pipefail
159 [Globbing] noglob -f nullglob failglob X dotglob
160 dashglob (true)
161 [Other Option] noclobber -C
162 [Debugging] errtrace -E extdebug X verbose xtrace -x
163 [Interactive] emacs vi
164 [Compat] eval_unsafe_arith ignore_flags_not_impl
165 ignore_shopt_not_impl
166 [Optimize] rewrite_extern
167)zZXx");
168
169GLOBAL_STR(gStr11, R"zZXx(Assignments and Expressions
170
171 [Literals] sh-init-list a=(v1 [i]=v2 [k]+=v3) a+=(v1 [i]=v2 [k]+=v3)
172 sh-array array[123]=v "${array[@]}"
173 sh-assoc assoc['k']=v "${assoc[@]}" "${!assoc[@]}"
174 [Operators] sh-assign str='xyz'
175 sh-append str+='abc'
176 [Builtins] local readonly export unset shift
177 declare typeset X let
178)zZXx");
179
180GLOBAL_STR(gStr12, R"zZXx(Plugins and Hooks
181
182 [Signals] SIGTERM SIGINT SIGQUIT
183 SIGTTIN SIGTTOU SIGWINCH
184 [Traps] DEBUG ERR EXIT X RETURN
185 [Words] PS1 X PS2 X PS3 PS4
186 [Completion] complete
187 [Other Plugin] PROMPT_COMMAND X command_not_found
188)zZXx");
189
190GLOBAL_STR(gStr13, R"zZXx(Special Variables
191
192 [Oils VM] OILS_VERSION LIB_OSH
193 [libc locale] osh-locale
194 [Interactive] OILS_COMP_UI HISTFILE
195 [POSIX Special] $@ $* $# $? $- $$ $! $0 $9
196 [Shell Vars] IFS X LANG X GLOBIGNORE
197 [Shell Options] SHELLOPTS X BASHOPTS
198 [Other Env] HOME PATH
199 [Other Special] BASH_REMATCH @PIPESTATUS
200 [Platform] HOSTNAME OSTYPE
201 [Call Stack] @BASH_SOURCE @FUNCNAME @BASH_LINENO
202 X @BASH_ARGV X @BASH_ARGC
203 [Tracing] LINENO
204 [Process State] UID EUID PPID X BASHPID
205X [Process Stack] BASH_SUBSHELL SHLVL
206X [Shell State] BASH_CMDS @DIRSTACK
207 [Completion] @COMP_WORDS COMP_CWORD COMP_LINE COMP_POINT
208 COMP_WORDBREAKS @COMPREPLY X COMP_KEY
209 X COMP_TYPE COMP_ARGV
210 [cd] PWD OLDPWD X CDPATH
211 [getopts] OPTIND OPTARG X OPTERR
212 [read] REPLY
213 [Functions] X RANDOM SECONDS
214)zZXx");
215
216GLOBAL_STR(gStr14, R"zZXx(Standard Library
217
218 [two] log die
219 [no-quotes] nq-assert nq-run
220 nq-capture nq-capture-2
221 nq-redir nq-redir-2
222 [bash-strict]
223 [task-five]
224)zZXx");
225
226GLOBAL_STR(gStr15, R"zZXx(OSH Types
227
228 [OSH] BashArray BashAssoc
229)zZXx");
230
231GLOBAL_STR(gStr16, R"zZXx(bin/osh is compatible with POSIX shell, bash, and other shells.
232
233Usage:
234 osh FLAG* SCRIPT ARG*
235 osh FLAG* -c COMMAND ARG*
236 osh FLAG*
237
238Examples:
239 osh -c 'echo hi'
240 osh myscript.sh
241 echo 'echo hi' | osh
242)zZXx");
243
244GLOBAL_STR(gStr17, R"zZXx(Word Language
245
246 [Glob] osh-glob *.py
247 [Joining] osh-word-join 'single'\'$myvar"double $x"
248 [Quotes] osh-string 'abc' $'line\n' "$var"
249 [Substitutions] command-sub $(date) `date`
250 var-sub ${var} $0 $9
251 arith-sub $((1 + 2))
252 tilde-sub ~/src
253 proc-sub diff <(sort L.txt) <(sort R.txt)
254 [Var Ops] op-bracket ${a[i+1]}, ${a[*]}
255 op-indirect ${!x}
256 op-test ${x:-default}
257 op-strip ${x%%suffix} etc.
258 op-patsub ${x//y/z}
259 op-slice ${a[@]:0:1}
260 op-format ${x@P} ${x@Q} etc.
261)zZXx");
262
263GLOBAL_STR(gStr18, R"zZXx(osh and ysh accept standard POSIX shell flags, like:
264
265 bin/osh -o errexit -c 'false'
266 bin/ysh -n myfile.ysh
267 bin/ysh +o errexit -c 'false; echo ok'
268
269They also accept these flags:
270
271 --eval FILE
272 Evaluate the given file, similar to the 'source' builtin. Specify it
273 multiple times to run multiple files.
274
275 If the errexit option is on (e.g. in YSH), then the shell stops when $?
276 is non-zero after evaluating a file. The $0 value is set to FILE.
277
278 --eval-pure FILE
279 Like --eval, but disallow I/O (known as "pure mode").
280
281 --location-str
282 Use this string to display error messages.
283 See 'help sourceCode' for an example.
284
285 --location-start-line
286 Use this line number offset to display error messages.
287
288 --tool Run a tool instead of the shell (cat-em|syntax-tree)
289 -n Parse the program but don't execute it. Print the AST.
290 --ast-format FMT The format for the AST (text|text-abbrev)
291
292Examples:
293
294 ysh --eval one.ysh --eval two.ysh -c 'echo hi' # Run 2 files first
295 osh -n -c 'hello' # pretty-print the AST
296 ysh --ast-format text -n -c 'hello' # in unabridged format
297)zZXx");
298
299GLOBAL_STR(gStr19, R"zZXx(Builtin Commands
300
301 [Memory] cmd/append Add elements to end of array
302 pp value proc test_
303 asdl_ cell_ X gc-stats_
304 [Handle Errors] error error 'failed' (status=2)
305 try Run with errexit, set _error
306 failed Test if _error.code !== 0
307 boolstatus Enforce 0 or 1 exit status
308 assert assert [42 === f(x)]
309 [Shell State] ysh-cd ysh-shopt compatible, and takes a block
310 shvar Temporary modify global settings
311 ctx Share and update a temporary "context"
312 push-registers Save registers like $?, PIPESTATUS
313 [Word Lookup] invoke Run a command, and control name lookup
314 runproc Run a proc; use as main entry point
315 X extern Run an external command, with an ENV
316 [Modules]
317 source-guard guard against duplicate 'source'
318 is-main false when sourcing a file
319 use create a module Obj from a source file
320 [I/O] ysh-read flags --all, -0
321 ysh-echo no -e -n with simple_echo
322 ysh-test --file --true etc.
323 ysh-wait wait --all --verbose
324 write Like echo, with --, --sep, --end
325 fork forkwait Replace & and (), and takes a block
326 redir Run a block, with redirects
327 [Run Code] ysh-trap
328 [Private] cat rm POSIX-compatible
329 sleep
330 [Hay Config] hay haynode For DSLs and config files
331 [Completion] compadjust compexport
332 [Data Formats] json read write
333 json8 read write
334)zZXx");
335
336GLOBAL_STR(gStr20, R"zZXx(Builtin Functions
337
338 [Values] len() func/type()
339 [Conversions] bool() int() float()
340 str() list() dict()
341 X runes() X encodeRunes()
342 X bytes() X encodeBytes()
343 [Str] strcmp() shSplit()
344 [List] join()
345 [Dict] keys() values() get()
346 [Float] floatsEqual() X isinf() X isnan()
347 [Obj] first() rest() get()
348 [Word] glob() maybe()
349 [Serialize] toJson() fromJson()
350 toJson8() fromJson8()
351 X toJ8Line() X fromJ8Line()
352 [Pattern] _group() _start() _end()
353 [Reflection] func/eval() func/evalExpr()
354 [Introspect] shvarGet() getVar() setVar()
355 parseCommand() X parseExpr() X bindFrame()
356 [Hay Config] parseHay() evalHay()
357X [Hashing] sha1dc() sha256()
358)zZXx");
359
360GLOBAL_STR(gStr21, R"zZXx(The reference is divided in to "chapters", each of which has its own table of
361contents. Type:
362
363 help ysh-$CHAPTER
364
365Where $CHAPTER is one of:
366
367 type-method
368 builtin-func
369 builtin-cmd
370 stdlib
371 front-end
372 cmd-lang
373 ysh-cmd
374 expr-lang
375 word-lang
376 option
377 special-var
378 plugin
379
380Example:
381
382 help ysh-expr-lang
383 help ysh-ysh-cmd # may change
384)zZXx");
385
386GLOBAL_STR(gStr22, R"zZXx(Command Language
387
388 [Commands] simple-command
389 command-lookup-order
390 ysh-prefix-binding
391 semicolon ;
392 [Redirects] ysh-here-str read <<< '''
393 [YSH Simple] typed-arg json write (x)
394 lazy-expr-arg assert [42 === x]
395 block-arg cd /tmp { echo $PWD }; cd /tmp (; ; blockexpr)
396 [YSH Cond] ysh-case case (x) { *.py { echo 'python' } }
397 ysh-if if (x > 0) { echo }
398 [YSH Iter] ysh-for for i, item in (mylist) { echo }
399 ysh-while while (x > 0) { echo }
400)zZXx");
401
402GLOBAL_STR(gStr23, R"zZXx(Expression Language and Assignments
403
404 [Assignment] assign =
405 aug-assign += -= *= /= **= //= %=
406 &= |= ^= <<= >>=
407 [Literals] atom-literal null true false
408 int-literal 42 65_536 0xFF 0o755 0b10
409 float-literal 3.14 1.5e-10
410 X num-suffix 42 K Ki M Mi G Gi T Ti / ms us
411 char-literal \\ \t \" \y00 \u{3bc}
412 ysh-string "x is $x" $"x is $x" r'[a-z]\n'
413 u'line\n' b'byte \yff'
414 triple-quoted """ $""" r''' u''' b'''
415 list-literal ['one', 'two', 3] :| unquoted words |
416 dict-literal {name: 'bob'} {a, b}
417 range 1 ..< n 1 ..= n
418 block-expr ^(echo $PWD)
419 expr-literal ^[1 + 2*3]
420 str-template ^"$a and $b" for Str.replace()
421 expr-sub $[myobj]
422 expr-splice @[myobj]
423 [Operators] op-precedence Like Python
424 concat s1 ++ s2, L1 ++ L2
425 ysh-equals === !== ~== is, is not
426 ysh-in in, not in
427 ysh-compare < <= > >= (numbers only)
428 ysh-logical not and or
429 ysh-arith + - * / // % **
430 ysh-unary + -
431 ysh-bitwise ~ & | ^ << >>
432 ysh-ternary '+' if x >= 0 else '-'
433 ysh-index s[0] mylist[3] mydict['key']
434 ysh-attr mydict.key mystr.startsWith('x')
435 ysh-slice a[1:-1] s[1:-1]
436 ysh-func-call f(x, y, ...pos; n=1, ...named)
437 thin-arrow mylist->pop()
438 fat-arrow mylist => join() => upper()
439 match-ops ~ !~ ~~ !~~
440 [Eggex] re-literal / d+ ; re-flags ; ERE /
441 re-primitive %zero 'sq'
442 class-literal [c a-z 'abc' @str_var \\ \y01 \u{3bc}]
443 re-chars \y01 \u{3bc}
444 named-class dot digit space word d s w
445 re-repeat d? d* d+ d{3} d{2,4}
446 re-compound seq1 seq2 alt1|alt2 (expr1 expr2)
447 re-capture <capture d+ as name: int>
448 re-splice Subpattern @subpattern
449 re-flags reg_icase reg_newline
450 X re-multiline ///
451)zZXx");
452
453GLOBAL_STR(gStr24, R"zZXx(Front End
454
455 [Usage] oils-usage ysh-usage shell-flags
456 config startup line-editing
457 exit-codes
458 [Lexing] comment # line-continuation \ ascii-whitespace [ \t\r\n]
459 [Lexing] doc-comment ### multiline-command ...
460 [Tools] cat-em syntax-tree
461)zZXx");
462
463GLOBAL_STR(gStr25, R"zZXx(Other Mini Languages
464
465 [Patterns] glob-pat *.py
466 [Other Sublang] braces {alice,bob}@example.com
467)zZXx");
468
469GLOBAL_STR(gStr26, R"zZXx(Global Shell Options
470
471 [Optimize] rewrite_extern
472 [Groups] strict:all ysh:upgrade ysh:all
473 [YSH Details] opts-redefine opts-internal
474)zZXx");
475
476GLOBAL_STR(gStr27, R"zZXx(Plugins and Hooks
477
478 [YSH] renderPrompt()
479)zZXx");
480
481GLOBAL_STR(gStr28, R"zZXx(Special Variables
482
483 [YSH Vars] ARGV ENV
484 __defaults__ __builtins__ X __sh_function__
485 _this_dir
486 [YSH Status] _error
487 _pipeline_status _process_sub_status
488 [YSH Tracing] SHX_indent SHX_punct SHX_pid_str
489 [YSH read] _reply
490 [Oils VM] OILS_VERSION LIB_YSH
491 OILS_GC_THRESHOLD OILS_GC_ON_EXIT
492 OILS_GC_STATS OILS_GC_STATS_FD
493 OILS_LOCALE_OK
494 [libc locale] ysh-locale
495 [Interactive] OILS_COMP_UI YSH_HISTFILE
496 [Float] NAN INFINITY
497 [Module] __provide__
498 [Other Env] HOME PATH
499)zZXx");
500
501GLOBAL_STR(gStr29, R"zZXx(Standard Library
502
503 [math] abs() max() min() X round()
504 sum()
505 [list] all() any() repeat()
506 [yblocks] yb-capture yb-capture-2
507 [args] parser flag arg rest
508 parseArgs()
509 [binascii] X toBase16() X fromBase16()
510 X toBase64() X fromBase64()
511)zZXx");
512
513GLOBAL_STR(gStr30, R"zZXx(Types and Methods
514
515 [Atoms] Null null
516 Bool expr/true expr/false
517 [Numbers] Int
518 Float
519 Range
520 [String] Str find() findLast()
521 contains() replace()
522 trim() trimStart() trimEnd()
523 startsWith() endsWith()
524 upper() lower()
525 search() leftMatch()
526 split() lines()
527 [Patterns] Eggex
528 Match group() start() end()
529 X groups() X groupDict()
530 [Containers] List List/append() pop() extend()
531 indexOf() lastIndexOf() X includes()
532 insert() remove()
533 reverse() List/clear()
534 Dict erase() X Dict/clear() X accum()
535 X update()
536 Place setValue()
537 [Code Types] Func
538 BuiltinFunc
539 BoundFunc
540 Proc docComment()
541 BuiltinProc
542 [Objects] Obj __invoke__ new
543 X __call__ __index__ X __str__
544 [Reflection] Command sourceCode()
545 Expr
546 Frame
547 DebugFrame toString()
548 io stdin io/eval() io/evalExpr()
549 captureStdout() captureAll() promptVal()
550 X time() X strftime() X glob()
551 vm getFrame() getDebugStack() id()
552)zZXx");
553
554GLOBAL_STR(gStr31, R"zZXx(bin/ysh is the shell with data tYpes, influenced by pYthon, JavaScript, ...
555
556Usage:
557 ysh FLAG* SCRIPT ARG*
558 ysh FLAG* -c COMMAND ARG*
559 ysh FLAG*
560
561Examples:
562 ysh -c 'echo hi'
563 ysh myscript.ysh
564 echo 'echo hi' | ysh
565
566Note that bin/ysh is the same as bin/osh with the ysh:all option group set:
567 osh -o ysh:all -c 'echo hi' # Same as YSH
568)zZXx");
569
570GLOBAL_STR(gStr32, R"zZXx(Word Language
571
572 [Glob] ysh-glob *.py
573 [Joining] ysh-word-join --flag='value' PATH="value"
574 [Quotes] ysh-string "x is $x" $"x is $x" r'[a-z]\n'
575 u'line\n' b'byte \yff'
576 triple-quoted """ $""" r''' u''' b'''
577 X tagged-str "<span id=$x>"html
578 [Substitutions] expr-sub echo $[42 + a[i]]
579 expr-splice echo @[split(x)]
580 var-splice @myarray @ARGV
581 command-sub $(date)
582 command-splice @(cat my-j8-lines.txt)
583 [Formatting] X ysh-printf ${x %.3f}
584 X ysh-format ${x|html}
585)zZXx");
586
587GLOBAL_STR(gStr33, R"zZXx(YSH Command Language Keywords
588
589 [Assignment] const var Declare variables
590 setvar setvar a[i] = 42
591 setglobal setglobal d.key = 'foo'
592 [Expression] equal = = 1 + 2*3
593 call call mylist->append(42)
594 [Definitions] proc proc p (s, ...rest) {
595 typed proc p (; typed, ...rest; n=0; b) {
596 func func f(x; opt1, opt2) { return (x + 1) }
597 ysh-return return (myexpr)
598)zZXx");
599
600GLOBAL_STR(gStr34, R"zZXx(# Can we define methods in pure YSH?
601#
602# (mylist->find(42) !== -1)
603#
604# instead of
605#
606# ('42' in mylist)
607#
608# Because 'in' is for Dict
609
610func find (haystack List, needle) {
611 for i, x in (haystack) {
612 if (x === needle) {
613 return (i)
614 }
615 }
616 return (-1)
617}
618)zZXx");
619
620GLOBAL_STR(gStr35, R"zZXx(# Bash strict mode, updated for 2024
621
622set -o nounset
623set -o pipefail
624set -o errexit
625shopt -s inherit_errexit
626shopt -s strict:all 2>/dev/null || true # dogfood for OSH
627
628)zZXx");
629
630GLOBAL_STR(gStr36, R"zZXx(# Library to turn a shell file into a "BYO test server"
631#
632# Usage:
633#
634# # from both bash and OSH
635# if test -z "$LIB_OSH"; then LIB_OSH=stdlib/osh; fi
636# source $LIB_OSH/byo-server-lib.sh
637#
638# The client creates a clean process state and directory state for each tests.
639#
640# (This file requires compgen -A, and maybe declare -f, so it's not POSIX
641# shell.)
642
643: ${LIB_OSH:-stdlib/osh}
644source $LIB_OSH/two.sh
645
646# List all functions defined in this file (and not in sourced files).
647_bash-print-funcs() {
648 ### Print shell functions in this file that don't start with _ (bash reflection)
649
650 local funcs
651 funcs=($(compgen -A function))
652
653 # extdebug makes `declare -F` print the file path, but, annoyingly, only
654 # if you pass the function names as arguments.
655 shopt -s extdebug
656
657 # bash format:
658 # func1 1 path1
659 # func2 2 path2 # where 2 is the linen umber
660
661 #declare -F "${funcs[@]}"
662
663 # TODO: do we need to normalize the LHS and RHS of $3 == path?
664 declare -F "${funcs[@]}" | awk -v "path=$0" '$3 == path { print $1 }'
665
666 shopt -u extdebug
667}
668
669_gawk-print-funcs() {
670 ### Print shell functions in this file that don't start with _ (awk parsing)
671
672 # Using gawk because it has match()
673 # - doesn't start with _
674
675 # space = / ' '* /
676 # shfunc = / %begin
677 # <capture !['_' ' '] ![' ']*>
678 # '()' space '{' space
679 # %end /
680 # docstring = / %begin
681 # space '###' ' '+
682 # <capture dot*>
683 # %end /
684 gawk '
685 match($0, /^([^_ ][^ ]*)\(\)[ ]*{[ ]*$/, m) {
686 #print NR " shfunc " m[1]
687 print m[1]
688 #print m[0]
689 }
690
691 match($0, /^[ ]*###[ ]+(.*)$/, m) {
692 print NR " docstring " m[1]
693 }
694' $0
695}
696
697_print-funcs() {
698 _bash-print-funcs
699 return
700
701 # TODO: make gawk work, with docstrings
702 if command -v gawk > /dev/null; then
703 _gawk-print-funcs
704 else
705 _bash-print-funcs
706 fi
707}
708
709
710byo-maybe-run() {
711 local command=${BYO_COMMAND:-}
712
713 case $command in
714 '')
715 # Do nothing if it's not specified
716 return
717 ;;
718
719 detect)
720 # all the commands supported, except 'detect'
721 echo list-tests
722 echo run-test
723
724 exit 66 # ASCII code for 'B' - what the protocol specifies
725 ;;
726
727 list-tests)
728 # TODO: use _bash-print-funcs? This fixes the transitive test problem,
729 # which happened in soil/web-remote-test.sh
730 # But it should work with OSH, not just bash! We need shopt -s extdebug
731 compgen -A function | grep '^test-'
732 exit 0
733 ;;
734
735 run-test)
736 local test_name=${BYO_ARG:-}
737 if test -z "$test_name"; then
738 die "BYO run-test: Expected BYO_ARG"
739 fi
740
741 # Avoid issues polluting recursive calls!
742 unset BYO_COMMAND BYO_ARG
743
744 # Shell convention: we name functions test-*
745 "$test_name"
746
747 # Only run if not set -e. Either way it's equivalent
748 exit $?
749 ;;
750
751 *)
752 die "Invalid BYO command '$command'"
753 ;;
754 esac
755
756 # Do nothing if BYO_COMMAND is not set.
757 # The program continues to its "main".
758}
759
760byo-must-run() {
761 local command=${BYO_COMMAND:-}
762 if test -z "$command"; then
763 die "Expected BYO_COMMAND= in environment"
764 fi
765
766 byo-maybe-run
767}
768)zZXx");
769
770GLOBAL_STR(gStr37, R"zZXx(#!/usr/bin/env bash
771#
772# Testing library for bash and OSH.
773#
774# Capture status/stdout/stderr, and nq-assert those values.
775
776: ${LIB_OSH=stdlib/osh}
777source $LIB_OSH/two.sh
778
779nq-assert() {
780 ### Assertion with same syntax as shell 'test'
781
782 if ! test "$@"; then
783 die "line ${BASH_LINENO[0]}: nq-assert $(printf '%q ' "$@") failed"
784 fi
785}
786
787# Problem: we want to capture status and stdout at the same time
788#
789# We use:
790#
791# __stdout=$(set -o errexit; "$@")
792# __status=$?
793#
794# However, we lose the trailing \n, since that's how command subs work.
795
796# Here is another possibility:
797#
798# shopt -s lastpipe # need this too
799# ( set -o errexit; "$@" ) | read -r -d __stdout
800# __status=${PIPESTATUS[0]}
801# shopt -u lastpipe
802#
803# But this feels complex for just the \n issue, which can be easily worked
804# around.
805
806nq-run() {
807 ### capture status only
808
809 local -n out_status=$1
810 shift
811
812 local __status
813
814 # Tricky: turn errexit off so we can capture it, but turn it on against
815 set +o errexit
816 ( set -o errexit; "$@" )
817 __status=$?
818 set -o errexit
819
820 out_status=$__status
821}
822
823nq-capture() {
824 ### capture status and stdout
825
826 local -n out_status=$1
827 local -n out_stdout=$2
828 shift 2
829
830 local __status
831 local __stdout
832
833 # Tricky: turn errexit off so we can capture it, but turn it on against
834 set +o errexit
835 __stdout=$(set -o errexit; "$@")
836 __status=$?
837 set -o errexit
838
839 out_status=$__status
840 out_stdout=$__stdout
841}
842
843nq-capture-2() {
844 ### capture status and stderr
845
846 # This is almost identical to the above
847
848 local -n out_status=$1
849 local -n out_stderr=$2
850 shift 2
851
852 local __status
853 local __stderr
854
855 # Tricky: turn errexit off so we can capture it, but turn it on against
856 set +o errexit
857 __stderr=$(set -o errexit; "$@" 2>&1)
858 __status=$?
859 set -o errexit
860
861 out_status=$__status
862 out_stderr=$__stderr
863}
864
865# 'byo test' can set this?
866: ${NQ_TEST_TEMP=/tmp}
867
868nq-redir() {
869 ### capture status and stdout
870
871 local -n out_status=$1
872 local -n out_stdout_file=$2
873 shift 2
874
875 local __status
876 local __stdout_file=$NQ_TEST_TEMP/nq-redir-$$.txt
877
878 # Tricky: turn errexit off so we can capture it, but turn it on against
879 set +o errexit
880 ( set -o errexit; "$@" ) > $__stdout_file
881 __status=$?
882 set -o errexit
883
884 out_status=$__status
885 out_stdout_file=$__stdout_file
886}
887
888nq-redir-2() {
889 ### capture status and stdout
890
891 local -n out_status=$1
892 local -n out_stderr_file=$2
893 shift 2
894
895 local __status
896 local __stderr_file=$NQ_TEST_TEMP/nq-redir-$$.txt
897
898 # Tricky: turn errexit off so we can capture it, but turn it on against
899 set +o errexit
900 ( set -o errexit; "$@" ) 2> $__stderr_file
901 __status=$?
902 set -o errexit
903
904 out_status=$__status
905 out_stderr_file=$__stderr_file
906}
907)zZXx");
908
909GLOBAL_STR(gStr38, R"zZXx(#!/usr/bin/env bash
910#
911# Common shell functions for task scripts.
912#
913# Usage:
914# source $LIB_OSH/task-five.sh
915#
916# test-foo() { # define task functions
917# echo foo
918# }
919# task-five "$@"
920
921# Definition of a "task"
922#
923# - File invokes task-five "$@"
924# - or maybe you can look at its source
925# - It's a shell function
926# - Has ### docstring
927# - Doesn't start with _
928
929: ${LIB_OSH=stdlib/osh}
930source $LIB_OSH/byo-server.sh
931
932_show-help() {
933 # TODO:
934 # - Use awk to find comments at the top of the file?
935 # - Use OSH to extract docstrings
936 # - BYO_COMMAND=list-tasks will reuse that logic? It only applies to the
937 # current file, not anything in a different file?
938
939 echo "Usage: $0 TASK_NAME ARGS..."
940 echo
941 echo "To complete tasks, run:"
942 echo " source devtools/completion.bash"
943 echo
944 echo "Tasks:"
945
946 if command -v column >/dev/null; then
947 _print-funcs | column
948 else
949 _print-funcs
950 fi
951}
952
953task-five() {
954 # Respond to BYO_COMMAND=list-tasks, etc. All task files need this.
955 byo-maybe-run
956
957 case ${1:-} in
958 ''|--help|-h)
959 _show-help
960 exit 0
961 ;;
962 esac
963
964 if ! declare -f "$1" >/dev/null; then
965 echo "$0: '$1' isn't an action in this task file. Try '$0 --help'"
966 exit 1
967 fi
968
969 "$@"
970}
971)zZXx");
972
973GLOBAL_STR(gStr39, R"zZXx(# Two functions I actually use, all the time.
974#
975# To keep depenedencies small, this library will NEVER grow other functions
976# (and is named to imply that.)
977#
978# Usage:
979# source --builtin two.sh
980#
981# Examples:
982# log 'hi'
983# die 'expected a number'
984
985if command -v source-guard >/dev/null; then # include guard for YSH
986 source-guard two || return 0
987fi
988
989log() {
990 ### Write a message to stderr.
991 echo "$@" >&2
992}
993
994die() {
995 ### Write an error message with the script name, and exit with status 1.
996 log "$0: fatal: $@"
997 exit 1
998}
999
1000)zZXx");
1001
1002GLOBAL_STR(gStr40, R"zZXx(# These were helpful while implementing args.ysh
1003# Maybe we will want to export them in a prelude so that others can use them too?
1004#
1005# Prior art: Rust has `todo!()` which is quite nice. Other languages allow
1006# users to `raise NotImplmentedError()`.
1007
1008# Andy comments:
1009# - 'pass' can be : or true in shell. It's a little obscure / confusing, but
1010# there is an argument for minimalism. Although I prefer words like 'true',
1011# and that already means something.
1012# - UPDATE: we once took 'pass' as a keyword, but users complained because
1013# there is a command 'pass'. So we probably can't have this by default.
1014# Need to discuss source --builtin.
1015
1016# - todo could be more static? Rust presumably does it at compile time
1017
1018proc todo () {
1019 ## Raises a not implemented error when run.
1020 error ("TODO: not implemented") # TODO: is error code 1 ok?
1021}
1022
1023proc pass () {
1024 ## Use when you want to temporarily leave a block empty.
1025 _ null
1026}
1027)zZXx");
1028
1029GLOBAL_STR(gStr41, R"zZXx(# args.ysh
1030#
1031# Usage:
1032# source --builtin args.sh
1033
1034const __provide__ = :| parser parseArgs |
1035
1036#
1037#
1038# parser (&spec) {
1039# flag -v --verbose (help="Verbosely") # default is Bool, false
1040#
1041# flag -P --max-procs (Int, default=-1, doc='''
1042# Run at most P processes at a time
1043# ''')
1044#
1045# flag -i --invert (Bool, default=true, doc='''
1046# Long multiline
1047# Description
1048# ''')
1049#
1050# arg src (help='Source')
1051# arg dest (help='Dest')
1052# arg times (help='Foo')
1053#
1054# rest files
1055# }
1056#
1057# var args = parseArgs(spec, ARGV)
1058#
1059# echo "Verbose $[args.verbose]"
1060
1061# TODO: See list
1062# - flag builtin:
1063# - handle only long flag or only short flag
1064# - flag aliases
1065# - assert that default value has the declared type
1066
1067proc parser (; place ; ; block_def) {
1068 ## Create an args spec which can be passed to parseArgs.
1069 ##
1070 ## Example:
1071 ##
1072 ## # NOTE: &spec will create a variable named spec
1073 ## parser (&spec) {
1074 ## flag -v --verbose (Bool)
1075 ## }
1076 ##
1077 ## var args = parseArgs(spec, ARGV)
1078
1079 var p = {flags: [], args: []}
1080 ctx push (p) {
1081 call io->eval(block_def, vars={flag, arg, rest})
1082 }
1083
1084 # Validate that p.rest = [name] or null and reduce p.rest into name or null.
1085 if ('rest' in p) {
1086 if (len(p.rest) > 1) {
1087 error '`rest` was called more than once' (code=3)
1088 } else {
1089 setvar p.rest = p.rest[0]
1090 }
1091 } else {
1092 setvar p.rest = null
1093 }
1094
1095 var names = {}
1096 for items in ([p.flags, p.args]) {
1097 for x in (items) {
1098 if (x.name in names) {
1099 error "Duplicate flag/arg name $[x.name] in spec" (code=3)
1100 }
1101
1102 setvar names[x.name] = null
1103 }
1104 }
1105
1106 # TODO: what about `flag --name` and then `arg name`?
1107
1108 call place->setValue(p)
1109}
1110
1111const kValidTypes = [Bool, Float, List[Float], Int, List[Int], Str, List[Str]]
1112const kValidTypeNames = []
1113for vt in (kValidTypes) {
1114 var name = vt.name if ('name' in propView(vt)) else vt.unique_id
1115 call kValidTypeNames->append(name)
1116}
1117
1118func isValidType (type) {
1119 for valid in (kValidTypes) {
1120 if (type is valid) {
1121 return (true)
1122 }
1123 }
1124 return (false)
1125}
1126
1127proc flag (short, long ; type=Bool ; default=null, help=null) {
1128 ## Declare a flag within an `arg-parse`.
1129 ##
1130 ## Examples:
1131 ##
1132 ## arg-parse (&spec) {
1133 ## flag -v --verbose
1134 ## flag -n --count (Int, default=1)
1135 ## flag -p --percent (Float, default=0.0)
1136 ## flag -f --file (Str, help="File to process")
1137 ## flag -e --exclude (List[Str], help="File to exclude")
1138 ## }
1139
1140 if (type !== null and not isValidType(type)) {
1141 var type_names = ([null] ++ kValidTypeNames) => join(', ')
1142 error "Expected flag type to be one of: $type_names" (code=2)
1143 }
1144
1145 # Bool has a default of false, not null
1146 if (type is Bool and default === null) {
1147 setvar default = false
1148 }
1149
1150 var name = long => trimStart('--')
1151
1152 ctx emit flags ({short, long, name, type, default, help})
1153}
1154
1155proc arg (name ; ; help=null) {
1156 ## Declare a positional argument within an `arg-parse`.
1157 ##
1158 ## Examples:
1159 ##
1160 ## arg-parse (&spec) {
1161 ## arg name
1162 ## arg config (help="config file path")
1163 ## }
1164
1165 ctx emit args ({name, help})
1166}
1167
1168proc rest (name) {
1169 ## Take the remaining positional arguments within an `arg-parse`.
1170 ##
1171 ## Examples:
1172 ##
1173 ## arg-parse (&grepSpec) {
1174 ## arg query
1175 ## rest files
1176 ## }
1177
1178 # We emit instead of set to detect multiple invocations of "rest"
1179 ctx emit rest (name)
1180}
1181
1182func parseArgs(spec, argv) {
1183 ## Given a spec created by `parser`. Parse an array of strings `argv` per
1184 ## that spec.
1185 ##
1186 ## See `parser` for examples of use.
1187
1188 var i = 0
1189 var positionalPos = 0
1190 var argc = len(argv)
1191 var args = {}
1192 var rest = []
1193
1194 var value
1195 var found
1196 var escape_remaining = false
1197 while (i < argc) {
1198 var arg = argv[i]
1199
1200 if (escape_remaining) {
1201 call rest->append(arg)
1202 } elif (arg === '--') {
1203 if (not spec.rest) {
1204 error "Unexpected '--' argument - extraneous positional arguments are prohibited" (code=2)
1205 }
1206
1207 setvar escape_remaining = true
1208 } elif (arg.startsWith('-')) {
1209 setvar found = false
1210
1211 for flag in (spec.flags) {
1212 if ( (flag.short and flag.short === arg) or
1213 (flag.long and flag.long === arg) ) {
1214 if (flag.type === null or flag.type is Bool) {
1215 setvar value = true
1216 } elif (flag.type is Int) {
1217 setvar i += 1
1218 if (i >= len(argv)) {
1219 error "Expected Int after '$arg'" (code=2)
1220 }
1221
1222 try { setvar value = int(argv[i]) }
1223 if (_status !== 0) {
1224 error "Expected Int after '$arg', got '$[argv[i]]'" (code=2)
1225 }
1226 } elif (flag.type is List[Int]) {
1227 setvar i += 1
1228 if (i >= len(argv)) {
1229 error "Expected Int after '$arg'" (code=2)
1230 }
1231
1232 setvar value = get(args, flag.name, [])
1233 try { call value->append(int(argv[i])) }
1234 if (_status !== 0) {
1235 error "Expected Int after '$arg', got '$[argv[i]]'" (code=2)
1236 }
1237 } elif (flag.type is Float) {
1238 setvar i += 1
1239 if (i >= len(argv)) {
1240 error "Expected Float after '$arg'" (code=2)
1241 }
1242
1243 try { setvar value = float(argv[i]) }
1244 if (_status !== 0) {
1245 error "Expected Float after '$arg', got '$[argv[i]]'" (code=2)
1246 }
1247 } elif (flag.type is List[Float]) {
1248 setvar i += 1
1249 if (i >= len(argv)) {
1250 error "Expected Float after '$arg'" (code=2)
1251 }
1252
1253 setvar value = get(args, flag.name, [])
1254 try { call value->append(float(argv[i])) }
1255 if (_status !== 0) {
1256 error "Expected Float after '$arg', got '$[argv[i]]'" (code=2)
1257 }
1258 } elif (flag.type is Str) {
1259 setvar i += 1
1260 if (i >= len(argv)) {
1261 error "Expected Str after '$arg'" (code=2)
1262 }
1263
1264 setvar value = argv[i]
1265 } elif (flag.type is List[Str]) {
1266 setvar i += 1
1267 if (i >= len(argv)) {
1268 error "Expected Str after '$arg'" (code=2)
1269 }
1270
1271 setvar value = get(args, flag.name, [])
1272 call value->append(argv[i])
1273 }
1274
1275 setvar args[flag.name] = value
1276 setvar found = true
1277 break
1278 }
1279 }
1280
1281 if (not found) {
1282 error "Unknown flag '$arg'" (code=2)
1283 }
1284 } elif (positionalPos >= len(spec.args)) {
1285 if (not spec.rest) {
1286 error "Too many arguments, unexpected '$arg'" (code=2)
1287 }
1288
1289 call rest->append(arg)
1290 } else {
1291 var pos = spec.args[positionalPos]
1292 setvar positionalPos += 1
1293 setvar value = arg
1294 setvar args[pos.name] = value
1295 }
1296
1297 setvar i += 1
1298 }
1299
1300 if (spec.rest) {
1301 setvar args[spec.rest] = rest
1302 }
1303
1304 # Set defaults for flags
1305 for flag in (spec.flags) {
1306 if (flag.name not in args) {
1307 setvar args[flag.name] = flag.default
1308 }
1309 }
1310
1311 # Raise error on missing args
1312 for arg in (spec.args) {
1313 if (arg.name not in args) {
1314 error "Usage Error: Missing required argument $[arg.name]" (code=2)
1315 }
1316 }
1317
1318 return (args)
1319}
1320)zZXx");
1321
1322GLOBAL_STR(gStr42, R"zZXx(const __provide__ = :| Dict |
1323
1324proc Dict ( ; out; ; block) {
1325 var d = io->evalToDict(block)
1326 call out->setValue(d)
1327}
1328
1329)zZXx");
1330
1331GLOBAL_STR(gStr43, R"zZXx(const __provide__ = :| any all repeat |
1332
1333func any(list) {
1334 ### Returns true if any value in the list is truthy.
1335 # Empty list: returns false
1336
1337 for item in (list) {
1338 if (item) {
1339 return (true)
1340 }
1341 }
1342 return (false)
1343}
1344
1345func all(list) {
1346 ### Returns true if all values in the list are truthy.
1347 # Empty list: returns true
1348
1349 for item in (list) {
1350 if (not item) {
1351 return (false)
1352 }
1353 }
1354 return (true)
1355}
1356
1357func repeat(x, n) {
1358 ### Repeats a given Str or List, returning another Str or List
1359
1360 # Like Python's 'foo'*3 or ['foo', 'bar']*3
1361 # negative numbers are like 0 in Python
1362
1363 var t = type(x)
1364 case (t) {
1365 Str {
1366 var parts = []
1367 for i in (0 ..< n) {
1368 call parts->append(x)
1369 }
1370 return (join(parts))
1371 }
1372 List {
1373 var result = []
1374 for i in (0 ..< n) {
1375 call result->extend(x)
1376 }
1377 return (result)
1378 }
1379 (else) {
1380 error "Expected Str or List, got $t"
1381 }
1382 }
1383}
1384)zZXx");
1385
1386GLOBAL_STR(gStr44, R"zZXx(const __provide__ = :| identity max min abs sum |
1387
1388func identity(x) {
1389 ### The identity function. Returns its argument.
1390
1391 return (x)
1392}
1393
1394func __math_select(list, cmp) {
1395 ## Internal helper for `max` and `min`.
1396 ##
1397 ## NOTE: If `list` is empty, then an error is thrown.
1398
1399 if (len(list) === 0) {
1400 error "Unexpected empty list" (code=3)
1401 }
1402
1403 if (len(list) === 1) {
1404 return (list[0])
1405 }
1406
1407 var match = list[0]
1408 for i in (1 ..< len(list)) {
1409 setvar match = cmp(list[i], match)
1410 }
1411 return (match)
1412}
1413
1414func max(...args) {
1415 ## Compute the maximum of 2 or more values.
1416 ##
1417 ## `max` takes two different signatures:
1418 ## - `max(a, b)` to return the maximum of `a`, `b`
1419 ## - `max(list)` to return the greatest item in the `list`
1420 ##
1421 ## So, for example:
1422 ##
1423 ## max(1, 2) # => 2
1424 ## max([1, 2, 3]) # => 3
1425
1426 case (len(args)) {
1427 (1) { return (__math_select(args[0], max)) }
1428 (2) {
1429 if (args[0] > args[1]) {
1430 return (args[0])
1431 } else {
1432 return (args[1])
1433 }
1434 }
1435 (else) { error "max expects 1 or 2 args" (code=3) }
1436 }
1437}
1438
1439func min(...args) {
1440 ## Compute the minimum of 2 or more values.
1441 ##
1442 ## `min` takes two different signatures:
1443 ## - `min(a, b)` to return the minimum of `a`, `b`
1444 ## - `min(list)` to return the least item in the `list`
1445 ##
1446 ## So, for example:
1447 ##
1448 ## min(2, 3) # => 2
1449 ## max([1, 2, 3]) # => 1
1450
1451 case (len(args)) {
1452 (1) { return (__math_select(args[0], min)) }
1453 (2) {
1454 if (args[0] < args[1]) {
1455 return (args[0])
1456 } else {
1457 return (args[1])
1458 }
1459 }
1460 (else) { error "min expects 1 or 2 args" (code=3) }
1461 }
1462}
1463
1464func abs(x) {
1465 ## Compute the absolute (positive) value of a number (float or int).
1466
1467 if (x < 0) {
1468 return (-x)
1469 } else {
1470 return (x)
1471 }
1472}
1473
1474func sum(list; start=0) {
1475 ### Returns the sum of all elements in the list.
1476 # Empty list: returns 0
1477
1478 var sum = start
1479 for item in (list) {
1480 setvar sum += item
1481 }
1482 return (sum)
1483}
1484)zZXx");
1485
1486GLOBAL_STR(gStr45, R"zZXx(const __provide__ = :| sh shell ninja make |
1487
1488# Issues for quoting:
1489#
1490# 1. What is the alphabet we're quoting to?
1491# - Output can be "all bytes", "all unicode strings", or "ASCII"
1492# 2. Start with a simple algorithm to quote everything
1493# - POSIX shell may take ' to ''\'''
1494# 3. Heuristic that may avoid quotes, to make it more readable
1495# - But really it should be \'
1496# - If the ' appears at the beginning or the end, we could have a different
1497# algorithm. Or we could strip leading and trailing ''
1498# 4. Are there any byte strings / unicode strings that can't be quoted?
1499# - e.g. NUL bytes?
1500# - for JSON/JS, binary strings? Only Unicode strings can be encoded.
1501#
1502# Builtins:
1503# toJson() toJson8()
1504# toJ8Line() - does the "maybe unquoted" logic
1505#
1506# Related functions:
1507# encode.base{16,32,64} - Crockford has a base32
1508# decode.base{16,32,64}
1509# Also base85, and maybe base58 base36
1510#
1511# In Python: bin() oct() hex() int(i, 9) and %o %x (there is no %b)
1512#
1513# Other:
1514# Punycode for Unicode domain names uses xn-- ?
1515# CSS has escapes with \
1516# HTTP Cookies have "" and \?
1517#
1518# Related:
1519# demo/url-search-params.ysh is the PARSER / unquoter for quote.urlParam()
1520
1521func sh(s) {
1522 ### Quote POSIX sh string
1523
1524 # replace ' with sequence ' \' '
1525 # Note: the \\ is hard to read - '' doesn't help
1526 return ("'" ++ s.replace("'", "'\\''") ++ "'")
1527}
1528
1529func shell(s) {
1530 ### Quote shell string, where bash and zsh style $'\n' is allowed
1531
1532 # TODO: Binding for j8_lite.MaybeShellEncode / ShellEncode
1533 return (toJson(s))
1534}
1535
1536func ysh(s) {
1537 ### Quote YSH string as b'\n'
1538
1539 # TODO: Binding for j8_lite.YshEncode(unquoted_ok)
1540 return (toJson(s))
1541}
1542
1543#
1544# Build Tools
1545#
1546
1547func make(s) {
1548 var out = []
1549 var n = len(s)
1550 for i in (0 ..< n) {
1551 var ch = s[i]
1552
1553 case (ch) {
1554 (\r) |
1555 (\n) {
1556 error "Can't quote newlines for Make?"
1557 }
1558 ('$') {
1559 call out->append('$')
1560 call out->append('$')
1561 }
1562 ('%') |
1563 (r'\') |
1564 # glob characters
1565 ('[') | (']') | ('*') | ('?') {
1566 call out->append(r'\')
1567 call out->append(ch)
1568 }
1569 (else) {
1570 call out->append(ch)
1571 }
1572 }
1573 }
1574 return (join(out, ''))
1575}
1576
1577# https://ninja-build.org/manual.html#ref_lexer
1578# $ escapes newline, space, : and $
1579# and then the rest is interpreted by /bin/sh
1580func ninja(s) {
1581 var out = []
1582 var n = len(s)
1583 for i in (0 ..< n) {
1584 var ch = s[i]
1585
1586 case (ch) {
1587 # Subtlety: Ninja allows $ to escape a newline, but it's only for
1588 # line continuations - for breaking long lists of files.
1589 # - A file itself should not have a newline.
1590 # - It strips literal newlines out of shell commands.
1591 # So disallow it
1592 (\r) |
1593 (\n) {
1594 error "Can't quote newlines for Ninja"
1595 }
1596 ('$') |
1597 (' ') |
1598 #('#') | # Ninja has no way to escape comments!
1599 (':') {
1600 call out->append('$')
1601 call out->append(ch)
1602 }
1603 (else) {
1604 call out->append(ch)
1605 }
1606 }
1607 }
1608 return (join(out, ''))
1609}
1610
1611# I don't know the rule here - uses \?
1612func cmake(s) {
1613 return (s)
1614}
1615
1616#
1617# Tables, Objects, Documents
1618#
1619
1620func csv(s) {
1621 # double up " etc.
1622 return (s)
1623}
1624
1625func sql(s) {
1626 # double up ' etc.
1627 return (s)
1628}
1629
1630# quote.json is just toJson()
1631
1632func html(s) {
1633 ### Quote shell string, where bash and zsh style $'\n' is allowed
1634
1635 # Binding for j8_lite.MaybeShellEncode / ShellEncode
1636 return (toJson(s))
1637}
1638
1639#
1640# Web
1641#
1642
1643func urlParam(s) {
1644 # urllib.quote
1645 # 'foo bar %' -> 'foo+bar %AB'
1646 return (toJson(s))
1647}
1648
1649#
1650# Programming Languages
1651#
1652
1653# Python 2 or 3
1654func py(s) {
1655 return (s)
1656}
1657
1658# C or C++
1659# Can it be Unicode?
1660func c(s) {
1661 return (s)
1662}
1663
1664# quote.js is just toJson() ?
1665# But it can't handle binary strings?
1666# We can make a table?
1667
1668#
1669# Windows
1670#
1671
1672# We want something that works everywhere -- it should never DOUBLE-ESCAPE, but
1673# it can "unnecessarily" ESCAPE certain characters.
1674
1675# C runtime
1676func win32_crt(s) {
1677 return (s)
1678}
1679
1680# win32 cmd.exe batch files?
1681func win32_cmd(s) {
1682 return (s)
1683}
1684
1685# batch different than cmd.exe?
1686func win32_batch(s) {
1687 return (s)
1688}
1689)zZXx");
1690
1691GLOBAL_STR(gStr46, R"zZXx(# stream.ysh
1692#
1693# Usage:
1694# source --builtin stream.ysh
1695#
1696# For reading lines, decoding, extracting, splitting
1697
1698shopt --unset no_osh_builtins # allow OSH 'local' for now
1699source $LIB_OSH/byo-server.sh # make this file a test server
1700
1701source $LIB_YSH/args.ysh
1702
1703proc slurp-by (; num_lines) {
1704 var buf = []
1705 for line in (io.stdin) {
1706 call buf->append(line)
1707 if (len(buf) === num_lines) {
1708 json write (buf, space=0)
1709
1710 # TODO:
1711 #call buf->clear()
1712 setvar buf = []
1713 }
1714 }
1715 if (buf) {
1716 json write (buf, space=0)
1717 }
1718}
1719
1720proc test-slurp-by {
1721 seq 8 | slurp-by (3)
1722}
1723
1724### Awk
1725
1726# Naming
1727#
1728# TEXT INPUT
1729# each-word # this doesn't go by lines, it does a global regex split or something?
1730#
1731# LINE INPUT
1732# each-line --j8 { echo "-- $_line" } # similar to @()
1733# each-line --j8 (^"-- $_line") # is this superfluous?
1734#
1735# each-split name1 name2
1736# (delim=' ')
1737# (ifs=' ')
1738# (pat=/d+/)
1739# # also assign names for each part?
1740#
1741# each-match # regex match
1742# must-match # assert that every line matches
1743#
1744# TABLE INPUT
1745# each-row # TSV and TSV8 input?
1746#
1747# They all take templates or blocks?
1748
1749proc each-line (...words; template=null; ; block=null) {
1750 # TODO:
1751 # parse --j8 --max-jobs flag
1752
1753 # parse template_str as string
1754 # TODO: this is dangerous though ... because you can execute code
1755 # I think you need a SAFE version
1756
1757 # evaluate template string expression - I guess that allows $(echo hi) and so
1758 # forth
1759
1760 # evaluate block with _line binding
1761 # block: execute in parallel with --max-jobs
1762
1763 for line in (stdin) {
1764 echo TODO
1765 }
1766}
1767
1768proc test-each-line {
1769 echo 'TODO: need basic test runner'
1770
1771 # ysh-tool test stream.ysh
1772 #
1773 # Col
1774}
1775
1776proc each-j8-line (; ; ; block) {
1777 for _line in (io.stdin) {
1778 # TODO: fromJ8Line() toJ8Line()
1779 # var _line = fromJson(_line)
1780 call io->eval(block, vars={_line})
1781 }
1782}
1783
1784proc test-each-j8-line {
1785 var lines = []
1786 var prefix = 'z'
1787
1788 # unquoted
1789 seq 3 | each-j8-line {
1790 call lines->append(prefix ++ _line)
1791 }
1792 pp test_ (lines)
1793
1794 # Note: no trailing new lines, since they aren't significant in Unix
1795 var expected = ['z1', 'z2', 'z3']
1796 assert [expected === lines]
1797}
1798
1799proc each-row (; ; block) {
1800 echo TODO
1801}
1802
1803proc split-by (; delim; ifs=null; block) {
1804
1805 # TODO: provide the option to bind names? Or is that a separate thing?
1806 # The output of this is "ragged"
1807
1808 for line in (io.stdin) {
1809 #pp (line)
1810 var parts = line.split(delim)
1811 pp (parts)
1812
1813 # variable number
1814 call io->eval(block, dollar0=line, pos_args=parts)
1815 }
1816}
1817
1818proc chop () {
1819 ### alias for split-by
1820 echo TODO
1821}
1822
1823proc test-split-by {
1824 var z = 'z' # test out scoping
1825 var count = 0 # test out mutation
1826
1827 # TODO: need split by space
1828 # Where the leading and trailing are split
1829 # if-split-by(' ') doesn't work well
1830
1831 line-data | split-by (/s+/) {
1832
1833 # how do we deal with nonexistent?
1834 # should we also bind _parts or _words?
1835
1836 echo "$z | $0 | $1 | $z"
1837
1838 setvar count += 1
1839 }
1840 echo "count = $count"
1841}
1842
1843proc must-split-by (; ; ifs=null; block) {
1844 ### like if-split-by
1845
1846 echo TODO
1847}
1848
1849# Naming: each-match, each-split?
1850
1851proc if-match (; pattern, template=null; ; block=null) {
1852 ### like 'grep' but with submatches
1853
1854 for line in (io.stdin) {
1855 var m = line.search(pattern)
1856 if (m) {
1857 #pp asdl_ (m)
1858 #var groups = m.groups()
1859
1860 # Should we also pass _line?
1861
1862 if (block) {
1863 call io->eval(block, dollar0=m.group(0))
1864 } elif (template) {
1865 echo TEMPLATE
1866 } else {
1867 echo TSV
1868 }
1869 }
1870 }
1871
1872 # always succeeds - I think must-match is the one that can fail
1873}
1874
1875proc must-match (; pattern; block) {
1876 ### like if-match
1877
1878 echo TODO
1879}
1880
1881proc line-data {
1882 # note: trailing ''' issue, I should probably get rid of the last line
1883
1884 write --end '' -- '''
1885 prefix 30 foo
1886 oils
1887 /// 42 bar
1888 '''
1889}
1890
1891const pat = /<capture d+> s+ <capture w+>/
1892
1893proc test-if-match {
1894 var z = 'z' # test out scoping
1895 var count = 0 # test out mutation
1896
1897 # Test cases should be like:
1898 # grep: print the matches, or just count them
1899 # sed: print a new line based on submatches
1900 # awk: re-arrange the cols, and also accumulate counters
1901
1902 line-data | if-match (pat) {
1903 echo "$z $0 $z"
1904 # TODO: need pos_args
1905
1906 #echo "-- $2 $1 --"
1907
1908 setvar count += 1
1909 }
1910 echo "count = $count"
1911}
1912
1913proc test-if-match-2 {
1914 # If there's no block or template, it should print out a TSV with:
1915 #
1916 # $0 ...
1917 # $1 $2
1918 # $_line maybe?
1919
1920 #line-data | if-match (pat)
1921
1922 var z = 'z' # scoping
1923 line-data | if-match (pat, ^"$z $0 $z")
1924 line-data | if-match (pat, ^"-- $0 --")
1925}
1926
1927# might be a nice way to write it, not sure if byo.sh can discover it
1928if false {
1929tests 'if-match' {
1930 proc case-block {
1931 echo TODO
1932 }
1933 proc case-template {
1934 echo TODO
1935 }
1936}
1937}
1938
1939# Protocol:
1940#
1941# - The file lists its tests the "actions"
1942# - Then the test harness runs them
1943# - But should it be ENV vars
1944#
1945# - BYO_LIST_TESTS=1
1946# - BYO_RUN_TEST=foo
1947# - $PWD is a CLEAN temp dir, the process doesn't have to do anything
1948
1949# - silent on success, but prints file on output
1950# - OK this makes sense
1951#
1952# The trivial test in Python:
1953#
1954# from test import byo
1955# byo.maybe_main()
1956#
1957# bash library:
1958# source --builtin byo-server.sh
1959#
1960# byo-maybe-main # reads env variables, and then exits
1961#
1962# source --builtin assertions.ysh
1963#
1964# assert-ok 'echo hi'
1965# assert-stdout 'hi' 'echo -n hi'
1966#
1967# "$@"
1968#
1969# Run all tests
1970# util/byo-client.sh run-tests $YSH stdlib/table.ysh
1971# util/byo-client.sh run-tests -f x $YSH stdlib/table.ysh
1972
1973# Clean process
1974# Clean working dir
1975
1976#
1977# Stream Protocol:
1978# #.byo - is this she-dot, that's for a file
1979# Do we need metadata?
1980#
1981
1982# The harness
1983#
1984# It's process based testing.
1985#
1986# Test runner process: bash or OSH (unlike sharness!)
1987# Tested process: any language - bash,
1988#
1989# Key point: you don't have to quote shell code?
1990
1991list-byo-tests() {
1992 echo TODO
1993}
1994
1995run-byo-tests() {
1996 # source it
1997 echo TODO
1998}
1999
2000byo-maybe-run
2001)zZXx");
2002
2003GLOBAL_STR(gStr47, R"zZXx(# table.ysh - Library for tables.
2004#
2005# Usage:
2006# source --builtin table.ysh
2007
2008shopt --unset no_osh_builtins # allow OSH 'local' for now
2009source $LIB_OSH/byo-server.sh # make this file a test server
2010
2011proc table (...words; place; ; block) {
2012 var n = len(words)
2013
2014 # TODO: parse flags
2015 #
2016 # --by-row
2017 # --by-col
2018 #
2019 # Place is optional
2020
2021 if (n === 0) {
2022 echo TODO
2023 return
2024 }
2025
2026 var action = words[0]
2027
2028 # textual operations
2029 case (action) {
2030 cat {
2031 echo todo
2032 }
2033 align {
2034 echo todo
2035 }
2036 tabify {
2037 echo todo
2038 }
2039 tabify {
2040 echo todo
2041 }
2042 header {
2043 echo todo
2044 }
2045 slice {
2046 # this has typed args
2047 # do we do some sort of splat?
2048 echo todo
2049 }
2050 to-tsv {
2051 echo todo
2052 }
2053 }
2054
2055 echo TODO
2056}
2057
2058proc test-table {
2059 return
2060
2061 table (&files1) {
2062 cols num_bytes path
2063 type Int Str
2064
2065 row 10 README.md
2066 row 12 main.py
2067
2068 row (12, 'lib.py')
2069 row (num_bytes=12, path='util.py')
2070 }
2071
2072 # 2 columns - The default is by column?
2073 assert ['Dict' === type(files1)]
2074 assert [2 === len(files1)]
2075
2076 # Same table
2077 table --by-row (&files2) {
2078 cols num_bytes path
2079 type Int Str
2080
2081 row 10 README.md
2082 row 12 main.py
2083
2084 row (12, 'lib.py')
2085 row (num_bytes=12, path='util.py')
2086 }
2087
2088 # 4 rows
2089 assert ['List' === type(files2)]
2090 assert [4 === len(files2)]
2091}
2092
2093# "subcommands" of the dialect
2094
2095proc cols (...names) {
2096 # cols name age
2097 echo TODO
2098}
2099
2100proc types (...types) {
2101 # types - Int? Str?
2102 echo TODO
2103}
2104
2105proc attr (name; ...values) {
2106 # attr units ('-', 'secs')
2107 echo TODO
2108}
2109
2110# is this allowed outside table {} blocks too?
2111proc row {
2112 echo TODO
2113}
2114
2115#
2116# dplyr names
2117#
2118
2119# TODO: can we parse select?
2120
2121proc where {
2122 echo
2123}
2124
2125# TODO: should be able to test argv[0] or something
2126# Or pass to _mutate-transmute
2127
2128proc mutate {
2129 echo TODO
2130}
2131
2132proc transmute {
2133 echo TODO
2134}
2135
2136proc rename {
2137 echo TODO
2138}
2139
2140proc group-by {
2141 echo TODO
2142}
2143
2144proc sort-by {
2145 echo TODO
2146}
2147
2148proc summary {
2149 echo TODO
2150}
2151
2152byo-maybe-run
2153)zZXx");
2154
2155GLOBAL_STR(gStr48, R"zZXx(#!/usr/bin/env bash
2156#
2157# Testing library for bash and OSH.
2158#
2159# Capture status/stdout/stderr, and nq-assert those values.
2160
2161const __provide__ = :| yb-capture yb-capture-2 |
2162
2163: ${LIB_OSH=stdlib/osh}
2164source $LIB_OSH/two.sh
2165
2166# There is no yb-run, because you can just use try { } and inspect _error.code
2167# There is no yb-redir, because you can just use try >$tmp { } and inspect _error.code
2168
2169proc yb-capture(; out; ; block) {
2170 ### capture status and stdout
2171
2172 var stdout = ''
2173 try {
2174 { call io->eval(block) } | read --all (&stdout)
2175
2176 # Note that this doesn't parse because of expression issue:
2177 # call io->eval(block) | read --all (&stdout)
2178 # used to be eval (block)
2179 }
2180 # TODO: if 'block' contains a pipeline, we lose this magic var
2181 var result = {status: _pipeline_status[0], stdout}
2182
2183 #echo 'result-1'
2184 #pp test_ (result)
2185
2186 call out->setValue(result)
2187}
2188
2189proc yb-capture-2(; out; ; block) {
2190 ### capture status and stderr
2191
2192 var stderr = ''
2193 try {
2194 redir 2>&1 { call io->eval(block); } | read --all (&stderr)
2195
2196 # Note that this doesn't parse because of expression issue:
2197 # call io->eval(block) 2>&1 | read --all (&stderr)
2198 # used to be eval (block) 2>&1
2199 }
2200 #pp test_ (_pipeline_status)
2201
2202 var result = {status: _pipeline_status[0], stderr}
2203 #echo 'result-2'
2204 #pp test_ (result)
2205
2206 call out->setValue(result)
2207}
2208)zZXx");
2209
2210
2211
2212TextFile array[] = {
2213 {.rel_path = "_devbuild/help/data-errors", .contents = gStr0},
2214 {.rel_path = "_devbuild/help/data-front-end", .contents = gStr1},
2215 {.rel_path = "_devbuild/help/data-j8-notation", .contents = gStr2},
2216 {.rel_path = "_devbuild/help/help", .contents = gStr3},
2217 {.rel_path = "_devbuild/help/oils-usage", .contents = gStr4},
2218 {.rel_path = "_devbuild/help/osh-builtin-cmd", .contents = gStr5},
2219 {.rel_path = "_devbuild/help/osh-chapters", .contents = gStr6},
2220 {.rel_path = "_devbuild/help/osh-cmd-lang", .contents = gStr7},
2221 {.rel_path = "_devbuild/help/osh-front-end", .contents = gStr8},
2222 {.rel_path = "_devbuild/help/osh-mini-lang", .contents = gStr9},
2223 {.rel_path = "_devbuild/help/osh-option", .contents = gStr10},
2224 {.rel_path = "_devbuild/help/osh-osh-assign", .contents = gStr11},
2225 {.rel_path = "_devbuild/help/osh-plugin", .contents = gStr12},
2226 {.rel_path = "_devbuild/help/osh-special-var", .contents = gStr13},
2227 {.rel_path = "_devbuild/help/osh-stdlib", .contents = gStr14},
2228 {.rel_path = "_devbuild/help/osh-type-method", .contents = gStr15},
2229 {.rel_path = "_devbuild/help/osh-usage", .contents = gStr16},
2230 {.rel_path = "_devbuild/help/osh-word-lang", .contents = gStr17},
2231 {.rel_path = "_devbuild/help/shell-flags", .contents = gStr18},
2232 {.rel_path = "_devbuild/help/ysh-builtin-cmd", .contents = gStr19},
2233 {.rel_path = "_devbuild/help/ysh-builtin-func", .contents = gStr20},
2234 {.rel_path = "_devbuild/help/ysh-chapters", .contents = gStr21},
2235 {.rel_path = "_devbuild/help/ysh-cmd-lang", .contents = gStr22},
2236 {.rel_path = "_devbuild/help/ysh-expr-lang", .contents = gStr23},
2237 {.rel_path = "_devbuild/help/ysh-front-end", .contents = gStr24},
2238 {.rel_path = "_devbuild/help/ysh-mini-lang", .contents = gStr25},
2239 {.rel_path = "_devbuild/help/ysh-option", .contents = gStr26},
2240 {.rel_path = "_devbuild/help/ysh-plugin", .contents = gStr27},
2241 {.rel_path = "_devbuild/help/ysh-special-var", .contents = gStr28},
2242 {.rel_path = "_devbuild/help/ysh-stdlib", .contents = gStr29},
2243 {.rel_path = "_devbuild/help/ysh-type-method", .contents = gStr30},
2244 {.rel_path = "_devbuild/help/ysh-usage", .contents = gStr31},
2245 {.rel_path = "_devbuild/help/ysh-word-lang", .contents = gStr32},
2246 {.rel_path = "_devbuild/help/ysh-ysh-cmd", .contents = gStr33},
2247 {.rel_path = "stdlib/methods.ysh", .contents = gStr34},
2248 {.rel_path = "stdlib/osh/bash-strict.sh", .contents = gStr35},
2249 {.rel_path = "stdlib/osh/byo-server.sh", .contents = gStr36},
2250 {.rel_path = "stdlib/osh/no-quotes.sh", .contents = gStr37},
2251 {.rel_path = "stdlib/osh/task-five.sh", .contents = gStr38},
2252 {.rel_path = "stdlib/osh/two.sh", .contents = gStr39},
2253 {.rel_path = "stdlib/prelude.ysh", .contents = gStr40},
2254 {.rel_path = "stdlib/ysh/args.ysh", .contents = gStr41},
2255 {.rel_path = "stdlib/ysh/def.ysh", .contents = gStr42},
2256 {.rel_path = "stdlib/ysh/list.ysh", .contents = gStr43},
2257 {.rel_path = "stdlib/ysh/math.ysh", .contents = gStr44},
2258 {.rel_path = "stdlib/ysh/quote.ysh", .contents = gStr45},
2259 {.rel_path = "stdlib/ysh/stream.ysh", .contents = gStr46},
2260 {.rel_path = "stdlib/ysh/table.ysh", .contents = gStr47},
2261 {.rel_path = "stdlib/ysh/yblocks.ysh", .contents = gStr48},
2262
2263 {.rel_path = nullptr, .contents = nullptr},
2264};
2265
2266} // namespace embedded_file
2267
2268TextFile* gEmbeddedFiles = embedded_file::array; // turn array into pointer